From 29d522eb19995ea084330867d7d37bb945e9dc7a Mon Sep 17 00:00:00 2001 From: zanefisher Date: Mon, 22 Jul 2013 17:22:44 -0400 Subject: [PATCH 001/119] Merge branch 'master', remote-tracking branch 'upstream/master' From 3557d594b6239eb9c4803c36565c3c5b29ac856a Mon Sep 17 00:00:00 2001 From: zanefisher Date: Tue, 23 Jul 2013 15:09:33 -0400 Subject: [PATCH 002/119] Revert "Print, and log, messages in command-line utilities." --- tuf/repo/signercli.py | 20 ++++---------------- tuf/tests/system_tests/util_test_tools.py | 4 ++-- tuf/tests/test_download.py | 11 +++++++---- tuf/tests/test_push.py | 4 ++-- tuf/tests/test_signerlib.py | 2 +- 5 files changed, 16 insertions(+), 25 deletions(-) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index 4aaed950..106d2af5 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -103,7 +103,6 @@ def _get_password(prompt='Password: ', confirm=False): else: message = 'Mismatch; try again.' logger.info(message) - print(message) @@ -211,15 +210,13 @@ def _list_keyids(keystore_directory, metadata_directory): if keyid in keyids: keyids_dict[keyid].append(targets_role) - # Print the keyids without the '.key' extension and the roles + # Log the keyids without the '.key' extension and the roles # associated with them. message = 'Listing the keyids in '+repr(keystore_directory) logger.info(message) - print(message) for keyid in keyids_dict: message = keyid+' : '+str(keyids_dict[keyid]) logger.info(message) - print(message) @@ -262,7 +259,6 @@ def _get_keyids(keystore_directory): if keyid not in loaded_keyid: message = 'Could not load keyid: '+keyid logger.error(message) - print(message) continue # Append 'keyid' to the loaded list of keyids. @@ -314,7 +310,6 @@ def _get_all_config_keyids(config_filepath, keystore_directory): if not loaded_key or keyid not in loaded_key: message = 'Could not load keyid: '+keyid logger.error(message) - print(message) continue loaded_keyids[key].append(keyid) break @@ -363,7 +358,6 @@ def _get_role_config_keyids(config_filepath, keystore_directory, role): if not loaded_key or keyid not in loaded_key: message = 'Could not load keyid: '+keyid logger.error(message) - print(message) continue role_keyids.append(keyid) break @@ -524,7 +518,7 @@ def generate_rsa_key(keystore_directory): try: rsa_key = save_rsa_key(keystore_directory=keystore_directory, password=password, bits=rsa_key_bits) - print('Generated a new key: '+rsa_key['keyid']) + logger.info('Generated a new key: '+rsa_key['keyid']) except (tuf.FormatError, tuf.CryptoError), e: message = 'The RSA key could not be generated. '+str(e)+'\n' raise tuf.RepositoryError(message) @@ -639,7 +633,6 @@ def dump_key(keystore_directory): message = '*WARNING* Printing the private key reveals' \ ' sensitive information *WARNING*' logger.warning(message) - print(message) input = _prompt(prompt, str) if input.lower() == 'private': show_private = True @@ -653,7 +646,7 @@ def dump_key(keystore_directory): raise tuf.RepositoryError(message) # Print the contents of the key metadata. - print(json.dumps(key_metadata, indent=2, sort_keys=True)) + logger.info(json.dumps(key_metadata, indent=2, sort_keys=True)) @@ -935,7 +928,6 @@ def sign_metadata_file(keystore_directory): # Retrieve the keyids of the signing keys from the user. message = 'The keyids that will sign the metadata file must be loaded.' logger.info(message) - print(message) loaded_keyids = _get_keyids(keystore_directory) if len(loaded_keyids) == 0: @@ -1016,7 +1008,7 @@ def make_delegation(keystore_directory): elif recursive_walk == 'N': recursive_walk = False else: - print("Sorry, I could not understand that; please try again.") + logger.warn("Sorry, I could not understand that; please try again.") # Get all the target roles and their respective keyids. # These keyids will let the user know which roles are currently known. @@ -1088,7 +1080,6 @@ def _load_parent_role(metadata_directory, keystore_directory, targets_roles): if parent_role not in targets_roles: message = 'Invalid role name entered' logger.info(message) - print(message) parent_role = None continue else: @@ -1110,7 +1101,6 @@ def _load_parent_role(metadata_directory, keystore_directory, targets_roles): if keyid not in loaded_keyid: message = 'The keyid could not be loaded.' logger.info(message) - print(message) continue parent_keyids.append(loaded_keyid[0]) break @@ -1140,7 +1130,6 @@ def _get_delegated_role(keystore_directory, metadata_directory): # Retrieve the delegated role\'s keyids from the user. message = 'The keyid of the delegated role must be loaded.' logger.info(message) - print(message) delegated_keyids = _get_keyids(keystore_directory) # Ensure at least one delegated key was loaded. @@ -1168,7 +1157,6 @@ def _make_delegated_metadata(metadata_directory, delegated_paths, parent_role, message = 'There are '+str(len(delegated_paths))+' target paths for '+\ str(delegated_role) logger.info(message) - print(message) # Create, sign, and write the delegated role's metadata file. # The first time a parent role creates a delegation, a directory diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/system_tests/util_test_tools.py index 7fc79c93..7b23ff1b 100755 --- a/tuf/tests/system_tests/util_test_tools.py +++ b/tuf/tests/system_tests/util_test_tools.py @@ -144,7 +144,7 @@ import tuf.repo.signerlib as signerlib import tuf.repo.keystore as keystore - +logger = logging.getLogger('tuf') # Disable logging for cleaner output. def disable_logging(): @@ -199,7 +199,7 @@ def cleanup(root_repo, server_process=None): if server_process.returncode is None: server_process.kill() - print 'Server terminated.\n' + logger.info('Server terminated.\n') # Clear the keystore. keystore.clear_keystore() diff --git a/tuf/tests/test_download.py b/tuf/tests/test_download.py index cb773e78..44b3c732 100755 --- a/tuf/tests/test_download.py +++ b/tuf/tests/test_download.py @@ -22,6 +22,7 @@ """ import tuf +import tuf.log import tuf.download as download import tuf.tests.unittest_toolbox as unittest_toolbox @@ -36,6 +37,8 @@ import SocketServer import SimpleHTTPServer +logger = logging.getLogger('tuf') + # Disable/Enable logging. Comment-out to Enable logging. logging.getLogger('tuf') logging.disable(logging.CRITICAL) @@ -61,9 +64,9 @@ def setUp(self): self.PORT = random.randint(30000, 45000) command = ['python', 'simple_server.py', str(self.PORT)] self.server_proc = subprocess.Popen(command, stderr=subprocess.PIPE) - print '\n\tServer process started.' - print '\tServer process id: '+str(self.server_proc.pid) - print '\tServing on port: '+str(self.PORT) + logger.info('\n\tServer process started.') + logger.info('\tServer process id: '+str(self.server_proc.pid)) + logger.info('\tServing on port: '+str(self.PORT)) junk, rel_target_filepath = os.path.split(target_filepath) self.url = 'http://localhost:'+str(self.PORT)+'/'+rel_target_filepath @@ -83,7 +86,7 @@ def setUp(self): def tearDown(self): unittest_toolbox.Modified_TestCase.tearDown(self) if self.server_proc.returncode is None: - print '\tServer process '+str(self.server_proc.pid)+' terminated.' + logger.info('\tServer process '+str(self.server_proc.pid)+' terminated.') self.server_proc.kill() self.target_fileobj.close() diff --git a/tuf/tests/test_push.py b/tuf/tests/test_push.py index ef5d7bfc..6b3b0609 100644 --- a/tuf/tests/test_push.py +++ b/tuf/tests/test_push.py @@ -32,7 +32,7 @@ class TestPush(unittest.TestCase): src_push_dict = {} - print scp.transfer + logger.info(scp.transfer) ORIGINAL_PUSH_CONFIG = pushtoolslib.PUSH_CONFIG @@ -161,4 +161,4 @@ def test_expected_behaviour_of_push_with_scp(self): # Run the unittests suite = unittest.TestLoader().loadTestsFromTestCase(TestPush) -unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file +unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index ececc08e..efdf208b 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -880,7 +880,7 @@ def _get_role_info(self, role, directory=None): return timestamp_meta, role_keyids, meta_dir else: - print '\nUnrecognized top-level role.' + logger.warning('\nUnrecognized top-level role.') From 04d96e62f134a90c2353431447d8eae06f1d3ca4 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Tue, 30 Jul 2013 14:41:07 -0400 Subject: [PATCH 003/119] Fixed various tests. --- setup.py | 2 +- quickstart.py => tuf/repo/quickstart.py | 0 tuf/tests/aggregate_tests.py | 27 ++++++++----------------- tuf/tests/test_hash.py | 15 +++++++++++--- tuf/tests/test_push.py | 7 ++++++- tuf/tests/test_quickstart.py | 2 +- 6 files changed, 28 insertions(+), 25 deletions(-) rename quickstart.py => tuf/repo/quickstart.py (100%) diff --git a/setup.py b/setup.py index 83e9e87c..4175561b 100755 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ 'tuf.tests' ], scripts=[ - 'quickstart.py', + 'tuf/repo/quickstart.py', 'tuf/pushtools/push.py', 'tuf/pushtools/receivetools/receive.py', 'tuf/repo/signercli.py' diff --git a/quickstart.py b/tuf/repo/quickstart.py similarity index 100% rename from quickstart.py rename to tuf/repo/quickstart.py diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index df1305d8..cf65a398 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -16,22 +16,11 @@ """ -# The unit tests listed below should provide their respective test suites -# and run on import. -import tuf.tests.test_download -import tuf.tests.test_formats -import tuf.tests.test_hash -import tuf.tests.test_keydb -import tuf.tests.test_keystore -import tuf.tests.test_mirrors -import tuf.tests.test_quickstart -import tuf.tests.test_roledb -import tuf.tests.test_rsa_key -import tuf.tests.test_schema -import tuf.tests.test_signercli -import tuf.tests.test_signerlib -import tuf.tests.test_sig -import tuf.tests.test_util -import tuf.tests.test_updater -import tuf.tests.system_tests.test_util_test_tools -import tuf.tests.system_tests.test_replay_attack +import glob + +tests_list = glob.glob('test_*.py') +for test in tests_list: + __import__(test[:-3]) + +import system_tests.test_util_test_tools +import system_tests.test_replay_attack diff --git a/tuf/tests/test_hash.py b/tuf/tests/test_hash.py index ba47f2d1..38537ec7 100755 --- a/tuf/tests/test_hash.py +++ b/tuf/tests/test_hash.py @@ -19,17 +19,26 @@ import os import StringIO +import logging import tempfile import unittest import tuf.hash +logger = logging.getLogger('tuf') + +if not 'hashlib' in tuf.hash._supported_libraries: + logger.warn('Not testing hashlib: could not be imported.') +if not 'pycrypto' in tuf.hash._supported_libraries: + logger.warn('Not testing pycrypto: could not be imported.') class TestHash(unittest.TestCase): def _run_with_all_hash_libraries(self, test_func): - test_func('hashlib') - test_func('pycrypto') + if 'hashlib' in tuf.hash._supported_libraries: + test_func('hashlib') + if 'pycrypto' in tuf.hash._supported_libraries: + test_func('pycrypto') def test_md5_update(self): @@ -217,4 +226,4 @@ def _do_update_file_obj(self, library): # Run unit test. suite = unittest.TestLoader().loadTestsFromTestCase(TestHash) -unittest.TextTestRunner(verbosity=2).run(suite) +unittest.TextTestRunner(verbosity=3).run(suite) diff --git a/tuf/tests/test_push.py b/tuf/tests/test_push.py index 6b3b0609..4de06b1e 100644 --- a/tuf/tests/test_push.py +++ b/tuf/tests/test_push.py @@ -18,6 +18,7 @@ import os import getpass +import logging import tempfile import unittest import ConfigParser @@ -26,9 +27,13 @@ import tuf.pushtools.push as push import tuf.pushtools.transfer.scp as scp import tuf.pushtools.pushtoolslib as pushtoolslib -import tuf.tests.system_tests.util_test_tools as util_test_tools +import system_tests.util_test_tools as util_test_tools +logger = logging.getLogger('tuf') +# Disable all logging calls of level CRITICAL and below. +# Comment the line below to enable logging. +logging.disable(logging.CRITICAL) class TestPush(unittest.TestCase): src_push_dict = {} diff --git a/tuf/tests/test_quickstart.py b/tuf/tests/test_quickstart.py index 2c2220af..831941d6 100755 --- a/tuf/tests/test_quickstart.py +++ b/tuf/tests/test_quickstart.py @@ -25,8 +25,8 @@ import shutil import unittest import logging +import tuf.repo.quickstart as quickstart -import quickstart import tuf.util import tuf.tests.unittest_toolbox From 2b8d654cebf6ce001548114e7af69d86bbadfccb Mon Sep 17 00:00:00 2001 From: zanefisher Date: Wed, 31 Jul 2013 19:01:19 -0400 Subject: [PATCH 004/119] Tests no longer run automatically when imported. aggregate_tests now loads all the unit tests into one suite and runs them together, so that any failures and errors show up together in a concise report. --- tuf/tests/aggregate_tests.py | 17 ++++++++++++----- tuf/tests/test_download.py | 4 ++-- tuf/tests/test_formats.py | 4 ++-- tuf/tests/test_hash.py | 4 ++-- tuf/tests/test_keydb.py | 4 ++-- tuf/tests/test_keystore.py | 4 ++-- tuf/tests/test_mirrors.py | 4 ++-- tuf/tests/test_push.py | 4 ++-- tuf/tests/test_pushtoolslib.py | 4 ++-- tuf/tests/test_quickstart.py | 4 ++-- tuf/tests/test_roledb.py | 5 +++-- tuf/tests/test_rsa_key.py | 4 ++-- tuf/tests/test_schema.py | 4 ++-- tuf/tests/test_sig.py | 4 ++-- tuf/tests/test_signercli.py | 15 +++++++++------ tuf/tests/test_signerlib.py | 15 +++++++++------ tuf/tests/test_updater.py | 30 +++++++++++++++++------------- tuf/tests/test_util.py | 4 ++-- 18 files changed, 76 insertions(+), 58 deletions(-) diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index cf65a398..39fe8457 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -4,6 +4,7 @@ Konstantin Andrianov + Zane Fisher January 26, 2013 @@ -12,15 +13,21 @@ See LICENSE for licensing information. - Run all the unit tests in 'tuf/tests'. + Run all the unit tests from every .py file beginning with "test_" in 'tuf/tests'. """ +import unittest import glob +import tuf.keydb as keydb +import tuf.repo.keystore as keystore +import tuf.roledb as roledb tests_list = glob.glob('test_*.py') -for test in tests_list: - __import__(test[:-3]) -import system_tests.test_util_test_tools -import system_tests.test_replay_attack +# Remove '.py' from each filename. +tests_list = [test[:-3] for test in tests_list] + +suite = unittest.TestLoader().loadTestsFromNames(tests_list) + +unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tuf/tests/test_download.py b/tuf/tests/test_download.py index 44b3c732..96716875 100755 --- a/tuf/tests/test_download.py +++ b/tuf/tests/test_download.py @@ -186,5 +186,5 @@ def test_download_url_to_tempfileobj(self): # Run unit test. -suite = unittest.TestLoader().loadTestsFromTestCase(TestDownload) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_formats.py b/tuf/tests/test_formats.py index 2e41e2f6..14b5281d 100755 --- a/tuf/tests/test_formats.py +++ b/tuf/tests/test_formats.py @@ -628,5 +628,5 @@ def test_encode_canonical(self): # Run unit test. -suite = unittest.TestLoader().loadTestsFromTestCase(TestFormats) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_hash.py b/tuf/tests/test_hash.py index 38537ec7..0a56c175 100755 --- a/tuf/tests/test_hash.py +++ b/tuf/tests/test_hash.py @@ -225,5 +225,5 @@ def _do_update_file_obj(self, library): # Run unit test. -suite = unittest.TestLoader().loadTestsFromTestCase(TestHash) -unittest.TextTestRunner(verbosity=3).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_keydb.py b/tuf/tests/test_keydb.py index fe61a976..5dedc411 100755 --- a/tuf/tests/test_keydb.py +++ b/tuf/tests/test_keydb.py @@ -217,5 +217,5 @@ def test_create_keydb_from_root_metadata(self): # Run unit test. -suite = unittest.TestLoader().loadTestsFromTestCase(TestKeydb) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_keystore.py b/tuf/tests/test_keystore.py index a78acbb6..196bb7d4 100755 --- a/tuf/tests/test_keystore.py +++ b/tuf/tests/test_keystore.py @@ -321,5 +321,5 @@ def test_internal_decrypt(self): # Run the unit tests. -suite = unittest.TestLoader().loadTestsFromTestCase(TestKeystore) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_mirrors.py b/tuf/tests/test_mirrors.py index 2c2e0e9d..4afca814 100755 --- a/tuf/tests/test_mirrors.py +++ b/tuf/tests/test_mirrors.py @@ -97,5 +97,5 @@ def test_get_list_of_mirrors(self): # Run the unittests -suite = unittest.TestLoader().loadTestsFromTestCase(TestMirrors) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_push.py b/tuf/tests/test_push.py index 4de06b1e..699881c0 100644 --- a/tuf/tests/test_push.py +++ b/tuf/tests/test_push.py @@ -165,5 +165,5 @@ def test_expected_behaviour_of_push_with_scp(self): # Run the unittests -suite = unittest.TestLoader().loadTestsFromTestCase(TestPush) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_pushtoolslib.py b/tuf/tests/test_pushtoolslib.py index a1938cd6..60c4975f 100644 --- a/tuf/tests/test_pushtoolslib.py +++ b/tuf/tests/test_pushtoolslib.py @@ -165,5 +165,5 @@ def test_exceptions_handeling_of_read_config_file(self): # Run the unittests -suite = unittest.TestLoader().loadTestsFromTestCase(TestPushtoolslib) -unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_quickstart.py b/tuf/tests/test_quickstart.py index 831941d6..771f397e 100755 --- a/tuf/tests/test_quickstart.py +++ b/tuf/tests/test_quickstart.py @@ -191,5 +191,5 @@ def _remove_repository_directories(repo_dir, keystore_dir, client_dir): # Run the unit tests. -suite = unittest.TestLoader().loadTestsFromTestCase(TestQuickstart) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_roledb.py b/tuf/tests/test_roledb.py index 63a34198..1154dc57 100755 --- a/tuf/tests/test_roledb.py +++ b/tuf/tests/test_roledb.py @@ -65,6 +65,7 @@ def test_clear_roledb(self): def test_add_role(self): # Test conditions where the arguments are valid. + print('ROLES: '+str(tuf.roledb._roledb_dict.keys())) self.assertEqual(0, len(tuf.roledb._roledb_dict)) rolename = 'targets' roleinfo = {'keyids': ['123'], 'threshold': 1} @@ -396,5 +397,5 @@ def _test_rolename(self, test_function): # Run the unit tests. -suite = unittest.TestLoader().loadTestsFromTestCase(TestRoledb) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_rsa_key.py b/tuf/tests/test_rsa_key.py index e63cca7d..d6543378 100755 --- a/tuf/tests/test_rsa_key.py +++ b/tuf/tests/test_rsa_key.py @@ -180,5 +180,5 @@ def test_verify_signature(self): # Run the unit tests. -suite = unittest.TestLoader().loadTestsFromTestCase(TestRsa_key) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_schema.py b/tuf/tests/test_schema.py index d263fe03..f72ae02b 100755 --- a/tuf/tests/test_schema.py +++ b/tuf/tests/test_schema.py @@ -326,5 +326,5 @@ def test_RegularExpression(self): # Run the unit tests. -suite = unittest.TestLoader().loadTestsFromTestCase(TestSchema) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_sig.py b/tuf/tests/test_sig.py index f60acfa3..dd302748 100755 --- a/tuf/tests/test_sig.py +++ b/tuf/tests/test_sig.py @@ -391,5 +391,5 @@ def test_signable_has_invalid_format(self): # Run unit test. -suite = unittest.TestLoader().loadTestsFromTestCase(TestSig) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index 97a56f25..ccf07741 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -1484,9 +1484,12 @@ def _mock_get_keyids(junk): # Run unit tests. -loader = unittest_toolbox.unittest.TestLoader -suite = loader().loadTestsFromTestCase(TestSignercli) -try: - unittest_toolbox.unittest.TextTestRunner(verbosity=2).run(suite) -finally: - unittest_toolbox.Modified_TestCase.clear_toolbox() +#loader = unittest_toolbox.unittest.TestLoader +#suite = loader().loadTestsFromTestCase(TestSignercli) +#try: +# unittest_toolbox.unittest.TextTestRunner(verbosity=2).run(suite) +#finally: +# unittest_toolbox.Modified_TestCase.clear_toolbox() + +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index efdf208b..cdda3bbe 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -898,9 +898,12 @@ def _get_signed_role_info(self, role, directory=None): # Run unit test. -suite = unittest.TestLoader().loadTestsFromTestCase(TestSignerlib) -try: - unittest.TextTestRunner(verbosity=2).run(suite) -finally: - unit_tbox.clear_toolbox() - tuf.repo.keystore.clear_keystore() +#suite = unittest.TestLoader().loadTestsFromTestCase(TestSignerlib) +#try: +# unittest.TextTestRunner(verbosity=2).run(suite) +#finally: +# unit_tbox.clear_toolbox() +# tuf.repo.keystore.clear_keystore() + +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index 75c7dc77..74a1d19a 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -408,7 +408,7 @@ def test_2__import_delegations(self): # Verify that there was no change in roledb and keydb dictionaries # by checking the number of elements in the dictionaries. - self.assertEqual(len(roledb._roledb_dict), 5) + self.assertEqual(len(roledb._roledb_dict), 5) self.assertEqual(len(keydb._keydb_dict), 5) # Test: normal case, first level delegation. @@ -1142,18 +1142,22 @@ def test_8_remove_obsolete_targets(self): # Run all unit tests. -loader = unittest_toolbox.unittest.TestLoader() -suite = unittest_toolbox.unittest.TestSuite() +#loader = unittest_toolbox.unittest.TestLoader() +#suite = unittest_toolbox.unittest.TestSuite() -class1_tests = loader.loadTestsFromTestCase(TestUpdater_init_) -class2_tests = loader.loadTestsFromTestCase(TestUpdater) +#class1_tests = loader.loadTestsFromTestCase(TestUpdater_init_) +#class2_tests = loader.loadTestsFromTestCase(TestUpdater) -suite.addTest(class1_tests) -suite.addTest(class2_tests) +#suite.addTest(class1_tests) +#suite.addTest(class2_tests) -try: - unittest_toolbox.unittest.TextTestRunner(verbosity=2).run(suite) -finally: - # Removing repositories. - setup.remove_all_repositories(TestUpdater.repositories['main_repository']) - unittest_toolbox.Modified_TestCase.clear_toolbox() +#try: +# unittest_toolbox.unittest.TextTestRunner(verbosity=2).run(suite) +#finally: +# # Removing repositories. +# setup.remove_all_repositories(TestUpdater.repositories['main_repository']) +# unittest_toolbox.Modified_TestCase.clear_toolbox() + + +if __name__ == '__main__': + unittest.main() diff --git a/tuf/tests/test_util.py b/tuf/tests/test_util.py index ec0ccfb2..9636d606 100755 --- a/tuf/tests/test_util.py +++ b/tuf/tests/test_util.py @@ -302,5 +302,5 @@ def test_B6_load_json_file(self): # Run unit test. -suite = unittest.TestLoader().loadTestsFromTestCase(TestUtil) -unittest.TextTestRunner(verbosity=2).run(suite) +if __name__ == '__main__': + unittest.main() From dd0e9cf2c1fe4290c2e5f0e442aa1d5c769fecf9 Mon Sep 17 00:00:00 2001 From: vladdd Date: Tue, 6 Aug 2013 13:42:44 -0400 Subject: [PATCH 005/119] Minor changes following review --- tuf/repo/quickstart.py | 4 ---- tuf/tests/aggregate_tests.py | 3 +++ tuf/tests/test_roledb.py | 1 - tuf/tests/test_signercli.py | 8 -------- tuf/tests/test_signerlib.py | 9 --------- tuf/tests/test_updater.py | 20 -------------------- 6 files changed, 3 insertions(+), 42 deletions(-) diff --git a/tuf/repo/quickstart.py b/tuf/repo/quickstart.py index 2215995f..c04df034 100755 --- a/tuf/repo/quickstart.py +++ b/tuf/repo/quickstart.py @@ -214,7 +214,6 @@ def build_repository(project_directory): except ValueError, e: message = 'Invalid expiration date entered' logger.error(message) - print(message) timeout = None continue @@ -273,7 +272,6 @@ def build_repository(project_directory): metadata_directory = os.path.join(repository_directory, 'metadata') message = 'Creating '+repr(metadata_directory) logger.info(message) - print(message) os.mkdir(metadata_directory) except OSError, e: if e.errno == errno.EEXIST: @@ -312,7 +310,6 @@ def build_repository(project_directory): except ValueError, e: message = 'Invalid role threshold entered' logger.warning(message) - print(message) role_threshold = None continue @@ -371,7 +368,6 @@ def build_repository(project_directory): repr(client_metadata_directory)+'. The client metadata '+\ 'will need to be manually created. See the README file.' logger.warn(message) - print(message) else: raise diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index 39fe8457..e71896b0 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -8,6 +8,9 @@ January 26, 2013 + + August 2013. Modified previous behavior that explicitly imported individual + unit tests. -Zane Fisher See LICENSE for licensing information. diff --git a/tuf/tests/test_roledb.py b/tuf/tests/test_roledb.py index 1154dc57..94e6c25b 100755 --- a/tuf/tests/test_roledb.py +++ b/tuf/tests/test_roledb.py @@ -65,7 +65,6 @@ def test_clear_roledb(self): def test_add_role(self): # Test conditions where the arguments are valid. - print('ROLES: '+str(tuf.roledb._roledb_dict.keys())) self.assertEqual(0, len(tuf.roledb._roledb_dict)) rolename = 'targets' roleinfo = {'keyids': ['123'], 'threshold': 1} diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index ccf07741..2183f6a3 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -1483,13 +1483,5 @@ def _mock_get_keyids(junk): signercli._get_metadata_directory = original_get_metadata_directory -# Run unit tests. -#loader = unittest_toolbox.unittest.TestLoader -#suite = loader().loadTestsFromTestCase(TestSignercli) -#try: -# unittest_toolbox.unittest.TextTestRunner(verbosity=2).run(suite) -#finally: -# unittest_toolbox.Modified_TestCase.clear_toolbox() - if __name__ == '__main__': unittest.main() diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index cdda3bbe..b5c71f35 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -896,14 +896,5 @@ def _get_signed_role_info(self, role, directory=None): return signed_meta, role_info - -# Run unit test. -#suite = unittest.TestLoader().loadTestsFromTestCase(TestSignerlib) -#try: -# unittest.TextTestRunner(verbosity=2).run(suite) -#finally: -# unit_tbox.clear_toolbox() -# tuf.repo.keystore.clear_keystore() - if __name__ == '__main__': unittest.main() diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index 74a1d19a..5c3bff2b 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -1139,25 +1139,5 @@ def test_8_remove_obsolete_targets(self): tuf.download.download_url_to_tempfileobj = original_download - - -# Run all unit tests. -#loader = unittest_toolbox.unittest.TestLoader() -#suite = unittest_toolbox.unittest.TestSuite() - -#class1_tests = loader.loadTestsFromTestCase(TestUpdater_init_) -#class2_tests = loader.loadTestsFromTestCase(TestUpdater) - -#suite.addTest(class1_tests) -#suite.addTest(class2_tests) - -#try: -# unittest_toolbox.unittest.TextTestRunner(verbosity=2).run(suite) -#finally: -# # Removing repositories. -# setup.remove_all_repositories(TestUpdater.repositories['main_repository']) -# unittest_toolbox.Modified_TestCase.clear_toolbox() - - if __name__ == '__main__': unittest.main() From 43db37c2abc25cbe3fd70fb42996b4ca72abf196 Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 7 Aug 2013 02:42:06 -0400 Subject: [PATCH 006/119] Improved checking of the "paths" and "path_hash_prefix" attributes. Removed checking whether "path_hash_prefix" is consistent with the delegated paths in the delegator, because now the delegated paths may list directories instead of simply files. --- docs/tuf-spec.txt | 9 +++------ tuf/formats.py | 15 ++++++++++----- tuf/repo/signercli.py | 38 ++++++++++++++++++-------------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/docs/tuf-spec.txt b/docs/tuf-spec.txt index c6071a89..06c83001 100644 --- a/docs/tuf-spec.txt +++ b/docs/tuf-spec.txt @@ -565,15 +565,12 @@ "name": ROLE, "keyids" : [ KEYID, ... ] , "threshold" : THRESHOLD, - "paths" : [ PATHPATTERN, ... ], - "path_hash_prefix" : HEX_STRING + ("paths" : [ PATHPATTERN, ... ] | "path_hash_prefix" : HEX_DIGEST) }, ... ] } - In order to discuss target paths, a role must specify either one of the - "paths" or "path_hash_prefix" attribute, each of which we discuss next. In - case the client specifies both attributes, "path_hash_prefix" is to be read - from or written to while "paths" should be ignored. + In order to discuss target paths, a role MUST specify only one of the + "paths" or "path_hash_prefix" attributes, each of which we discuss next. The "paths" list describes paths that the role is trusted to provide. Clients MUST check that a target is in one of the trusted paths of all roles diff --git a/tuf/formats.py b/tuf/formats.py index 43487f42..66426794 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -875,12 +875,17 @@ def make_role_metadata(keyids, threshold, name=None, paths=None, role_meta['name'] = name # According to the specification, the 'paths' and 'path_hash_prefix' must be - # mutually exclusive. In case both are specified, 'paths' is ignored while - # 'path_hash_prefix' is recorded. - if path_hash_prefix is not None: - role_meta['path_hash_prefix'] = path_hash_prefix - elif paths is not None: + # mutually exclusive. However, at the time of writing we do not always ensure + # that this is the case with the schema checks (see #83). Therefore, we must + # do it for ourselves. + + if paths is not None and path_hash_prefix is not None: + raise tuf.FormatError('Both "paths" and "path_hash_prefix" are specified!') + + if paths is not None: role_meta['paths'] = paths + elif path_hash_prefix is not None: + role_meta['path_hash_prefix'] = path_hash_prefix # Does 'role_meta' have the correct type? # This check ensures 'role_meta' conforms to diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index 3b598477..60c0f778 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -1146,7 +1146,8 @@ def make_delegation(keystore_directory): # Update the parent role's metadata file. The parent role's delegation # field must be updated with the newly created delegated role. _update_parent_metadata(metadata_directory, delegated_role, delegated_keyids, - delegated_paths, parent_role, parent_keyids) + parent_role, parent_keyids, + delegated_paths=delegated_paths) @@ -1337,39 +1338,36 @@ def _make_delegated_metadata(metadata_directory, delegated_targets, def _update_parent_metadata(metadata_directory, delegated_role, - delegated_keyids, delegated_paths, parent_role, - parent_keyids, path_hash_prefix=None): + delegated_keyids, parent_role, + parent_keyids, delegated_paths=None, + path_hash_prefix=None): """ Update the parent role's metadata file. The delegations field of the metadata file is updated with the key and role information belonging to the newly added delegated role. Finally, the metadata file is signed and written to the metadata directory. - If the optional 'path_hash_prefix' is specified with the required - 'delegated_paths', then 'path_hash_prefix' is checked to be consistent with - 'delegated_paths', and then 'path_hash_prefix', instead of - 'delegated_paths', is written to the parent role metadata file. Otherwise, - 'delegated_paths' is written to the parent role metadata file in - the absense of 'path_hash_prefix'. - """ + # According to the specification, the 'paths' and 'path_hash_prefix' must be + # mutually exclusive. However, at the time of writing we do not always ensure + # that this is the case with the schema checks (see #83). Therefore, we must + # do it for ourselves. + + if delegated_paths is not None and path_hash_prefix is not None: + raise tuf.RepositoryError('Both "paths" and "path_hash_prefix" are ' \ + 'specified!') + + if delegated_paths is None and path_hash_prefix is None: + raise tuf.RepositoryError('Neither "paths" nor`"path_hash_prefix" is ' \ + 'specified!') + # The 'delegated_paths' are relative to 'repository'. # The 'relative_paths' are relative to 'repository/targets'. relative_paths = [] for path in delegated_paths: relative_paths.append(os.path.sep.join(path.split(os.path.sep)[1:])) - if path_hash_prefix: - # Ensure that 'delegated_paths' is consistent with 'path_hash_prefix'. - if not tuf.repo.signerlib.paths_are_consistent_with_hash_prefix( - relative_paths, path_hash_prefix): - raise tuf.RepositoryError('path_hash_prefix '+str(path_hash_prefix)+ - ' is inconsistent with paths: '+ - str(delegated_paths)) - else: - logger.debug('"path_hash_prefix" is unspecified; reading "paths" instead.') - # Extract the metadata from the parent role's file. parent_filename = os.path.join(metadata_directory, parent_role) parent_filename = parent_filename+'.txt' From 891e0399cc78dd8438de6f6d24dfe59ce500793f Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 7 Aug 2013 05:42:10 -0400 Subject: [PATCH 007/119] Bugfix. --- tuf/repo/signercli.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index 60c0f778..6752b866 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -1364,9 +1364,12 @@ def _update_parent_metadata(metadata_directory, delegated_role, # The 'delegated_paths' are relative to 'repository'. # The 'relative_paths' are relative to 'repository/targets'. - relative_paths = [] - for path in delegated_paths: - relative_paths.append(os.path.sep.join(path.split(os.path.sep)[1:])) + if delegated_paths is None: + relative_paths = None + else: + relative_paths = [] + for path in delegated_paths: + relative_paths.append(os.path.sep.join(path.split(os.path.sep)[1:])) # Extract the metadata from the parent role's file. parent_filename = os.path.join(metadata_directory, parent_role) @@ -1398,16 +1401,11 @@ def _update_parent_metadata(metadata_directory, delegated_role, threshold = len(delegated_keyids) delegated_role = parent_role+'/'+delegated_role - # If the "path_hash_prefix" attribute is available, write it. - # Otherwise, write the "paths" attribute. - if path_hash_prefix is None: - role_metadata = tuf.formats.make_role_metadata(delegated_keyids, threshold, - name=delegated_role, - paths=relative_paths) - else: - role_metadata = tuf.formats.make_role_metadata(delegated_keyids, threshold, - name=delegated_role, - path_hash_prefix=path_hash_prefix) + # Write either the "paths" or the "path_hash_prefix" attribute. + role_metadata = tuf.formats.make_role_metadata(delegated_keyids, threshold, + name=delegated_role, + paths=relative_paths, + path_hash_prefix=path_hash_prefix) # Find the appropriate role to create or update. role_index = tuf.repo.signerlib.find_delegated_role(roles, delegated_role) From dd44dba7cc3378c487e4243e4c45a669bc6157db Mon Sep 17 00:00:00 2001 From: vladdd Date: Wed, 7 Aug 2013 10:45:25 -0400 Subject: [PATCH 008/119] Remove list.sort() and cleanup try-except blocks in signercli.py Previously, _make_delegated_metadata() attempted to minimize the number of target directories in the "paths" field of delegations by calculating common root-most directories. This bahavior was found to be unsafe and removed, and as a result, the sort of delegated targets is no longer needed. --- tuf/repo/signercli.py | 56 ++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index 5b2c624f..d3d7b1d9 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -440,7 +440,7 @@ def _get_metadata_expiration(): If the entered date is valid, it is returned unmodified. - tuf.Error, if the entered expiration date is invalid. + tuf.RepositoryError, if the entered expiration date is invalid. """ @@ -452,11 +452,11 @@ def _get_metadata_expiration(): input_date = input_date+' UTC' expiration_date = tuf.formats.parse_time(input_date) except (tuf.FormatError, ValueError), e: - raise tuf.Error('Invalid date entered.') + raise tuf.RepositoryError('Invalid date entered.') if expiration_date < time.time(): message = 'The expiration date must occur after the current date.' - raise tuf.Error(message) + raise tuf.RepositoryError(message) return input_date @@ -832,12 +832,10 @@ def make_targets_metadata(keystore_directory): # newer. version = _get_metadata_version(targets_filename) - # Prompt the user the metadata file's expiration date. - try: - expiration_date = _get_metadata_expiration() - except tuf.Error, e: - message = str(e)+'\n' - raise tuf.RepositoryError(message) + # Prompt the user the metadata file's expiration date. + # Raise 'tuf.RepositoryError' if invalid date is entered + # by the user. + expiration_date = _get_metadata_expiration() # Get the configuration file. @@ -908,12 +906,10 @@ def make_release_metadata(keystore_directory): # newer. version = _get_metadata_version(release_filename) - # Prompt the user the metadata file's expiration date. - try: - expiration_date = _get_metadata_expiration() - except tuf.Error, e: - message = str(e)+'\n' - raise tuf.RepositoryError(message) + # Prompt the user the metadata file's expiration date. + # Raise 'tuf.RepositoryError' if invalid date is entered + # by the user. + expiration_date = _get_metadata_expiration() # Get the configuration file. config_filepath = _prompt('\nEnter the configuration file path: ', str) @@ -976,12 +972,10 @@ def make_timestamp_metadata(keystore_directory): # newer. version = _get_metadata_version(timestamp_filename) - # Prompt the user the metadata file's expiration date. - try: - expiration_date = _get_metadata_expiration() - except tuf.Error, e: - message = str(e)+'\n' - raise tuf.RepositoryError(message) + # Prompt the user the metadata file's expiration date. + # Raise 'tuf.RepositoryError' if invalid date is entered + # by the user. + expiration_date = _get_metadata_expiration() # Get the configuration file. config_filepath = _prompt('\nEnter the configuration file path: ', str) @@ -1258,11 +1252,9 @@ def _make_delegated_metadata(metadata_directory, delegated_targets, # The 'delegated_paths' list contains either file paths or the paths of # directories. A child role may list any target(s) under a directory or sub- - # directory. Below, sort 'delegated_targets' so that root-most directories - # are easier to calculate (i.e., replicate directory wildcards using - # os.path.commonprefix() instead of regular expressions, which may be abused - # by input carefully-crafted for this purpose). - delegated_targets.sort() + # directory. Replicate directory wildcards using os.path.commonprefix() + # instead of regular expressions, which may be abused by input + # carefully-crafted for this purpose. for path in delegated_targets: path = os.path.abspath(path) relative_path = path[len(repository_directory)+1:] @@ -1288,7 +1280,7 @@ def _make_delegated_metadata(metadata_directory, delegated_targets, # has not been added to 'delegated_paths', nor a parent directory of it. else: delegated_paths.append(relative_path+os.sep) - message = 'There are '+str(len(delegated_filepaths))+' target paths for '+\ + message = 'There are '+repr(len(delegated_filepaths))+' target paths for '+\ repr(delegated_role) logger.info(message) @@ -1314,12 +1306,10 @@ def _make_delegated_metadata(metadata_directory, delegated_targets, else: raise - # Prompt the user for the metadata file's expiration date. - try: - expiration_date = _get_metadata_expiration() - except tuf.Error, e: - message = str(e)+'\n' - raise tuf.RepositoryError(message) + # Prompt the user the metadata file's expiration date. + # Raise 'tuf.RepositoryError' if invalid date is entered + # by the user. + expiration_date = _get_metadata_expiration() # Sign and write the delegated metadata file. delegated_role_filename = delegated_role+'.txt' From 1cd9b1251e90c42ea73fd9975115d1d188d7ccc6 Mon Sep 17 00:00:00 2001 From: vladdd Date: Wed, 7 Aug 2013 15:38:16 -0400 Subject: [PATCH 009/119] Update updater.py to retrieve compressed versions of Targets metadata This change addresses issue #85. The previous implementation only recognized compressed versions of "release.txt". --- tuf/client/updater.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 26fd3d87..97582c4f 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -631,7 +631,7 @@ def _update_metadata(self, metadata_role, fileinfo=None, compression=None): # Construct the metadata filename as expected by the download/mirror modules. metadata_filename = metadata_role + '.txt' - # The 'release' metadata file may be compressed. Add the appropriate + # The 'release' or Targets metadata may be compressed. Add the appropriate # extension to 'metadata_filename'. if compression == 'gzip': metadata_filename = metadata_filename + '.gz' @@ -846,14 +846,25 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas logger.info('Metadata '+repr(metadata_filename)+' has changed.') - # There might be a compressed version of the 'release' metadata - # that may be downloaded. Check the 'meta' field of - # 'referenced_metadata' to see if it is listed. + # There might be a compressed version of 'release.txt' or Targets + # metadata available for download. Check the 'meta' field of + # 'referenced_metadata' to see if it is listed when 'metadata_role' + # is 'release'. Check the 'meta' field of 'release' when 'metadata_role' + # is Targets metadata. The full rolename for delegated Targets metadata + # must begin with 'targets/'. The Release role lists all the Targets + # metadata available on the repository, including any that may be in + # compressed form. compression = None + gzip_path = metadata_filename + '.gz' if metadata_role == 'release': - gzip_path = metadata_filename + '.gz' if gzip_path in self.metadata['current'][referenced_metadata]['meta']: compression = 'gzip' + elif metadata_role.startswith('targets/'): + if gzip_path in self.metadata['current']['release']['meta']: + compression = 'gzip' + else: + message = 'Compressed version of '+repr(metadata_filename)+' not available.' + logger.debug(message) try: self._update_metadata(metadata_role, fileinfo=new_fileinfo, @@ -1503,7 +1514,7 @@ def target(self, target_filepath): In case of an unforeseen runtime error. - The metadata for updated delegated roles are download and stored. + The metadata for updated delegated roles are downloaded and stored. The target information for 'target_filepath', conformant to @@ -1516,7 +1527,7 @@ def target(self, target_filepath): tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) # Refresh the target metadata for all the delegated roles. - self._refresh_targets_metadata(include_delegations=True) + #self._refresh_targets_metadata(include_delegations=True) # The target is assumed to be missing until proven otherwise. target = None From adb5ea003ec895b01fa1a6e7b0c0a0b2f6aec74f Mon Sep 17 00:00:00 2001 From: vladdd Date: Wed, 7 Aug 2013 19:30:22 -0400 Subject: [PATCH 010/119] Preliminary update preceding the major path_hash_prefix changes This update addresses issue #86. It begins by removing the wholesale downloading of all targets metadata and only downloads & verifies the metadata for the roles it only needs; the "lazy walk" scheme. --- tuf/client/updater.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 97582c4f..7d7c633a 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1512,6 +1512,9 @@ def target(self, target_filepath): Exception: In case of an unforeseen runtime error. + + TODO: Update these exceptions once the final 'path_hash_prefix' + changes have been implemented. The metadata for updated delegated roles are downloaded and stored. @@ -1526,8 +1529,13 @@ def target(self, target_filepath): # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) - # Refresh the target metadata for all the delegated roles. - #self._refresh_targets_metadata(include_delegations=True) + # Ensure the client has the most up-to-date version of 'targets.txt'. + # Raise 'tuf.MetadataNotAvailableError' if the changed metadata + # cannot be successfully downloaded and 'tuf.RepositoryError' if the + # referenced metadata is missing. Target methods such as this one + # are called after the top-level metadata have been refreshed (i.e., + # updater.refresh()). + self._update_metadata_if_changed('targets') # The target is assumed to be missing until proven otherwise. target = None @@ -1540,6 +1548,14 @@ def target(self, target_filepath): while len(role_names) > 0 and target is None: # Pop the role name from the top of the stack. role_name = role_names.pop(-1) + + # The metadata for 'role_name' must be downloaded/updated before + # its targets, delegations, and child roles can be inspected. + # self.metadata['current'][role_name] is currently missing. + # _refresh_targets_metadata() does not refresh 'targets.txt', it + # expects _update_metadata_if_changed() to have already refreshed it, + # which this function has checked above. + self._refresh_targets_metadata(role_name, include_delegations=False) role_metadata = current_metadata[role_name] targets = role_metadata['targets'] delegations = role_metadata.get('delegations', {}) From 74120cc507f90633518240b9b658394c827dca0f Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 8 Aug 2013 12:23:40 -0400 Subject: [PATCH 011/119] Update updater.py to also check for compressed versions of targets.txt Also added a comment to make it obvious that referenced_metadata should always be release for delegated roles. --- tuf/client/updater.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 7d7c633a..dce742b5 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -859,8 +859,13 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas if metadata_role == 'release': if gzip_path in self.metadata['current'][referenced_metadata]['meta']: compression = 'gzip' - elif metadata_role.startswith('targets/'): - if gzip_path in self.metadata['current']['release']['meta']: + # Check for available compressed versions of 'targets.txt' and delegated + # Targets, which also start with 'targets'. + elif metadata_role.startswith('targets'): + # For 'targets.txt' and delegated metadata, 'referenced_metata' + # should always be 'release'. 'release.txt' specifies all roles + # provided by a repository, including their file sizes and hashes. + if gzip_path in self.metadata['current'][referenced_metadata]['meta']: compression = 'gzip' else: message = 'Compressed version of '+repr(metadata_filename)+' not available.' From d4a381ce3d2fb2ad2005051986b4df23849880a3 Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 8 Aug 2013 12:52:19 -0400 Subject: [PATCH 012/119] Fix some bugs with tests. --- tuf/formats.py | 1 + tuf/tests/aggregate_tests.py | 2 ++ tuf/tests/test_signercli.py | 2 ++ tuf/tests/test_updater.py | 1 + 4 files changed, 6 insertions(+) diff --git a/tuf/formats.py b/tuf/formats.py index 66426794..776baf4a 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -67,6 +67,7 @@ import string import time +import tuf import tuf.schema as SCHEMA diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index e71896b0..3ac9ecac 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ aggregate_tests.py diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index 99886123..aa91d862 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -39,7 +39,9 @@ class guarantees the order of unit tests. So that, 'test_something_A' import os import time import logging +import unittest +import tuf import tuf.formats import tuf.util import tuf.repo.keystore as keystore diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index 150749e6..d32664e7 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -41,6 +41,7 @@ class guarantees the order of unit tests. So that, 'test_something_A' import shutil import tempfile import logging +import unittest import tuf.util import tuf.formats From 06082bb86cdd88e1c31cebcea0c7616ebebe2677 Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 8 Aug 2013 12:52:19 -0400 Subject: [PATCH 013/119] Fix bugs with a library and a few unit tests. Include a missing import for a library so that it does not throw a runtime error. Include missing imports for unit tests so that they are standalone. --- tuf/formats.py | 5 ++--- tuf/tests/aggregate_tests.py | 2 ++ tuf/tests/test_signercli.py | 2 ++ tuf/tests/test_updater.py | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index c83cdf28..a9ac4db3 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -67,9 +67,8 @@ import string import time -import tuf.schema - -SCHEMA = tuf.schema +import tuf +import tuf.schema as SCHEMA # Note that in the schema definitions below, the 'SCHEMA.Object' types allow diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index e71896b0..3ac9ecac 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ aggregate_tests.py diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index 99886123..aa91d862 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -39,7 +39,9 @@ class guarantees the order of unit tests. So that, 'test_something_A' import os import time import logging +import unittest +import tuf import tuf.formats import tuf.util import tuf.repo.keystore as keystore diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index 150749e6..d32664e7 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -41,6 +41,7 @@ class guarantees the order of unit tests. So that, 'test_something_A' import shutil import tempfile import logging +import unittest import tuf.util import tuf.formats From 4e524a7dbfe0023446cc27339866054a1904de17 Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 8 Aug 2013 14:46:33 -0400 Subject: [PATCH 014/119] Accommodate including compressed versions of release in timestamp. --- tuf/repo/signerlib.py | 112 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 91 insertions(+), 21 deletions(-) diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 640f13f4..298d1612 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -19,6 +19,7 @@ """ +import gzip import os import ConfigParser import logging @@ -493,9 +494,9 @@ def generate_timestamp_metadata(release_filename, version, Conformant to 'tuf.formats.TIME_SCHEMA'. compressions: - Compression extensions (e.g., 'gz' and 'tgz'). If 'release.txt' is also - saved in compressed form, these compression extensions should be stored - in 'compressions' so the compressed timestamp files can be added to the + Compression extensions (e.g., 'gz'). If 'release.txt' is also saved in + compressed form, these compression extensions should be stored in + 'compressions' so the compressed timestamp files can be added to the timestamp metadata object. @@ -524,8 +525,13 @@ def generate_timestamp_metadata(release_filename, version, # Save the file info of the compressed versions of 'timestamp.txt'. for file_extension in compressions: compressed_filename = release_filename + '.' + file_extension - compressed_fileinfo = get_metadata_file_info(compressed_filename) - fileinfo['release.txt.' + file_extension] = compressed_fileinfo + try: + compressed_fileinfo = get_metadata_file_info(compressed_filename) + except: + logger.warn('Could not get fileinfo about '+str(compressed_fileinfo)) + else: + logger.info('Including fileinfo about '+str(compressed_filename)) + fileinfo['release.txt.' + file_extension] = compressed_fileinfo # Generate the timestamp metadata object. timestamp_metadata = tuf.formats.TimestampFile.make_metadata(version, @@ -538,7 +544,7 @@ def generate_timestamp_metadata(release_filename, version, -def write_metadata_file(metadata, filename): +def write_metadata_file(metadata, filename, compression=None): """ Create the file containing the metadata. @@ -551,11 +557,17 @@ def write_metadata_file(metadata, filename): The filename (absolute path) of the metadata to be written (e.g., 'root.txt'). + compression: + Specify an algorithm as a string to compress the file; otherwise, the + file will be left uncompressed. Available options are 'gz' (gzip). + tuf.FormatError, if the arguments are improperly formatted. tuf.Error, if 'filename' doesn't exist. + Any other runtime (e.g. IO) exception. + The 'filename' file is created or overwritten if it exists. @@ -569,20 +581,44 @@ def write_metadata_file(metadata, filename): tuf.formats.SIGNABLE_SCHEMA.check_match(metadata) tuf.formats.PATH_SCHEMA.check_match(filename) - # Split 'filename' into head and tail. Verify that head exists. - check_directory(os.path.split(filename)[0]) + # Verify 'filename' directory. + check_directory(os.path.dirname(filename)) - logger.info('Writing to '+repr(filename)) - file_object = open(filename, 'w') + # We choose a file-like object that depends on the compression algorithm. + file_object = None + # We may modify the filename, depending on the compression algorithm, so we + # store it separately. + filename_with_compression = filename - # The metadata object is saved to 'file_object'. The keys - # of the objects are sorted and indentation is used. - json.dump(metadata, file_object, indent=1, sort_keys=True) + # Take care of compression. + if compression is None: + logger.info('No compression for '+str(filename)) + file_object = open(filename_with_compression, 'w') + elif compression == 'gz': + logger.info('gzip compression for '+str(filename)) + filename_with_compression += '.gz' + file_object = gzip.open(filename_with_compression, 'w') + else: + raise tuf.FormatError('Unknown compression algorithm: '+str(compression)) - file_object.write('\n') - file_object.close() + try: + tuf.formats.PATH_SCHEMA.check_match(filename_with_compression) + logger.info('Writing to '+str(filename_with_compression)) - return filename + # The metadata object is saved to 'file_object'. The keys + # of the objects are sorted and indentation is used. + json.dump(metadata, file_object, indent=1, sort_keys=True) + + file_object.write('\n') + except: + # Raise any runtime exception. + raise + else: + # Otherwise, return the written filename. + return filename_with_compression + finally: + # Always close the file. + file_object.close() @@ -1131,7 +1167,7 @@ def build_targets_file(target_paths, targets_keyids, metadata_directory, def build_release_file(release_keyids, metadata_directory, - version, expiration_date): + version, expiration_date, compress=False): """ Build the release metadata file using the signing keys in 'release_keyids'. @@ -1152,6 +1188,10 @@ def build_release_file(release_keyids, metadata_directory, The expiration date, in UTC, of the metadata file. Conformant to 'tuf.formats.TIME_SCHEMA'. + compress: + Should we *include* a compressed version of the release file? By default, + the answer is no. + tuf.FormatError, if any of the arguments are improperly formatted. @@ -1182,14 +1222,27 @@ def build_release_file(release_keyids, metadata_directory, version, expiration_date) signable = sign_metadata(release_metadata, release_keyids, release_filepath) - return write_metadata_file(signable, release_filepath) + # Should we also include a compressed version of release.txt? + if compress: + # If so, write a gzip version of release.txt. + compressed_written_filepath = \ + write_metadata_file(signable, release_filepath, compression='gz') + logger.info('Wrote '+str(compressed_written_filepath)) + else: + logger.debug('No compressed version of release metadata will be included.') + + written_filepath = write_metadata_file(signable, release_filepath) + logger.info('Wrote '+str(written_filepath)) + + return written_filepath def build_timestamp_file(timestamp_keyids, metadata_directory, - version, expiration_date): + version, expiration_date, + include_compressed_release=True): """ Build the timestamp metadata file using the signing keys in 'timestamp_keyids'. @@ -1209,6 +1262,10 @@ def build_timestamp_file(timestamp_keyids, metadata_directory, expiration_date: The expiration date, in UTC, of the metadata file. Conformant to 'tuf.formats.TIME_SCHEMA'. + + include_compressed_release: + Should the timestamp role *include* compression versions of the release + metadata, if any? We do this by default. tuf.FormatError, if any of the arguments are improperly formatted. @@ -1236,11 +1293,24 @@ def build_timestamp_file(timestamp_keyids, metadata_directory, release_filepath = os.path.join(metadata_directory, RELEASE_FILENAME) timestamp_filepath = os.path.join(metadata_directory, TIMESTAMP_FILENAME) + # Should we include compressed versions of release in timestamp? + compressions = () + if include_compressed_release: + # Presently, we include only gzip versions by default. + compressions = ('gz',) + logger.info('Including '+str(compressions)+' versions of release in '\ + 'timestamp.') + else: + logger.warn('No compressed versions of release will be included in '\ + 'timestamp.') + # Generate and sign the timestamp metadata. timestamp_metadata = generate_timestamp_metadata(release_filepath, version, - expiration_date) - signable = sign_metadata(timestamp_metadata, timestamp_keyids, timestamp_filepath) + expiration_date, + compressions=compressions) + signable = sign_metadata(timestamp_metadata, timestamp_keyids, + timestamp_filepath) return write_metadata_file(signable, timestamp_filepath) From aa6f2b6ddf432914bca2c57ece23136a1ed8863c Mon Sep 17 00:00:00 2001 From: ttgump Date: Thu, 8 Aug 2013 16:24:03 -0400 Subject: [PATCH 015/119] unicode fix in python2 --- tuf/repo/signercli.py | 4 ++++ tuf/tests/system_tests/test_endless_data_attack.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index d3d7b1d9..a4c76f86 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -116,6 +116,10 @@ def _prompt(message, result_type=str): caller. """ + # we need to use unicode in python 2 + python_version = sys.version_info[0] + if python_version==2: + str = unicode return result_type(raw_input(message)) diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 0530f128..99dbd6dd 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -42,11 +42,14 @@ import tuf from tuf.interposition import urllib_tuf +import logging + # Disable logging. util_test_tools.disable_logging() +logger = logging.getLogger('tuf') class EndlessDataAttack(Exception): From c3cc9685940050383f74f8ed30a68bcbc9fad26b Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 8 Aug 2013 17:07:00 -0400 Subject: [PATCH 016/119] Fix bug due to typo. --- tuf/repo/signerlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 298d1612..99b44a00 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -528,7 +528,7 @@ def generate_timestamp_metadata(release_filename, version, try: compressed_fileinfo = get_metadata_file_info(compressed_filename) except: - logger.warn('Could not get fileinfo about '+str(compressed_fileinfo)) + logger.warn('Could not get fileinfo about '+str(compressed_filename)) else: logger.info('Including fileinfo about '+str(compressed_filename)) fileinfo['release.txt.' + file_extension] = compressed_fileinfo From d9c10c3eedeac0e85238a6df69d77f1a3f8d0a1c Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 8 Aug 2013 17:41:46 -0400 Subject: [PATCH 017/119] Try decompressing alleged JSON files with gzip in some cases. --- tuf/tests/test_util.py | 2 +- tuf/util.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tuf/tests/test_util.py b/tuf/tests/test_util.py index 9636d606..aa4f0d9b 100755 --- a/tuf/tests/test_util.py +++ b/tuf/tests/test_util.py @@ -295,7 +295,7 @@ def test_B6_load_json_file(self): util.json.dump(data, fileobj) fileobj.close() self.assertEquals(data, util.load_json_file(filepath)) - Errors = (tuf.FormatError, tuf.Error) + Errors = (tuf.FormatError, IOError) for bogus_arg in ['a', 1, ['a'], {'a':'b'}]: self.assertRaises(Errors, util.load_json_file, bogus_arg) diff --git a/tuf/util.py b/tuf/util.py index 0fb6291d..c1a114a9 100755 --- a/tuf/util.py +++ b/tuf/util.py @@ -518,7 +518,7 @@ def load_json_file(filepath): tuf.FormatError: If 'filepath' is improperly formatted. - tuf.Error: If 'filepath' could not be opened. + IOError in case of runtime IO exceptions. None. @@ -531,13 +531,18 @@ def load_json_file(filepath): # Making sure that the format of 'filepath' is a path string. # tuf.FormatError is raised on incorrect format. tuf.formats.PATH_SCHEMA.check_match(filepath) - - try: + + # The file is mostly likely gzipped. + if filepath.endswith('.gz'): + logger.debug('gzip.open('+str(filepath)+')') + fileobject = gzip.open(filepath) + else: + logger.debug('open('+str(filepath)+')') fileobject = open(filepath) - except IOError, err: - raise tuf.Error(err) try: return json.load(fileobject) finally: fileobject.close() + + From 609bbe084e70fa08f54902fbfe501dcba48c8182 Mon Sep 17 00:00:00 2001 From: vladdd Date: Fri, 9 Aug 2013 08:29:57 -0400 Subject: [PATCH 018/119] Update tuf-spec.txt and implement "lazy bin walk" tuf-spec.txt was updated to include the latest metadata changes, such as version numbers, and the "lazy bin walk" scheme was implemented in updater.py. --- docs/tuf-spec.txt | 64 ++++++++++++++++++++-------------------- tuf/client/updater.py | 68 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 88 insertions(+), 44 deletions(-) diff --git a/docs/tuf-spec.txt b/docs/tuf-spec.txt index 0c88c437..517d17e3 100644 --- a/docs/tuf-spec.txt +++ b/docs/tuf-spec.txt @@ -32,7 +32,10 @@ in all popular Linux package managers. More information and current versions of this document can be found at https://www.updateframework.com/ - The development of TUF is supported by GENI (http://www.geni.net/). + The Global Environment for Network Innovations (GENI) and the National + Science Foundation (NSF) have provided support for the development of TUF. + (http://www.geni.net/) + (http://www.nsf.gov/) TUF's Python implementation is based heavily on Thandy, the application updater for Tor (http://www.torproject.org/). Its design and this spec are @@ -409,26 +412,24 @@ 4.2. File formats: general principles All signed files are of the format: - { "signed" : X, + { "signed" : ROLE, "signatures" : [ - { "keyid" : K, - "method" : M, - "sig" : S } + { "keyid" : KEYID, + "method" : METHOD, + "sig" : SIGNATURE } , ... ] } - where: X is a list whose first element describes the signed object. - K is the identifier of a key signing the document - M is the method to be used to make the signature - S is a signature of the canonical encoding of X using the - identified key. + where: ROLE is a dictionary whose "_type" field describes the role type. + KEYID is the identifier of the key signing the ROLE dictionary. + METHOD is the key signing method used to generate the signature. + SIGNATURE is a signature of the canonical encoding of ROLE using the + signing key belonging to KEYID. We define one signing method at present: - sha256-pkcs1 : A base64 encoded signature of the SHA256 hash of the - canonical encoding of X, using PKCS-1 padding. + "evp" : An interface to OpenSSL's EVP functions. - All times are given as strings of the format "YYYY-MM-DD HH:MM:SS", - in UTC. + All times are given as strings of the format "YYYY-MM-DD HH:MM:SS UTC". All keys are of the format: { "keytype" : KEYTYPE, @@ -443,13 +444,12 @@ We define one keytype at present: 'rsa'. Its format is: { "keytype" : "rsa", - "keyval" : { "e" : E, - "n" : N } + "keyval" : { "public" : PUBLIC, + "private" : PRIVATE } } - where E and N are the binary representations of the exponent and - modulus, encoded as big-endian numbers in base64. All RSA keys must - be at least 2048 bits long. + where PUBLIC and PRIVATE are in PEM format and are strings. All RSA keys + must be at least 2048 bits long. 4.3. File formats: root.txt @@ -462,7 +462,7 @@ The format of root.txt is as follows: { "_type" : "Root", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "keys" : { KEYID : KEY @@ -474,12 +474,11 @@ , ... } } - The "ts" line describes when this file was updated. Clients - MUST NOT replace a file with an older one, and SHOULD NOT accept a - file too far in the future. + VERSION is an integer that is greater than 0. Clients MUST NOT replace a + metadata file with a version number less than the one currently trusted. - The "expires" line states when the metadata should be considered expired - and no longer trusted by clients. Clients MUST NOT trust an expired file. + EXPIRES determines when metadata should be considered expired and no longer + trusted by clients. Clients MUST NOT trust an expired file. A ROLE is one of "root", "release", "targets", "timestamp", or "mirrors". A role for each of "root", "release", "timestamp", and "targets" MUST be @@ -505,7 +504,7 @@ The format of release.txt is as follows: { "_type" : "Release", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "meta" : METAFILES } @@ -527,7 +526,7 @@ The format of targets.txt is as follows: { "_type" : "Targets", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "targets" : TARGETS, ("delegations" : DELEGATIONS) @@ -572,10 +571,9 @@ The "paths" list describes paths that the role is trusted to provide. Clients MUST check that a target is in one of the trusted paths of all roles in a delegation chain, not just in a trusted path of the role that describes - the target file. The format of a PATHPATTERN may be either a path to a - single file or a path to a directory and end with "/**" to indicate all - files under that directory. The value of "/**" by itself therefore means - all files. + the target file. The format of a PATHPATTERN may be either a path to a single + file, or a path to a directory to indicate all files and/or subdirectories + under that directory. We are currently investigating a few "priority tag" schemes to resolve conflicts between delegated roles that share responsibility for overlapping @@ -610,7 +608,7 @@ The format of the timestamp file is as follows: { "_type" : "Timestamp", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "meta" : METAFILES } @@ -628,7 +626,7 @@ The format of mirrors.txt is as follows: { "_type" : "Mirrorlist", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "mirrors" : [ { "urlbase" : URLBASE, diff --git a/tuf/client/updater.py b/tuf/client/updater.py index dce742b5..acc6daf5 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1534,6 +1534,13 @@ def target(self, target_filepath): # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) + # The algorithm used by the repository to generate the hashes of the + # target filepaths. The repository may optionally organize + # targets into hashed bins to ease target delegations and role metadata + # management. The use of consistent hashing allows for a uniform + # distribution of targets into bins. + HASH_PATH_ALGORITHM = 'sha256' + # Ensure the client has the most up-to-date version of 'targets.txt'. # Raise 'tuf.MetadataNotAvailableError' if the changed metadata # cannot be successfully downloaded and 'tuf.RepositoryError' if the @@ -1545,12 +1552,23 @@ def target(self, target_filepath): # The target is assumed to be missing until proven otherwise. target = None + # Calculate the hash of the filepath to determine which bin to find the + # target. The client currently assumes the repository uses + # 'HASH_PATH_ALGORITHM' to generate hashes. + # TODO: Should the TUF spec restrict the repository to one particular + # algorithm? Should we allow the repository to specify in the role + # dictionary the algorithm used for these generated hashed paths? + digest_object = tuf.hash.digest(HASH_PATH_ALGORITHM) + digest_object.update(target_filepath) + target_file_path_hash = digest_object.hexdigest() + try: current_metadata = self.metadata['current'] role_names = ['targets'] # Preorder depth-first traversal of the tree of target delegations. while len(role_names) > 0 and target is None: + # Pop the role name from the top of the stack. role_name = role_names.pop(-1) @@ -1575,20 +1593,48 @@ def target(self, target_filepath): break # Push children in reverse order of appearance onto the stack. + # NOTE: This may be a slow operation if there are many delegated roles + # or bins. for child_role in reversed(child_roles): child_role_name = child_role['name'] - child_role_paths = child_role['paths'] + child_role_paths = child_role.get('paths') + child_role_path_hash_prefix = child_role.get('path_hash_prefix') - # Ensure that we explore only delegated roles trusted with the target. - # We assume conservation of delegated paths in the complete tree of - # delegations. Note that the call to _ensure_all_targets_allowed in - # _update_metadata should already ensure that all targets metadata is - # valid; i.e. that the targets signed by a delegatee is a proper - # subset of the targets delegated to it by the delegator. - # Nevertheless, we check it again here for performance and safety - # reasons. - if target_filepath in child_role_paths: - role_names.append(child_role_name) + if child_role_path_hash_prefix is not None: + if target_file_path_hash.startswith(child_role_path_hash_prefix): + + # Found a matching path hash prefix. The metadata for + # 'child_role_name' will be retrieved on the next iteration + # of the while-loop. + role_names.append(child_role_name) + elif child_role_paths is not None: + + # Ensure that we explore only delegated roles trusted with the target. + # We assume conservation of delegated paths in the complete tree of + # delegations. Note that the call to _ensure_all_targets_allowed in + # _update_metadata should already ensure that all targets metadata is + # valid; i.e. that the targets signed by a delegatee is a proper + # subset of the targets delegated to it by the delegator. + # Nevertheless, we check it again here for performance and safety + # reasons. + for child_role_path in child_role_paths: + + # A child role path may be a filepath or directory. Explore + # directories which may contain 'target_filepath'. + prefix = os.path.commonprefix([target_filepath, child_role_path]) + if target_filepath in child_role_paths: + + # The metadata for 'child_role_name' will be retrieved on the next + # iteration of the while-loop. + role_names.append(child_role_name) + else: + + # 'role_name' should have been validated when it was downloaded. + # The 'paths' or 'path_hash_prefix' fields should not be missing, + # but log a warning if this else clause is reached. + message = repr(child_role)+' unexpectedly did not contain one of '+\ + 'the required fields ("paths" or "path_hash_prefix").' + logger.warn(message) except: raise finally: From be669795f9eac5bda6393802b36f6be479c804be Mon Sep 17 00:00:00 2001 From: vladdd Date: Fri, 9 Aug 2013 10:43:26 -0400 Subject: [PATCH 019/119] Expand comment and add missing prefix comparison in updater.target() --- tuf/client/updater.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index acc6daf5..5fe9d0a9 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1619,10 +1619,11 @@ def target(self, target_filepath): # reasons. for child_role_path in child_role_paths: - # A child role path may be a filepath or directory. Explore - # directories which may contain 'target_filepath'. + # A child role path may be a filepath or directory. The child + # role 'child_role_name' is added if 'target_filepath' is located + # under 'child_role_path'. Explicit filepaths are also added. prefix = os.path.commonprefix([target_filepath, child_role_path]) - if target_filepath in child_role_paths: + if prefix == child_role_path: # The metadata for 'child_role_name' will be retrieved on the next # iteration of the while-loop. From 7f8bbd400303d6288b03a0c26f01ad617420e4b7 Mon Sep 17 00:00:00 2001 From: vladdd Date: Fri, 9 Aug 2013 12:13:01 -0400 Subject: [PATCH 020/119] Modify _ensure_all_targets_allowed() to also work with path_hash_prefix --- tuf/client/updater.py | 88 +++++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 5fe9d0a9..83f5f878 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -912,7 +912,9 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): under 'paths'. A parent role may delegate trust to all files under a particular directory, including files in subdirectories, by simply listing the directory (e.g., 'packages/source/Django/', the equivalent - of 'packages/source/Django/*'). + of 'packages/source/Django/*'). Targets listed in hashed bins are + also validated (i.e., its calculated path hash prefix must be delegated + by the parent role. metadata_role: @@ -928,7 +930,8 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): tuf.RepositoryError: If the targets of 'metadata_role' are not allowed according to - the parent's metadata file. + the parent's metadata file. The 'paths' and 'path_hash_prefix' fields + are verified. None. @@ -938,6 +941,13 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): """ + # The algorithm used by the repository to generate the hashes of the + # target filepaths. The repository may optionally organize + # targets into hashed bins to ease target delegations and role metadata + # management. The use of consistent hashing allows for a uniform + # distribution of targets into bins. + HASH_PATH_ALGORITHM = 'sha256' + # Return if 'metadata_role' is 'targets'. 'targets' is not # a delegated role. if metadata_role == 'targets': @@ -955,30 +965,60 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): role_index = tuf.repo.signerlib.find_delegated_role(roles, metadata_role) # Ensure the delegated role exists prior to extracting trusted paths - # from the parent's 'paths'. + # from the parent's 'paths', or trusted path hash prefixes from the parent's + # 'path_hash_prefix'. if role_index is not None: role = roles[role_index] - allowed_child_paths = role['paths'] + allowed_child_paths = role.get('paths') + allowed_child_path_hash_prefix = role.get('path_hash_prefix') actual_child_targets = metadata_object['targets'].keys() - - # Check that each delegated target is either explicitly listed or a parent - # directory is found under role['paths'], otherwise raise an exception. - # If the parent role explicitly lists target file paths in 'paths', - # this loop will run in O(n^2), the worst-case. The repository - # maintainer will likely delegate entire directories, and opt for - # explicit file paths if the targets in a directory are delegated to - # different roles/developers. - for child_target in actual_child_targets: - for allowed_child_path in allowed_child_paths: - prefix = os.path.commonprefix([child_target, allowed_child_path]) - if prefix == allowed_child_path: - break - else: - message = 'Role '+repr(metadata_role)+' specifies target '+\ - repr(child_target)+' which is not an allowed path according '+\ - 'to the delegations set by '+repr(parent_role)+'.' - raise tuf.RepositoryError(message) + if allowed_child_path_hash_prefix is not None: + for child_target in actual_child_targets: + # Calculate the hash of 'child_target' to determine if it has been + # placed in the correct bin. The client currently assumes the + # repository uses 'HASH_PATH_ALGORITHM' to generate hashes. + # TODO: Should the TUF spec restrict the repository to one particular + # algorithm? Should we allow the repository to specify in the role + # dictionary the algorithm used for these generated hashed paths? + digest_object = tuf.hash.digest(HASH_PATH_ALGORITHM) + digest_object.update(child_target) + child_target_path_hash = digest_object.hexdigest() + + if not child_target_path_hash.startswith(allowed_child_path_hash_prefix): + message = 'Role '+repr(metadata_role)+' specifies target '+\ + repr(child_target)+ ' which does not have a path hash prefix '+\ + 'matching the prefix listed by the parent role '+\ + repr(parent_role)+'.' + raise tuf.RepositoryError(message) + elif allowed_child_paths is not None: + + # Check that each delegated target is either explicitly listed or a parent + # directory is found under role['paths'], otherwise raise an exception. + # If the parent role explicitly lists target file paths in 'paths', + # this loop will run in O(n^2), the worst-case. The repository + # maintainer will likely delegate entire directories, and opt for + # explicit file paths if the targets in a directory are delegated to + # different roles/developers. + for child_target in actual_child_targets: + for allowed_child_path in allowed_child_paths: + prefix = os.path.commonprefix([child_target, allowed_child_path]) + if prefix == allowed_child_path: + break + else: + message = 'Role '+repr(metadata_role)+' specifies target '+\ + repr(child_target)+' which is not an allowed path according '+\ + 'to the delegations set by '+repr(parent_role)+'.' + raise tuf.RepositoryError(message) + else: + + # 'role' should have been validated when it was downloaded. + # The 'paths' or 'path_hash_prefix' fields should not be missing, + # so log a warning if this else clause is reached. + message = repr(role)+' unexpectedly did not contain one of '+\ + 'the required fields ("paths" or "path_hash_prefix").' + logger.warn(message) + # Raise an exception if the parent has not delegated to the specified # 'metadata_role' child role. else: @@ -1014,7 +1054,7 @@ def _fileinfo_has_changed(self, metadata_filename, new_fileinfo): dict conforms to 'tuf.formats.FILEINFO_SCHEMA' and has the form: {'length': 23423 - 'hashes': {'sha256': adfbc32343..}} + 'hashes': {'sha256': /dfbc32343..}} None. @@ -1632,7 +1672,7 @@ def target(self, target_filepath): # 'role_name' should have been validated when it was downloaded. # The 'paths' or 'path_hash_prefix' fields should not be missing, - # but log a warning if this else clause is reached. + # so log a warning if this else clause is reached. message = repr(child_role)+' unexpectedly did not contain one of '+\ 'the required fields ("paths" or "path_hash_prefix").' logger.warn(message) From 58a6b5e337becb3855590689c580670cd13b07c6 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 9 Aug 2013 12:48:42 -0400 Subject: [PATCH 021/119] Interim bugfixes in updating. --- tuf/client/updater.py | 156 +++++++++++++++++++++++------------------- 1 file changed, 87 insertions(+), 69 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index dce742b5..30880664 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -109,6 +109,7 @@ import tuf.conf import tuf.download import tuf.formats +import tuf.hash import tuf.keydb import tuf.log import tuf.mirrors @@ -725,7 +726,8 @@ def _update_metadata(self, metadata_role, fileinfo=None, compression=None): # Reject the metadata if any specified targets are not allowed. if metadata_signable['signed']['_type'] == 'Targets': - self._ensure_all_targets_allowed(metadata_role, metadata_signable['signed']) + #self._ensure_all_targets_allowed(metadata_role, metadata_signable['signed']) + pass # The metadata has been verified. Move the metadata file into place. # First, move the 'current' metadata file to the 'previous' directory @@ -827,12 +829,9 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas # The 'root' role may be updated without having 'release' # available. if referenced_metadata not in self.metadata['current']: - if metadata_role == 'root': - new_fileinfo = None - else: - message = 'Cannot update '+repr(metadata_role)+' because ' \ - +referenced_metadata+' is missing.' - raise tuf.RepositoryError(message) + message = 'Cannot update '+repr(metadata_role)+' because ' \ + +referenced_metadata+' is missing.' + raise tuf.RepositoryError(message) # The referenced metadata has been loaded. Extract the new # fileinfo for 'metadata_role' from it. else: @@ -856,9 +855,12 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas # compressed form. compression = None gzip_path = metadata_filename + '.gz' + if metadata_role == 'release': if gzip_path in self.metadata['current'][referenced_metadata]['meta']: compression = 'gzip' + new_fileinfo = self.metadata['current'][referenced_metadata] \ + ['meta'][gzip_path] # Check for available compressed versions of 'targets.txt' and delegated # Targets, which also start with 'targets'. elif metadata_role.startswith('targets'): @@ -867,6 +869,8 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas # provided by a repository, including their file sizes and hashes. if gzip_path in self.metadata['current'][referenced_metadata]['meta']: compression = 'gzip' + new_fileinfo = self.metadata['current'][referenced_metadata] \ + ['meta'][gzip_path] else: message = 'Compressed version of '+repr(metadata_filename)+' not available.' logger.debug(message) @@ -961,13 +965,12 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): allowed_child_paths = role['paths'] actual_child_targets = metadata_object['targets'].keys() - # Check that each delegated target is either explicitly listed or a parent - # directory is found under role['paths'], otherwise raise an exception. - # If the parent role explicitly lists target file paths in 'paths', - # this loop will run in O(n^2), the worst-case. The repository - # maintainer will likely delegate entire directories, and opt for - # explicit file paths if the targets in a directory are delegated to - # different roles/developers. + # Check that each delegated target is either explicitly listed or a + # parent directory is found under role['paths'], otherwise raise an + # exception. If the parent role explicitly lists target file paths in + # 'paths', this loop will run in O(n^2). The repository maintainer will + # likely delegate entire directories, and opt for explicit file paths if + # the targets in a directory are delegated to different roles/developers. for child_target in actual_child_targets: for allowed_child_path in allowed_child_paths: prefix = os.path.commonprefix([child_target, allowed_child_path]) @@ -1515,11 +1518,7 @@ def target(self, target_filepath): tuf.RepositoryError: If 'target_filepath' was not found. - Exception: - In case of an unforeseen runtime error. - - TODO: Update these exceptions once the final 'path_hash_prefix' - changes have been implemented. + Any other unforeseen runtime exception. The metadata for updated delegated roles are downloaded and stored. @@ -1545,61 +1544,80 @@ def target(self, target_filepath): # The target is assumed to be missing until proven otherwise. target = None - try: - current_metadata = self.metadata['current'] - role_names = ['targets'] + # According to the specification, the target_filepath must be hashed with + # the SHA256 hash function in order to be compared with the + # "path_hash_prefix" attribute. + target_filepath_digest = tuf.hash.digest(algorithm='sha256') + target_filepath_digest.update(target_filepath) + target_filepath_hash = target_filepath_digest.hexdigest() - # Preorder depth-first traversal of the tree of target delegations. - while len(role_names) > 0 and target is None: - # Pop the role name from the top of the stack. - role_name = role_names.pop(-1) - - # The metadata for 'role_name' must be downloaded/updated before - # its targets, delegations, and child roles can be inspected. - # self.metadata['current'][role_name] is currently missing. - # _refresh_targets_metadata() does not refresh 'targets.txt', it - # expects _update_metadata_if_changed() to have already refreshed it, - # which this function has checked above. - self._refresh_targets_metadata(role_name, include_delegations=False) - role_metadata = current_metadata[role_name] - targets = role_metadata['targets'] - delegations = role_metadata.get('delegations', {}) - child_roles = delegations.get('roles', []) + current_metadata = self.metadata['current'] + role_names = ['targets'] - # Does the current role name have our target? - logger.info('Asking role '+role_name+' about target '+target_filepath) - for filepath, fileinfo in targets.iteritems(): - if filepath == target_filepath: - logger.info('Found target '+target_filepath+' in role '+role_name) - target = {'filepath': filepath, 'fileinfo': fileinfo} - break + # Preorder depth-first traversal of the tree of target delegations. + while len(role_names) > 0 and target is None: + # Pop the role name from the top of the stack. + role_name = role_names.pop(-1) - # Push children in reverse order of appearance onto the stack. - for child_role in reversed(child_roles): - child_role_name = child_role['name'] - child_role_paths = child_role['paths'] + # The metadata for 'role_name' must be downloaded/updated before + # its targets, delegations, and child roles can be inspected. + # self.metadata['current'][role_name] is currently missing. + # _refresh_targets_metadata() does not refresh 'targets.txt', it + # expects _update_metadata_if_changed() to have already refreshed it, + # which this function has checked above. + self._refresh_targets_metadata(role_name, include_delegations=False) + role_metadata = current_metadata[role_name] + targets = role_metadata['targets'] + delegations = role_metadata.get('delegations', {}) + child_roles = delegations.get('roles', []) - # Ensure that we explore only delegated roles trusted with the target. - # We assume conservation of delegated paths in the complete tree of - # delegations. Note that the call to _ensure_all_targets_allowed in - # _update_metadata should already ensure that all targets metadata is - # valid; i.e. that the targets signed by a delegatee is a proper - # subset of the targets delegated to it by the delegator. - # Nevertheless, we check it again here for performance and safety - # reasons. - if target_filepath in child_role_paths: + # Does the current role name have our target? + logger.info('Asking role '+role_name+' about target '+target_filepath) + for filepath, fileinfo in targets.iteritems(): + if filepath == target_filepath: + logger.info('Found target '+target_filepath+' in role '+role_name) + target = {'filepath': filepath, 'fileinfo': fileinfo} + break + + # Push children in reverse order of appearance onto the stack. + for child_role in reversed(child_roles): + child_role_name = child_role['name'] + child_role_paths = child_role.get('paths') + child_role_path_hash_prefix = child_role.get('path_hash_prefix') + + # Ensure that we explore only delegated roles trusted with the target. + # We assume conservation of delegated paths in the complete tree of + # delegations. Note that the call to _ensure_all_targets_allowed in + # _update_metadata should already ensure that all targets metadata is + # valid; i.e. that the targets signed by a delegatee is a proper + # subset of the targets delegated to it by the delegator. + # Nevertheless, we check it again here for performance and safety + # reasons. + + if child_role_path_hash_prefix is not None: + if target_filepath_hash.startswith(child_role_path_hash_prefix): role_names.append(child_role_name) - except: - raise - finally: - # Raise an exception if the target information could not be retrieved. - if target is None: - message = target_filepath+' not found.' - logger.error(message) - raise tuf.RepositoryError(message) - # Otherwise, return the found target. - else: - return target + elif child_role_paths is not None: + # TODO: is child_role_paths directories or paths? + for child_role_path in child_role_paths: + if child_role_path.endswith('/'): + if target_filepath.startswith(child_role_path): + role_names.append(child_role_name) + else: + if target_filepath == child_role_path: + role_names.append(child_role_name) + else: + raise tuf.RepositoryError(str(child_role_name)+' has neither ' \ + '"paths" nor "path_hash_prefix"!') + + # Raise an exception if the target information could not be retrieved. + if target is None: + message = target_filepath+' not found.' + logger.error(message) + raise tuf.RepositoryError(message) + # Otherwise, return the found target. + else: + return target From 9599f3e58c285a81a756be7b170e63d3c08bdee1 Mon Sep 17 00:00:00 2001 From: vladdd Date: Fri, 9 Aug 2013 16:05:49 -0400 Subject: [PATCH 022/119] Allow compressed versions of release.txt to properly update --- tuf/client/updater.py | 78 +++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 835d7701..d699cb04 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -824,56 +824,54 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas metadata_filename = metadata_role + '.txt' - # Need to ensure the referenced metadata has been loaded. - # The 'root' role may be updated without having 'release' - # available. + # Ensure the referenced metadata has been loaded. The 'root' role may be + # updated without having 'release' available. if referenced_metadata not in self.metadata['current']: - message = 'Cannot update '+repr(metadata_role)+' because ' \ - +referenced_metadata+' is missing.' + message = 'Cannot update '+repr(metadata_role)+' because '+\ + repr(referenced_metadata)+' is missing.' raise tuf.RepositoryError(message) - # The referenced metadata has been loaded. Extract the new - # fileinfo for 'metadata_role' from it. else: - new_fileinfo = self.metadata['current'][referenced_metadata] \ - ['meta'][metadata_filename] + message = repr(metadata_role)+' referenced in '+\ + repr(referenced_metadata)+'. '+repr(metadata_role)+' may be updated.' + logger.debug(message) + + # There might be a compressed version of 'release.txt' or Targets + # metadata available for download. Check the 'meta' field of + # 'referenced_metadata' to see if it is listed when 'metadata_role' + # is 'release'. The full rolename for delegated Targets metadata + # must begin with 'targets/'. The Release role lists all the Targets + # metadata available on the repository, including any that may be in + # compressed form. + compression = None + gzip_metadata_filename = metadata_filename + '.gz' + + # Extract the new fileinfo of the uncompressed version of 'metadata_role'. + new_fileinfo = self.metadata['current'][referenced_metadata] \ + ['meta'][metadata_filename] + + # Check for availability of compressed versions of 'release.txt', + # 'targets.txt', and delegated Targets, which also start with 'targets'. + # For 'targets.txt' and delegated metadata, 'referenced_metata' + # should always be 'release'. 'release.txt' specifies all roles + # provided by a repository, including their file sizes and hashes. + if metadata_role == 'release' or metadata_role.startswith('targets'): + if gzip_metadata_filename in self.metadata['current'] \ + [referenced_metadata]['meta']: + compression = 'gzip' + new_fileinfo = self.metadata['current'][referenced_metadata] \ + ['meta'][gzip_metadata_filename] + else: + message = 'Compressed version of '+repr(metadata_filename)+\ + ' not available.' + logger.debug(message) - # Simply return if the fileinfo has not changed according to the + # Simply return if the fileinfo has not changed, according to the # fileinfo provided by the referenced metadata. if not self._fileinfo_has_changed(metadata_filename, new_fileinfo): return logger.info('Metadata '+repr(metadata_filename)+' has changed.') - # There might be a compressed version of 'release.txt' or Targets - # metadata available for download. Check the 'meta' field of - # 'referenced_metadata' to see if it is listed when 'metadata_role' - # is 'release'. Check the 'meta' field of 'release' when 'metadata_role' - # is Targets metadata. The full rolename for delegated Targets metadata - # must begin with 'targets/'. The Release role lists all the Targets - # metadata available on the repository, including any that may be in - # compressed form. - compression = None - gzip_path = metadata_filename + '.gz' - - if metadata_role == 'release': - if gzip_path in self.metadata['current'][referenced_metadata]['meta']: - compression = 'gzip' - new_fileinfo = self.metadata['current'][referenced_metadata] \ - ['meta'][gzip_path] - # Check for available compressed versions of 'targets.txt' and delegated - # Targets, which also start with 'targets'. - elif metadata_role.startswith('targets'): - # For 'targets.txt' and delegated metadata, 'referenced_metata' - # should always be 'release'. 'release.txt' specifies all roles - # provided by a repository, including their file sizes and hashes. - if gzip_path in self.metadata['current'][referenced_metadata]['meta']: - compression = 'gzip' - new_fileinfo = self.metadata['current'][referenced_metadata] \ - ['meta'][gzip_path] - else: - message = 'Compressed version of '+repr(metadata_filename)+' not available.' - logger.debug(message) - try: self._update_metadata(metadata_role, fileinfo=new_fileinfo, compression=compression) From d3eb6de8a7144905224078013dbc232744f56d32 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 10 Aug 2013 20:07:21 -0400 Subject: [PATCH 023/119] Correctly mock some HTTP headers. --- tuf/interposition/updater.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 8be219f5..1a54ac47 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -132,12 +132,25 @@ def open(self, url, data=None): def retrieve(self, url, filename=None, reporthook=None, data=None): INTERPOSITION_MESSAGE = "Interposing for {url}" - # TODO: set valid headers - content_type, content_encoding = mimetypes.guess_type(url) - headers = {"content-type": content_type} - Logger.info(INTERPOSITION_MESSAGE.format(url=url)) + + # What is the actual target to download given the URL? Sometimes we would + # like to transform the given URL to the intended target; e.g. "/simple/" + # => "/simple/index.html". target_filepath = self.get_target_filepath(url) + + # TODO: Set valid headers fetched from the actual download. + # NOTE: Important to guess the mime type from the target_filepath, not the + # unmodified URL. + content_type, content_encoding = mimetypes.guess_type(target_filepath) + headers = { + # NOTE: pip refers to this same header in at least these two duplicate + # ways. + "content-type": content_type, + "Content-Type": content_type, + } + + # Download the target filepath determined by the original URL. temporary_directory, temporary_filename = self.download_target(target_filepath) if filename is None: From 8d53442cb096326a6ae0a0283e5978ffbb048084 Mon Sep 17 00:00:00 2001 From: vladdd Date: Sun, 11 Aug 2013 17:47:37 -0400 Subject: [PATCH 024/119] Fix mistaken argument to method and reduce logger messages The argument to metadata_object.move() should have been the uncompressed filename. metadata_object was saving the uncompressed version of targets but naming them as if it was compressed. Modified a few logger calls to reduce the number of messages. --- tuf/client/updater.py | 39 ++++++++++++++++++++++++++------------- tuf/hash.py | 4 ++-- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index d699cb04..2eb0d159 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -631,6 +631,7 @@ def _update_metadata(self, metadata_role, fileinfo=None, compression=None): # Construct the metadata filename as expected by the download/mirror modules. metadata_filename = metadata_role + '.txt' + uncompressed_metadata_filename = metadata_filename # The 'release' or Targets metadata may be compressed. Add the appropriate # extension to 'metadata_filename'. @@ -661,6 +662,7 @@ def _update_metadata(self, metadata_role, fileinfo=None, compression=None): # 'tuf.formats.SIGNABLE_SCHEMA'. metadata_file_object = None metadata_signable = None + compressed_file_object = None for mirror_url in get_mirrors('meta', metadata_filename.encode("utf-8"), self.mirrors): try: metadata_file_object = download_file(mirror_url, file_hashes, @@ -669,6 +671,8 @@ def _update_metadata(self, metadata_role, fileinfo=None, compression=None): logger.warn('Download failed from '+mirror_url+'.') continue if compression: + compressed_file_object = tuf.util.TempFile() + shutil.copyfileobj(metadata_file_object, compressed_file_object) metadata_file_object.decompress_temp_file_object(compression) # Read and load the downloaded file. @@ -747,7 +751,14 @@ def _update_metadata(self, metadata_role, fileinfo=None, compression=None): # Next, move the verified updated metadata file to the 'current' directory. # Note that the 'move' method comes from tuf.util's TempFile class. # 'metadata_file_object' is an instance of tuf.util.TempFile. - metadata_file_object.move(current_filepath) + if compression == 'gzip': + current_uncompressed_filepath = os.path.join(self.metadata_directory['current'], + uncompressed_metadata_filename) + current_uncompressed_filepath = os.path.abspath(current_uncompressed_filepath) + metadata_file_object.move(current_uncompressed_filepath) + compressed_file_object.move(current_filepath) + else: + metadata_file_object.move(current_filepath) # Extract the metadata object so we can store it to the metadata store. # 'current_metadata_object' set to 'None' if there is not an object @@ -756,7 +767,7 @@ def _update_metadata(self, metadata_role, fileinfo=None, compression=None): current_metadata_object = self.metadata['current'].get(metadata_role) # Finally, update the metadata and fileinfo stores. - logger.debug('Updated '+current_filepath+'.') + logger.debug('Updated '+repr(current_filepath)+'.') self.metadata['previous'][metadata_role] = current_metadata_object self.metadata['current'][metadata_role] = updated_metadata_object self._update_fileinfo(metadata_filename) @@ -843,7 +854,6 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas # metadata available on the repository, including any that may be in # compressed form. compression = None - gzip_metadata_filename = metadata_filename + '.gz' # Extract the new fileinfo of the uncompressed version of 'metadata_role'. new_fileinfo = self.metadata['current'][referenced_metadata] \ @@ -855,11 +865,13 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas # should always be 'release'. 'release.txt' specifies all roles # provided by a repository, including their file sizes and hashes. if metadata_role == 'release' or metadata_role.startswith('targets'): + gzip_metadata_filename = metadata_filename + '.gz' if gzip_metadata_filename in self.metadata['current'] \ [referenced_metadata]['meta']: compression = 'gzip' new_fileinfo = self.metadata['current'][referenced_metadata] \ ['meta'][gzip_metadata_filename] + metadata_filename = gzip_metadata_filename else: message = 'Compressed version of '+repr(metadata_filename)+\ ' not available.' @@ -870,7 +882,7 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas if not self._fileinfo_has_changed(metadata_filename, new_fileinfo): return - logger.info('Metadata '+repr(metadata_filename)+' has changed.') + logger.debug('Metadata '+repr(metadata_filename)+' has changed.') try: self._update_metadata(metadata_role, fileinfo=new_fileinfo, @@ -1046,7 +1058,7 @@ def _fileinfo_has_changed(self, metadata_filename, new_fileinfo): dict conforms to 'tuf.formats.FILEINFO_SCHEMA' and has the form: {'length': 23423 - 'hashes': {'sha256': /dfbc32343..}} + 'hashes': {'sha256': adfbc32343..}} None. @@ -1104,7 +1116,7 @@ def _update_fileinfo(self, metadata_filename): Update the 'self.fileinfo' entry for the metadata belonging to 'metadata_filename'. If the 'current' metadata for 'metadata_filename' - cannot be loaded, set the its fileinfo' to 'None' to signal that + cannot be loaded, set its fileinfo' to 'None' to signal that it is not in the 'self.fileinfo' AND it also doesn't exist locally. @@ -1117,7 +1129,7 @@ def _update_fileinfo(self, metadata_filename): The file details of 'metadata_filename' is calculated and - stored to the 'self.fileinfo' store. + stored in 'self.fileinfo'. None. @@ -1645,11 +1657,11 @@ def _preorder_depth_first_walk(self, target_filepath): if child_role_name is None: logger.debug('Skipping child role '+repr(child_role_name)) else: - logger.info('Adding child role '+repr(child_role_name)) + logger.debug('Adding child role '+repr(child_role_name)) role_names.append(child_role_name) else: - logger.info('Found target in current role '+repr(role_name)) + logger.debug('Found target in current role '+repr(role_name)) return target @@ -1689,10 +1701,11 @@ def _get_target_from_targets_role(self, role_name, targets, target_filepath): target = None # Does the current role name have our target? - logger.info('Asking role '+role_name+' about target '+target_filepath) + logger.debug('Asking role '+repr(role_name)+' about target '+\ + repr(target_filepath)) for filepath, fileinfo in targets.iteritems(): if filepath == target_filepath: - logger.info('Found target '+target_filepath+' in role '+role_name) + logger.debug('Found target '+target_filepath+' in role '+role_name) target = {'filepath': filepath, 'fileinfo': fileinfo} break else: @@ -1757,7 +1770,7 @@ def _visit_child_role(self, child_role, target_filepath): target_filepath_hash = self._get_target_hash(target_filepath) if target_filepath_hash.startswith(child_role_path_hash_prefix): - logger.info('Child role '+repr(child_role_name)+' has target '+ + logger.debug('Child role '+repr(child_role_name)+' has target '+ repr(target_filepath)) child_role_is_relevant = True else: @@ -1774,7 +1787,7 @@ def _visit_child_role(self, child_role, target_filepath): prefix = os.path.commonprefix([target_filepath, child_role_path]) if prefix == child_role_path: - logger.info('Child role '+repr(child_role_name)+' has target '+ + logger.debug('Child role '+repr(child_role_name)+' has target '+ repr(target_filepath)) child_role_is_relevant = True else: diff --git a/tuf/hash.py b/tuf/hash.py index 13f12ea0..9421ce35 100755 --- a/tuf/hash.py +++ b/tuf/hash.py @@ -50,7 +50,7 @@ from Crypto.Hash import SHA512 _supported_libraries.append('pycrypto') except ImportError: - logger.warn('Pycrypto hash algorithms could not be imported. ' + logger.debug('Pycrypto hash algorithms could not be imported. ' 'Supported libraries: '+str(_SUPPORTED_LIB_LIST)) pass @@ -61,7 +61,7 @@ import hashlib _supported_libraries.append('hashlib') except ImportError: - logger.warn('Hashlib could not be imported. ' + logger.debug('Hashlib could not be imported. ' 'Supported libraries: '+str(_SUPPORTED_LIB_LIST)) pass From 77a868c58dc7c00f793f8142b7c540fffe227d11 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 12 Aug 2013 00:08:22 -0400 Subject: [PATCH 025/119] Read from and write to a list of path hash prefixes. --- docs/tuf-spec.txt | 50 ++++++++------- tuf/client/updater.py | 137 ++++++++++++++++++++++++++---------------- tuf/formats.py | 35 +++++++---- tuf/repo/signercli.py | 35 ++++++----- tuf/repo/signerlib.py | 41 ------------- 5 files changed, 153 insertions(+), 145 deletions(-) diff --git a/docs/tuf-spec.txt b/docs/tuf-spec.txt index abd17cae..52eab91c 100644 --- a/docs/tuf-spec.txt +++ b/docs/tuf-spec.txt @@ -564,29 +564,21 @@ "name": ROLE, "keyids" : [ KEYID, ... ] , "threshold" : THRESHOLD, - ("paths" : [ PATHPATTERN, ... ] | "path_hash_prefix" : HEX_DIGEST) + ("path_hash_prefixes" : [ HEX_DIGEST, ... ] | + "paths" : [ PATHPATTERN, ... ]) }, ... ] } In order to discuss target paths, a role MUST specify only one of the - "paths" or "path_hash_prefix" attributes, each of which we discuss next. + "path_hash_prefixes" or "paths" attributes, each of which we discuss next. - The "paths" list describes paths that the role is trusted to provide. - Clients MUST check that a target is in one of the trusted paths of all roles - in a delegation chain, not just in a trusted path of the role that describes - the target file. The format of a PATHPATTERN may be either a path to a single - file, or a path to a directory to indicate all files and/or subdirectories - under that directory. - - A path to a directory is used to indicate all possible targets sharing that - directory as a prefix; e.g. if the directory is "targets/A", then targets - which match that directory include "targets/A/B.txt" and - "targets/A/B/C.txt". - - The "path_hash_prefix" is used to succinctly describe a set of target paths. - The target paths must meet this condition: each target path, when hashed - with the SHA-256 hash function to produce a 64-byte hexadecimal digest, must - share the same prefix as the specified "path_hash_prefix". This is useful to + The "path_hash_prefixes" list is used to succinctly describe a set of target + paths. Specifically, each HEX_DIGEST in "path_hash_prefixes" describes a set + of target paths; therefore, "path_hash_prefixes" is the union over each + prefix of its set of target paths. The target paths must meet this + condition: each target path, when hashed with the SHA-256 hash function to + produce a 64-byte hexadecimal digest (HEX_DIGEST), must share the same + prefix as one of the prefixes in "path_hash_prefixes". This is useful to split a large number of targets into separate bins identified by consistent hashing. @@ -594,17 +586,29 @@ algorithm? Should we allow the repository to specify in the role dictionary the algorithm used for these generated hashed paths? + The "paths" list describes paths that the role is trusted to provide. + Clients MUST check that a target is in one of the trusted paths of all roles + in a delegation chain, not just in a trusted path of the role that describes + the target file. The format of a PATHPATTERN may be either a path to a + single file, or a path to a directory to indicate all files and/or + subdirectories under that directory. + + A path to a directory is used to indicate all possible targets sharing that + directory as a prefix; e.g. if the directory is "targets/A", then targets + which match that directory include "targets/A/B.txt" and + "targets/A/B/C.txt". + We are currently investigating a few "priority tag" schemes to resolve conflicts between delegated roles that share responsibility for overlapping target paths. One of the simplest of such schemes is for the client to consider metadata in order of appearance of delegations; we treat the order of delegations such that the first delegation is trusted more than the second one, the second delegation is trusted more than the third one, and so - on. The metadata of the first delegation will override that of the second delegation, - the metadata of the second delegation will override that of the third - delegation, and so on. In order to accommodate this scheme, the "roles" key - in the DELEGATIONS object above points to an array, instead of a hash - table, of delegated roles. + on. The metadata of the first delegation will override that of the second + delegation, the metadata of the second delegation will override that of the + third delegation, and so on. In order to accommodate this scheme, the + "roles" key in the DELEGATIONS object above points to an array, instead of a + hash table, of delegated roles. Another priority tag scheme would have the clients prefer the delegated role with the latest metadata for a conflicting target path. Similar ideas were diff --git a/tuf/client/updater.py b/tuf/client/updater.py index d699cb04..26e7f4d7 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -935,9 +935,9 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): tuf.RepositoryError: If the targets of 'metadata_role' are not allowed according to - the parent's metadata file. The 'paths' and 'path_hash_prefix' fields - are verified. - + the parent's metadata file. The 'paths' and 'path_hash_prefixes' + attributes are verified. + None. @@ -962,27 +962,24 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): roles = self.metadata['current'][parent_role]['delegations']['roles'] role_index = tuf.repo.signerlib.find_delegated_role(roles, metadata_role) - # Ensure the delegated role exists prior to extracting trusted paths - # from the parent's 'paths', or trusted path hash prefixes from the parent's - # 'path_hash_prefix'. + # Ensure the delegated role exists prior to extracting trusted paths from + # the parent's 'paths', or trusted path hash prefixes from the parent's + # 'path_hash_prefixes'. if role_index is not None: role = roles[role_index] allowed_child_paths = role.get('paths') - allowed_child_path_hash_prefix = role.get('path_hash_prefix') + allowed_child_path_hash_prefixes = role.get('path_hash_prefixes') actual_child_targets = metadata_object['targets'].keys() - - if allowed_child_path_hash_prefix is not None: - for child_target in actual_child_targets: - # Calculate the hash of 'child_target' to determine if it has been - # placed in the correct bin. - child_target_path_hash = self._get_target_hash(child_target) - if not child_target_path_hash.startswith(allowed_child_path_hash_prefix): - message = 'Role '+repr(metadata_role)+' specifies target '+\ - repr(child_target)+ ' which does not have a path hash prefix '+\ - 'matching the prefix listed by the parent role '+\ - repr(parent_role)+'.' - raise tuf.RepositoryError(message) + if allowed_child_path_hash_prefixes is not None: + consistent = self._paths_are_consistent_with_hash_prefixes + if not consistent(actual_child_targets, + allowed_child_path_hash_prefixes): + raise tuf.RepositoryError('Role '+repr(metadata_role)+' specifies '+\ + 'target which does not have a path hash '+\ + 'prefix matching the prefix listed by '+\ + 'the parent role '+repr(parent_role)+'.') + elif allowed_child_paths is not None: # Check that each delegated target is either explicitly listed or a parent @@ -1002,14 +999,14 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): repr(child_target)+' which is not an allowed path according '+\ 'to the delegations set by '+repr(parent_role)+'.' raise tuf.RepositoryError(message) + else: - + # 'role' should have been validated when it was downloaded. - # The 'paths' or 'path_hash_prefix' fields should not be missing, - # so log a warning if this else clause is reached. - message = repr(role)+' unexpectedly did not contain one of '+\ - 'the required fields ("paths" or "path_hash_prefix").' - logger.warn(message) + # The 'paths' or 'path_hash_prefixes' attributes should not be missing, + # so log a warning if this clause is reached. + logger.warn(repr(role)+' unexpectedly did not contain one of '+\ + 'the required fields ("paths" or "path_hash_prefixes").') # Raise an exception if the parent has not delegated to the specified # 'metadata_role' child role. @@ -1017,7 +1014,55 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): message = repr(parent_role)+' has not delegated to '+\ repr(metadata_role)+'.' raise tuf.RepositoryError(message) - + + + + + + def _paths_are_consistent_with_hash_prefixes(paths, path_hash_prefixes): + """ + + Determine whether a list of paths are consistent with theirs alleged + path hash prefixes. By default, the SHA256 hash function will be used. + + + paths: + A list of paths for which their hashes will be checked. + + path_hash_prefixes: + The list of path hash prefixes with which to check the list of paths. + + + No known exceptions. + + + No known side effects. + + + A Boolean indicating whether or not the paths are consistent with the + hash prefix. + """ + + # Assume that 'paths' and 'path_hash_prefixes' are inconsistent until + # proven otherwise. + consistent = False + + if len(paths) > 0 and len(path_hash_prefixes) > 0: + for path in paths: + path_hash = self._get_target_hash(path) + # Assume that every path is inconsistent until proven otherwise. + consistent = False + + for path_hash_prefix in path_hash_prefixes: + if path_hash.startswith(path_hash_prefix): + consistent = True + break + + # This path has no matching path_hash_prefix. Stop looking further. + if not consistent: break + + return consistent + @@ -1726,7 +1771,7 @@ def _visit_child_role(self, child_role, target_filepath): child_role: The delegation targets role object of 'child_role', containing its - paths, path_hash_prefix, keys and so on. + paths, path_hash_prefixes, keys and so on. target_filepath: The path to the target file on the repository. This will be relative to @@ -1748,50 +1793,40 @@ def _visit_child_role(self, child_role, target_filepath): child_role_name = child_role['name'] child_role_paths = child_role.get('paths') - child_role_path_hash_prefix = child_role.get('path_hash_prefix') + child_role_path_hash_prefixes = child_role.get('path_hash_prefixes') # A boolean indicator that tell us whether 'child_role' has been delegated # the target with the name 'target_filepath'. child_role_is_relevant = False - if child_role_path_hash_prefix is not None: + if child_role_path_hash_prefixes is not None: target_filepath_hash = self._get_target_hash(target_filepath) - - if target_filepath_hash.startswith(child_role_path_hash_prefix): - logger.info('Child role '+repr(child_role_name)+' has target '+ - repr(target_filepath)) - child_role_is_relevant = True - else: - logger.debug('Child role '+repr(child_role_name)+ - ' does not have target '+repr(target_filepath)) + for child_role_path_hash_prefix in child_role_path_hash_prefixes: + if target_filepath_hash.startswith(child_role_path_hash_prefix): + child_role_is_relevant = True elif child_role_paths is not None: - for child_role_path in child_role_paths: - # A child role path may be a filepath or directory. The child # role 'child_role_name' is added if 'target_filepath' is located # under 'child_role_path'. Explicit filepaths are also added. prefix = os.path.commonprefix([target_filepath, child_role_path]) - if prefix == child_role_path: - logger.info('Child role '+repr(child_role_name)+' has target '+ - repr(target_filepath)) child_role_is_relevant = True - else: - logger.debug('Child role '+repr(child_role_name)+ - ' does not have target '+repr(target_filepath)) else: - # 'role_name' should have been validated when it was downloaded. - # The 'paths' or 'path_hash_prefix' fields should not be missing, - # so log a warning if this else clause is reached. + # The 'paths' or 'path_hash_prefixes' fields should not be missing, + # so we raise a format error here in case they are both missing. raise tuf.FormatError(repr(child_role_name)+' has neither ' \ - '"paths" nor "path_hash_prefix"!') + '"paths" nor "path_hash_prefixes"!') if child_role_is_relevant: + logger.info('Child role '+repr(child_role_name)+' has target '+ + repr(target_filepath)) return child_role_name else: + logger.debug('Child role '+repr(child_role_name)+ + ' does not have target '+repr(target_filepath)) return None @@ -1802,8 +1837,8 @@ def _get_target_hash(self, target_filepath, hash_function='sha256'): """ Compute the hash of 'target_filepath'. This is useful in conjunction with - the "path_hash_prefix" attribute in a delegated targets role, which tells - us which paths it is implicitly responsible for. + the "path_hash_prefixes" attribute in a delegated targets role, which + tells us which paths it is implicitly responsible for. target_filepath: diff --git a/tuf/formats.py b/tuf/formats.py index 776baf4a..0208726e 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -270,6 +270,11 @@ targets_directory=PATH_SCHEMA, backup_directory=PATH_SCHEMA)) +# A path hash prefix is a hexadecimal string. +PATH_HASH_PREFIX_SCHEMA = HEX_SCHEMA +# A list of path hash prefixes. +PATH_HASH_PREFIXES_SCHEMA = SCHEMA.ListOf(PATH_HASH_PREFIX_SCHEMA) + # Role object in {'keyids': [keydids..], 'name': 'ABC', 'threshold': 1, # 'paths':[filepaths..]} # format. ROLE_SCHEMA = SCHEMA.Object( @@ -278,7 +283,7 @@ name=SCHEMA.Optional(ROLENAME_SCHEMA), threshold=THRESHOLD_SCHEMA, paths=SCHEMA.Optional(RELPATHS_SCHEMA), - path_hash_prefix=SCHEMA.Optional(HEX_SCHEMA)) + path_hash_prefixes=SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA)) # A dict of roles where the dict keys are role names and the dict values holding # the role data/information. @@ -830,7 +835,7 @@ def make_fileinfo(length, hashes, custom=None): def make_role_metadata(keyids, threshold, name=None, paths=None, - path_hash_prefix=None): + path_hash_prefixes=None): """ Create a dictionary conforming to 'tuf.formats.ROLE_SCHEMA', @@ -852,7 +857,12 @@ def make_role_metadata(keyids, threshold, name=None, paths=None, The 'Target' role stores the paths of target files in its metadata file. 'paths' is a list of file paths. - + + path_hash_prefixes: + The 'Target' role stores the paths of target files in its metadata file. + 'path_hash_prefixes' is a succint way to describe a set of paths to + target files. + tuf.FormatError, if the returned role meta is formatted incorrectly. @@ -875,18 +885,19 @@ def make_role_metadata(keyids, threshold, name=None, paths=None, if name is not None: role_meta['name'] = name - # According to the specification, the 'paths' and 'path_hash_prefix' must be - # mutually exclusive. However, at the time of writing we do not always ensure - # that this is the case with the schema checks (see #83). Therefore, we must - # do it for ourselves. + # According to the specification, the 'paths' and 'path_hash_prefixes' must + # be mutually exclusive. However, at the time of writing we do not always + # ensure that this is the case with the schema checks (see #83). Therefore, + # we must do it for ourselves. - if paths is not None and path_hash_prefix is not None: - raise tuf.FormatError('Both "paths" and "path_hash_prefix" are specified!') + if paths is not None and path_hash_prefixes is not None: + raise \ + tuf.FormatError('Both "paths" and "path_hash_prefixes" are specified!') - if paths is not None: + if path_hash_prefixes is not None: + role_meta['path_hash_prefixes'] = path_hash_prefixes + elif paths is not None: role_meta['paths'] = paths - elif path_hash_prefix is not None: - role_meta['path_hash_prefix'] = path_hash_prefix # Does 'role_meta' have the correct type? # This check ensures 'role_meta' conforms to diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index 5ea437d6..a2971351 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -1328,9 +1328,8 @@ def _make_delegated_metadata(metadata_directory, delegated_targets, def _update_parent_metadata(metadata_directory, delegated_role, - delegated_keyids, parent_role, - parent_keyids, delegated_paths=None, - path_hash_prefix=None): + delegated_keyids, parent_role, parent_keyids, + delegated_paths=None, path_hash_prefixes=None): """ Update the parent role's metadata file. The delegations field of the metadata file is updated with the key and role information belonging @@ -1339,18 +1338,18 @@ def _update_parent_metadata(metadata_directory, delegated_role, """ - # According to the specification, the 'paths' and 'path_hash_prefix' must be - # mutually exclusive. However, at the time of writing we do not always ensure - # that this is the case with the schema checks (see #83). Therefore, we must - # do it for ourselves. + # According to the specification, the 'paths' and 'path_hash_prefixes' + # attributes must be mutually exclusive. However, at the time of writing we + # do not always ensure that this is the case with the schema checks (see + # #83). Therefore, we must do it for ourselves. - if delegated_paths is not None and path_hash_prefix is not None: - raise tuf.RepositoryError('Both "paths" and "path_hash_prefix" are ' \ - 'specified!') + if delegated_paths is not None and path_hash_prefixes is not None: + raise \ + tuf.FormatError('Both "paths" and "path_hash_prefixes" are specified!') - if delegated_paths is None and path_hash_prefix is None: - raise tuf.RepositoryError('Neither "paths" nor`"path_hash_prefix" is ' \ - 'specified!') + if delegated_paths is None and path_hash_prefixes is None: + raise \ + tuf.FormatError('Neither "paths" nor`"path_hash_prefixes" is specified!') # The 'delegated_paths' are relative to 'repository'. # The 'relative_paths' are relative to 'repository/targets'. @@ -1391,11 +1390,11 @@ def _update_parent_metadata(metadata_directory, delegated_role, threshold = len(delegated_keyids) delegated_role = parent_role+'/'+delegated_role - # Write either the "paths" or the "path_hash_prefix" attribute. - role_metadata = tuf.formats.make_role_metadata(delegated_keyids, threshold, - name=delegated_role, - paths=relative_paths, - path_hash_prefix=path_hash_prefix) + # Write either the "paths" or the "path_hash_prefixes" attribute. + role_metadata = \ + tuf.formats.make_role_metadata(delegated_keyids, threshold, + name=delegated_role, paths=relative_paths, + path_hash_prefixes=path_hash_prefixes) # Find the appropriate role to create or update. role_index = tuf.repo.signerlib.find_delegated_role(roles, delegated_role) diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 99b44a00..ab30c84f 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -1546,44 +1546,3 @@ def get_targets(files_directory, recursive_walk=False, followlinks=True, -def paths_are_consistent_with_hash_prefix(paths, path_hash_prefix, - hash_function='sha256'): - """ - - Determine whether a list of paths are consistent with their alleged path - hash prefix. By default, the SHA256 hash function will be used. - - - paths: - A list of paths for which their hashes will be checked. - - path_hash_prefix: - The hash prefix with which to check the list of paths. - - - No known exceptions. - - - No known side effects. - - - A Boolean indicating whether or not the paths are consistent with the hash - prefix. - """ - - consistent = True - - for path in paths: - digest = tuf.hash.digest(algorithm='sha256') - digest.update(path) - path_hash = digest.hexdigest() - if not path_hash.startswith(path_hash_prefix): - consistent = False - break - - return consistent - - - - - From a823464ad98bc6f1214533bd2e6f49d2d0e33c68 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 12 Aug 2013 12:03:41 -0400 Subject: [PATCH 026/119] Better formatting of error message. --- tuf/client/updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 4a7692ce..8b020905 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1326,7 +1326,7 @@ def _ensure_not_expired(self, metadata_role): # convert it to seconds since the epoch, which is the time format # returned by time.time() (i.e., current time), before comparing. if tuf.formats.parse_time(expires) < time.time(): - message = 'Metadata '+repr(rolepath)+' expired on '+expires+' UTC.' + message = 'Metadata '+repr(rolepath)+' expired on '+repr(expires)+'.' raise tuf.ExpiredMetadataError(message) From dc3f9680fe74c34ea219cd039077c62a2565acb6 Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 12 Aug 2013 12:20:48 -0400 Subject: [PATCH 027/119] Review and confirm issue #63 --- tuf/repo/keystore.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index c2ca98c0..c8fe5afd 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -147,7 +147,7 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): directory_name: The name of the directory containing the key files ('.key'), - conformant to tuf.formats.RELPATH_SCHEMA. + conformant to 'tuf.formats.RELPATH_SCHEMA'. keyids: A list containing the keyids of the signing keys to load. @@ -188,10 +188,8 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): logger.info('Loading private key(s) from '+repr(directory_name)) - # Make sure the directory exists. - if not os.path.exists(directory_name): - logger.warn('...no such directory. Keystore cannot be loaded.') - else: + # Load the private key(s) if 'directory_name' exists, otherwise log a warning. + if os.path.exists(directory_name): # Decrypt the keys we can from those stored in 'keyids'. for keyid in keyids: try: @@ -243,7 +241,11 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): logger.warn(repr(full_filepath)+' contains an invalid key type.') continue + else: + logger.warn('...no such directory. Keystore cannot be loaded.') + logger.info('Done.') + return loaded_keys From a6ccdc9eecd3a99e92bcbfbc01df3669df22351c Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 12 Aug 2013 12:47:26 -0400 Subject: [PATCH 028/119] Confirm unit tests run properly individually & fix test_keystore "test_keystore.py" logged TUF messages if run individually. --- tuf/tests/test_keystore.py | 7 +++++++ tuf/tests/test_push.py | 0 tuf/tests/test_pushtoolslib.py | 0 3 files changed, 7 insertions(+) mode change 100644 => 100755 tuf/tests/test_push.py mode change 100644 => 100755 tuf/tests/test_pushtoolslib.py diff --git a/tuf/tests/test_keystore.py b/tuf/tests/test_keystore.py index 196bb7d4..816f400a 100755 --- a/tuf/tests/test_keystore.py +++ b/tuf/tests/test_keystore.py @@ -19,12 +19,19 @@ import unittest import shutil import os +import logging import tuf.repo.keystore import tuf.rsa_key import tuf.formats import tuf.util +logger = logging.getLogger('tuf') + +# Disable all logging calls of level CRITICAL and below. +# Comment the line below to enable logging. +logging.disable(logging.CRITICAL) + # We'll need json module for testing '_encrypt()' and '_decrypt()' # internal function. json = tuf.util.import_json() diff --git a/tuf/tests/test_push.py b/tuf/tests/test_push.py old mode 100644 new mode 100755 diff --git a/tuf/tests/test_pushtoolslib.py b/tuf/tests/test_pushtoolslib.py old mode 100644 new mode 100755 From 853331b4ff7d13a421d323782d1a4760ec1b8b46 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 12 Aug 2013 17:29:57 -0400 Subject: [PATCH 029/119] Updaters now clean up after deconfiguration. --- tuf/interposition/configuration.py | 2 -- tuf/interposition/updater.py | 20 +++++++++++++++++++- tuf/interposition/utility.py | 5 +++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/tuf/interposition/configuration.py b/tuf/interposition/configuration.py index 5a66e4af..295ccac6 100644 --- a/tuf/interposition/configuration.py +++ b/tuf/interposition/configuration.py @@ -1,5 +1,4 @@ import os.path -import tempfile import types import urlparse @@ -43,7 +42,6 @@ def __init__(self, hostname, port, repository_directory, repository_mirrors, self.repository_mirrors = repository_mirrors self.target_paths = target_paths self.ssl_certificates = ssl_certificates - self.tempdir = tempfile.mkdtemp() def __repr__(self): diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 1a54ac47..15a789f7 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -2,6 +2,7 @@ import os.path import re import shutil +import tempfile import urllib import urlparse @@ -37,7 +38,12 @@ class Updater(object): def __init__(self, configuration): + CREATED_TEMPDIR_MESSAGE = "Created temporary directory at {tempdir}" + self.configuration = configuration + # A temporary directory used for this updater over runtime. + self.tempdir = tempfile.mkdtemp() + Logger.debug(CREATED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir)) # must switch context before instantiating updater # because updater depends on some module (tuf.conf) variables @@ -46,11 +52,19 @@ def __init__(self, configuration): self.configuration.repository_mirrors) + def cleanup(self): + """Clean up after certain side effects, such as temporary directories.""" + + DELETED_TEMPDIR_MESSAGE = "Deleted temporary directory at {tempdir}" + shutil.rmtree(self.tempdir) + Logger.debug(DELETED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir)) + + def download_target(self, target_filepath): """Downloads target with TUF as a side effect.""" # download file into a temporary directory shared over runtime - destination_directory = self.configuration.tempdir + destination_directory = self.tempdir filename = os.path.join(destination_directory, target_filepath) self.switch_context() # switch TUF context @@ -314,8 +328,12 @@ def remove(self, configuration): assert configuration.hostname in self.__updaters assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) + # Get the updater. + updater = self.__updaters.get(configuration.hostname) + # If all is well, remove the stored Updater as well as its associated # repository mirror hostnames. + updater.cleanup() del self.__updaters[configuration.hostname] self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames) diff --git a/tuf/interposition/utility.py b/tuf/interposition/utility.py index c68ad777..8e1c4cd9 100644 --- a/tuf/interposition/utility.py +++ b/tuf/interposition/utility.py @@ -23,6 +23,11 @@ class Logger(object): __logger = logging.getLogger("tuf.interposition") + @staticmethod + def debug(message): + Logger.__logger.debug(message) + + @staticmethod def exception(message): Logger.__logger.exception(message) From 8d70be920c908e0cb73976c91b5691c92fbbbcbe Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 12 Aug 2013 19:18:54 -0400 Subject: [PATCH 030/119] Return interposition configurations. --- tuf/interposition/__init__.py | 46 +++++++++++++++++------ tuf/interposition/updater.py | 5 +++ tuf/tests/system_tests/util_test_tools.py | 17 ++++++--- 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index db2262a9..00c53e7c 100644 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -172,14 +172,21 @@ def __read_configuration(configuration_handler, parent_repository_directory=None, parent_ssl_certificates_directory=None): """ - A generic function to read a TUF interposition configuration off the disk, - and handle it. configuration_handler must be a function which accepts a - tuf.interposition.Configuration instance.""" + A generic function to read TUF interposition configurations off a file, and + then handle those configurations with a given function. configuration_handler + must be a function which accepts a tuf.interposition.Configuration + instance. + + Returns the parsed configurations as a dictionary of configurations indexed + by hostnames.""" INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" + # Configurations indexed by hostnames. + parsed_configurations = {} + try: with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) @@ -197,6 +204,7 @@ def __read_configuration(configuration_handler, configuration = configuration_parser.parse() configuration_handler(configuration) + parsed_configurations[configuration.hostname] = configuration except: Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) @@ -206,6 +214,10 @@ def __read_configuration(configuration_handler, Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise + else: + return parsed_configurations + + @@ -218,8 +230,7 @@ def configure(filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): - """ - The optional parent_repository_directory parameter is used to specify the + """The optional parent_repository_directory parameter is used to specify the containing parent directory of the "repository_directory" specified in a configuration for *all* network locations, because sometimes the absolute location of the "repository_directory" is only known at runtime. If you @@ -259,20 +270,26 @@ def configure(filename="tuf.interposition.json", Unless any "url_prefix" begins with "https://", "ssl_certificates" is optional; it must specify certificates bundled as PEM (RFC 1422). - """ - __read_configuration(__updater_controller.add, filename=filename, - parent_repository_directory=parent_repository_directory, - parent_ssl_certificates_directory=parent_ssl_certificates_directory) + Returns the parsed configurations as a dictionary of configurations indexed + by hostnames.""" + + configurations = \ + __read_configuration(__updater_controller.add, filename=filename, + parent_repository_directory=parent_repository_directory, + parent_ssl_certificates_directory=parent_ssl_certificates_directory) + + return configurations -def deconfigure(filename="tuf.interposition.json"): - """Remove TUF interposition for a previously read configuration.""" +def deconfigure(configurations): + """Remove TUF interposition for previously read configurations.""" - __read_configuration(__updater_controller.remove, filename=filename) + for configuration in configurations.itervalues(): + __updater_controller.remove(configuration) @@ -328,3 +345,8 @@ def wrapper(self, *args, **kwargs): # Build and monkey patch public copies of the urllib and urllib2 modules. __monkey_patch() + + + + + diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 15a789f7..6c83e964 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -338,3 +338,8 @@ def remove(self, configuration): self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames) Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration)) + + + + + diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/system_tests/util_test_tools.py index 1c263068..1e5714c0 100755 --- a/tuf/tests/system_tests/util_test_tools.py +++ b/tuf/tests/system_tests/util_test_tools.py @@ -154,6 +154,8 @@ def disable_logging(): PASSWD = 'test' version = 1 +# Where we keep TUF configurations, if any, between every iteration. +tuf_configurations = None def init_repo(tuf=False, port=None): @@ -196,6 +198,8 @@ def init_repo(tuf=False, port=None): def cleanup(root_repo, server_process=None): + global tuf_configurations + if server_process is not None: if server_process.returncode is None: server_process.kill() @@ -206,9 +210,9 @@ def cleanup(root_repo, server_process=None): keystore.clear_keystore() # Deconfigure interposition. - interpose_json = os.path.join(root_repo, 'tuf.interposition.json') - if os.path.exists(interpose_json): - tuf.interposition.deconfigure(filename=interpose_json) + if tuf_configurations is not None: + tuf.interposition.deconfigure(tuf_configurations) + tuf_configurations = None # Removing repository directory. try: @@ -365,7 +369,9 @@ def create_interposition_config(root_repo, url): (urllib_tuf replaces urllib module) urllib_tuf.urlretrieve(url, filename) - """ + """ + + global tuf_configurations tuf_repo = os.path.join(root_repo, 'tuf_repo') tuf_client = os.path.join(root_repo, 'tuf_client') @@ -396,7 +402,8 @@ def create_interposition_config(root_repo, url): with open(interpose_json, 'wb') as fileobj: tuf.util.json.dump(interposition_dict, fileobj) - tuf.interposition.configure(filename=interpose_json) + assert tuf_configurations is None + tuf_configurations = tuf.interposition.configure(filename=interpose_json) From 54e7ba0f32b39600c945e380f4bb61d6b280e892 Mon Sep 17 00:00:00 2001 From: vladdd Date: Tue, 13 Aug 2013 12:19:31 -0400 Subject: [PATCH 031/119] Refactor log.py and change the default logging behavior Previously, logging messages were written to "tuf.log" *and* and the console, by default. Modules had to explicitly disable the logger to silence console messages. TUF, when integrated by a software updater, should not log messages to console by default. The design change now forces modules to call tuf.log.add_console_handler() to enable logging messages to console. The logger, file, and console handlers may have independent logging levels. --- tuf/log.py | 187 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 156 insertions(+), 31 deletions(-) diff --git a/tuf/log.py b/tuf/log.py index cd61ecf7..f3c018bc 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -12,10 +12,9 @@ See LICENSE for licensing information. - A central location for all logging-related configuration. - This module should be imported once by the main program. - If other modules wish to incorporate 'tuf' logging, they - should do the following: + A central location for all logging-related configuration. This module should + be imported once by the main program. If other modules wish to incorporate + 'tuf' logging, they should do the following: import logging logger = logging.getLogger('tuf') @@ -26,18 +25,28 @@ instance. In this 'log.py' module, we perform the initial setup for the name 'tuf'. The 'log.py' module should only be imported once by the main program. When any other module does a logging.getLogger('tuf'), it is referring to the - same 'tuf' instance and its associated settings we set up here in 'log.py'. - See http://docs.python.org/library/logging.html#logger-objects - for more information. + same 'tuf' instance, and its associated settings, set here in 'log.py'. + See http://docs.python.org/library/logging.html#logger-objects for more + information. We use multiple handlers to process log messages in various ways and to configure each one independently. Instead of using one single manner of processing log messages, we can use two built-in handlers that have already been configured for us. For example, the built-in FileHandler will catch - log message and dump them to a file. If we wanted, we could set this file - handler to only catch CRITICAL (and greater) messages and save them to a - file. The other stream handler would still handle DEBUG-level (and greater) - messages. + log messages and dump them to a file. If we wanted, we could set this file + handler to only catch CRITICAL (and greater) messages and save them to a + file. Other handlers (e.g., StreamHandler) could handle INFO-level + (and greater) messages. + + Logging Levels: + + --Level-- --Value-- + logging.CRITICAL 50 + logging.ERROR 40 + logging.WARNING 30 + logging.INFO 20 + logging.DEBUG 10 + logging.NOTSET 0 """ @@ -45,35 +54,46 @@ import logging import time +import tuf -_DEFAULT_LOG_LEVEL = logging.INFO +# Setting a handler's log level filters only logging messages of that level +# (and above). For example, setting the built-in StreamHandler's log level to +# 'logging.WARNING' will cause the stream handler to only process messages +# of levels: WARNING, ERROR, and CRITICAL. _DEFAULT_LOG_FILENAME = 'tuf.log' +_DEFAULT_LOG_LEVEL = logging.DEBUG +_DEFAULT_CONSOLE_LOG_LEVEL = logging.INFO +_DEFAULT_FILE_LOG_LEVEL = logging.DEBUG # Set the format for logging messages. +# Example format for '_FORMAT_STRING': +# [2013-08-13 15:21:18,068 UTC] [tuf] [INFO][_update_metadata:851@updater.py] _FORMAT_STRING = '[%(asctime)s UTC] [%(name)s] [%(levelname)s]'+\ '[%(funcName)s:%(lineno)s@%(filename)s] %(message)s' + + logging.Formatter.converter = time.gmtime formatter = logging.Formatter(_FORMAT_STRING) -# Set the handlers for the logger. -# The built-in stream handler will log -# messages to 'sys.stderr' and capture -# '_DEFAULT_LOG_LEVEL' messages. -stream_handler = logging.StreamHandler() -stream_handler.setLevel(_DEFAULT_LOG_LEVEL) -stream_handler.setFormatter(formatter) +# Set the handlers for the logger. The console handler is unset by default. A +# module importing 'log.py' should explicitly set the console handler if +# outputting log messages to the screen is needed. Adding a console handler +# can be done with tuf.log.add_console_handler(). Logging messages to a file +# *is* set by default. +console_handler = None -# Set the built-in file handler. Messages -# will be logged to '_DEFAULT_LOG_FILENAME' -# and use the logger's default log level. -# The file will be opened in append mode. +# Set the built-in file handler. Messages will be logged to +# '_DEFAULT_LOG_FILENAME', and only those messages with a log level of +# '_DEFAULT_LOG_LEVEL'. The log level of messages handled by 'file_handler' +# may be modified with 'set_filehandler_log_level()'. '_DEFAULT_LOG_FILENAME' +# will be opened in append mode. file_handler = logging.FileHandler(_DEFAULT_LOG_FILENAME) +file_handler.setLevel(_DEFAULT_LOG_LEVEL) file_handler.setFormatter(formatter) # Set the logger and its settings. logger = logging.getLogger('tuf') logger.setLevel(_DEFAULT_LOG_LEVEL) -logger.addHandler(stream_handler) logger.addHandler(file_handler) # Silently ignore logger exceptions. @@ -83,27 +103,132 @@ -def set_log_level(log_level): +def set_log_level(log_level=_DEFAULT_LOG_LEVEL): """ Allow the default log level to be overridden. log_level: - The log level to set for the logger and handler(s). - E.g., logging.INFO; logging.CRITICAL. + The log level to set for the 'log.py' file handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. None. - Overrides the logging level for the internal - 'logger' and 'handler'. + Overrides the logging level for the 'log.py' file handler. None. """ - + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + logger.setLevel(log_level) - stream_handler.setLevel(log_level) + + + + + +def set_filehandler_log_level(log_level=_DEFAULT_FILE_LOG_LEVEL): + """ + + Allow the default file handler log level to be overridden. + + + log_level: + The log level to set for the 'log.py' file handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + None. + + + Overrides the logging level for the 'log.py' file handler. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + file_handler.setLevel(log_level) + + + + + +def set_console_log_level(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): + """ + + Allow the default log level for console messages to be overridden. + + + log_level: + The log level to set for the console handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + tuf.Error, if the 'log.py' console handler has not been set yet with + add_console_handler(). + + + Overrides the logging level for the console handler. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + if console_handler is not None: + console_handler.setLevel(log_level) + else: + message = 'The console handler has not been set with add_console_handler().' + raise tuf.Error(message) + + + + +def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): + """ + + Add a console handler and set its log level to 'log_level'. + + + log_level: + The log level to set for the console handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + None. + + + Adds a console handler to the 'log.py' logger and sets its logging level to + 'log_level'. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + # Set the console handler for the logger. The built-in console handler will + # log messages to 'sys.stderr' and capture 'log_level' messages. + console_handler = logging.StreamHandler() + console_handler.setLevel(log_level) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) From 1e3b9cfcad5a639f3d4ef66f271bd825705b1375 Mon Sep 17 00:00:00 2001 From: ttgump Date: Tue, 13 Aug 2013 14:57:03 -0400 Subject: [PATCH 032/119] Pull from upstream --- docs/tuf-spec.txt | 64 +++--- tuf/client/updater.py | 155 +++++++++++---- tuf/log.py | 187 +++++++++++++++--- tuf/repo/keystore.py | 12 +- .../test_extraneous_dependencies_attack.py | 1 + tuf/tests/test_keystore.py | 7 + tuf/tests/test_push.py | 0 tuf/tests/test_pushtoolslib.py | 0 8 files changed, 323 insertions(+), 103 deletions(-) mode change 100644 => 100755 tuf/tests/test_push.py mode change 100644 => 100755 tuf/tests/test_pushtoolslib.py diff --git a/docs/tuf-spec.txt b/docs/tuf-spec.txt index 0c88c437..517d17e3 100644 --- a/docs/tuf-spec.txt +++ b/docs/tuf-spec.txt @@ -32,7 +32,10 @@ in all popular Linux package managers. More information and current versions of this document can be found at https://www.updateframework.com/ - The development of TUF is supported by GENI (http://www.geni.net/). + The Global Environment for Network Innovations (GENI) and the National + Science Foundation (NSF) have provided support for the development of TUF. + (http://www.geni.net/) + (http://www.nsf.gov/) TUF's Python implementation is based heavily on Thandy, the application updater for Tor (http://www.torproject.org/). Its design and this spec are @@ -409,26 +412,24 @@ 4.2. File formats: general principles All signed files are of the format: - { "signed" : X, + { "signed" : ROLE, "signatures" : [ - { "keyid" : K, - "method" : M, - "sig" : S } + { "keyid" : KEYID, + "method" : METHOD, + "sig" : SIGNATURE } , ... ] } - where: X is a list whose first element describes the signed object. - K is the identifier of a key signing the document - M is the method to be used to make the signature - S is a signature of the canonical encoding of X using the - identified key. + where: ROLE is a dictionary whose "_type" field describes the role type. + KEYID is the identifier of the key signing the ROLE dictionary. + METHOD is the key signing method used to generate the signature. + SIGNATURE is a signature of the canonical encoding of ROLE using the + signing key belonging to KEYID. We define one signing method at present: - sha256-pkcs1 : A base64 encoded signature of the SHA256 hash of the - canonical encoding of X, using PKCS-1 padding. + "evp" : An interface to OpenSSL's EVP functions. - All times are given as strings of the format "YYYY-MM-DD HH:MM:SS", - in UTC. + All times are given as strings of the format "YYYY-MM-DD HH:MM:SS UTC". All keys are of the format: { "keytype" : KEYTYPE, @@ -443,13 +444,12 @@ We define one keytype at present: 'rsa'. Its format is: { "keytype" : "rsa", - "keyval" : { "e" : E, - "n" : N } + "keyval" : { "public" : PUBLIC, + "private" : PRIVATE } } - where E and N are the binary representations of the exponent and - modulus, encoded as big-endian numbers in base64. All RSA keys must - be at least 2048 bits long. + where PUBLIC and PRIVATE are in PEM format and are strings. All RSA keys + must be at least 2048 bits long. 4.3. File formats: root.txt @@ -462,7 +462,7 @@ The format of root.txt is as follows: { "_type" : "Root", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "keys" : { KEYID : KEY @@ -474,12 +474,11 @@ , ... } } - The "ts" line describes when this file was updated. Clients - MUST NOT replace a file with an older one, and SHOULD NOT accept a - file too far in the future. + VERSION is an integer that is greater than 0. Clients MUST NOT replace a + metadata file with a version number less than the one currently trusted. - The "expires" line states when the metadata should be considered expired - and no longer trusted by clients. Clients MUST NOT trust an expired file. + EXPIRES determines when metadata should be considered expired and no longer + trusted by clients. Clients MUST NOT trust an expired file. A ROLE is one of "root", "release", "targets", "timestamp", or "mirrors". A role for each of "root", "release", "timestamp", and "targets" MUST be @@ -505,7 +504,7 @@ The format of release.txt is as follows: { "_type" : "Release", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "meta" : METAFILES } @@ -527,7 +526,7 @@ The format of targets.txt is as follows: { "_type" : "Targets", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "targets" : TARGETS, ("delegations" : DELEGATIONS) @@ -572,10 +571,9 @@ The "paths" list describes paths that the role is trusted to provide. Clients MUST check that a target is in one of the trusted paths of all roles in a delegation chain, not just in a trusted path of the role that describes - the target file. The format of a PATHPATTERN may be either a path to a - single file or a path to a directory and end with "/**" to indicate all - files under that directory. The value of "/**" by itself therefore means - all files. + the target file. The format of a PATHPATTERN may be either a path to a single + file, or a path to a directory to indicate all files and/or subdirectories + under that directory. We are currently investigating a few "priority tag" schemes to resolve conflicts between delegated roles that share responsibility for overlapping @@ -610,7 +608,7 @@ The format of the timestamp file is as follows: { "_type" : "Timestamp", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "meta" : METAFILES } @@ -628,7 +626,7 @@ The format of mirrors.txt is as follows: { "_type" : "Mirrorlist", - "ts" : TIME, + "version" : VERSION, "expires" : EXPIRES, "mirrors" : [ { "urlbase" : URLBASE, diff --git a/tuf/client/updater.py b/tuf/client/updater.py index b976ecf5..c0710e73 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -912,7 +912,9 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): under 'paths'. A parent role may delegate trust to all files under a particular directory, including files in subdirectories, by simply listing the directory (e.g., 'packages/source/Django/', the equivalent - of 'packages/source/Django/*'). + of 'packages/source/Django/*'). Targets listed in hashed bins are + also validated (i.e., its calculated path hash prefix must be delegated + by the parent role. metadata_role: @@ -928,7 +930,8 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): tuf.RepositoryError: If the targets of 'metadata_role' are not allowed according to - the parent's metadata file. + the parent's metadata file. The 'paths' and 'path_hash_prefix' fields + are verified. None. @@ -938,6 +941,13 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): """ + # The algorithm used by the repository to generate the hashes of the + # target filepaths. The repository may optionally organize + # targets into hashed bins to ease target delegations and role metadata + # management. The use of consistent hashing allows for a uniform + # distribution of targets into bins. + HASH_PATH_ALGORITHM = 'sha256' + # Return if 'metadata_role' is 'targets'. 'targets' is not # a delegated role. if metadata_role == 'targets': @@ -955,30 +965,60 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): role_index = tuf.repo.signerlib.find_delegated_role(roles, metadata_role) # Ensure the delegated role exists prior to extracting trusted paths - # from the parent's 'paths'. + # from the parent's 'paths', or trusted path hash prefixes from the parent's + # 'path_hash_prefix'. if role_index is not None: role = roles[role_index] - allowed_child_paths = role['paths'] + allowed_child_paths = role.get('paths') + allowed_child_path_hash_prefix = role.get('path_hash_prefix') actual_child_targets = metadata_object['targets'].keys() - - # Check that each delegated target is either explicitly listed or a parent - # directory is found under role['paths'], otherwise raise an exception. - # If the parent role explicitly lists target file paths in 'paths', - # this loop will run in O(n^2), the worst-case. The repository - # maintainer will likely delegate entire directories, and opt for - # explicit file paths if the targets in a directory are delegated to - # different roles/developers. - for child_target in actual_child_targets: - for allowed_child_path in allowed_child_paths: - prefix = os.path.commonprefix([child_target, allowed_child_path]) - if prefix == allowed_child_path: - break - else: - message = 'Role '+repr(metadata_role)+' specifies target '+\ - repr(child_target)+' which is not an allowed path according '+\ - 'to the delegations set by '+repr(parent_role)+'.' - raise tuf.RepositoryError(message) + if allowed_child_path_hash_prefix is not None: + for child_target in actual_child_targets: + # Calculate the hash of 'child_target' to determine if it has been + # placed in the correct bin. The client currently assumes the + # repository uses 'HASH_PATH_ALGORITHM' to generate hashes. + # TODO: Should the TUF spec restrict the repository to one particular + # algorithm? Should we allow the repository to specify in the role + # dictionary the algorithm used for these generated hashed paths? + digest_object = tuf.hash.digest(HASH_PATH_ALGORITHM) + digest_object.update(child_target) + child_target_path_hash = digest_object.hexdigest() + + if not child_target_path_hash.startswith(allowed_child_path_hash_prefix): + message = 'Role '+repr(metadata_role)+' specifies target '+\ + repr(child_target)+ ' which does not have a path hash prefix '+\ + 'matching the prefix listed by the parent role '+\ + repr(parent_role)+'.' + raise tuf.RepositoryError(message) + elif allowed_child_paths is not None: + + # Check that each delegated target is either explicitly listed or a parent + # directory is found under role['paths'], otherwise raise an exception. + # If the parent role explicitly lists target file paths in 'paths', + # this loop will run in O(n^2), the worst-case. The repository + # maintainer will likely delegate entire directories, and opt for + # explicit file paths if the targets in a directory are delegated to + # different roles/developers. + for child_target in actual_child_targets: + for allowed_child_path in allowed_child_paths: + prefix = os.path.commonprefix([child_target, allowed_child_path]) + if prefix == allowed_child_path: + break + else: + message = 'Role '+repr(metadata_role)+' specifies target '+\ + repr(child_target)+' which is not an allowed path according '+\ + 'to the delegations set by '+repr(parent_role)+'.' + raise tuf.RepositoryError(message) + else: + + # 'role' should have been validated when it was downloaded. + # The 'paths' or 'path_hash_prefix' fields should not be missing, + # so log a warning if this else clause is reached. + message = repr(role)+' unexpectedly did not contain one of '+\ + 'the required fields ("paths" or "path_hash_prefix").' + logger.warn(message) + # Raise an exception if the parent has not delegated to the specified # 'metadata_role' child role. else: @@ -1014,7 +1054,7 @@ def _fileinfo_has_changed(self, metadata_filename, new_fileinfo): dict conforms to 'tuf.formats.FILEINFO_SCHEMA' and has the form: {'length': 23423 - 'hashes': {'sha256': adfbc32343..}} + 'hashes': {'sha256': /dfbc32343..}} None. @@ -1534,6 +1574,13 @@ def target(self, target_filepath): # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) + # The algorithm used by the repository to generate the hashes of the + # target filepaths. The repository may optionally organize + # targets into hashed bins to ease target delegations and role metadata + # management. The use of consistent hashing allows for a uniform + # distribution of targets into bins. + HASH_PATH_ALGORITHM = 'sha256' + # Ensure the client has the most up-to-date version of 'targets.txt'. # Raise 'tuf.MetadataNotAvailableError' if the changed metadata # cannot be successfully downloaded and 'tuf.RepositoryError' if the @@ -1545,12 +1592,23 @@ def target(self, target_filepath): # The target is assumed to be missing until proven otherwise. target = None + # Calculate the hash of the filepath to determine which bin to find the + # target. The client currently assumes the repository uses + # 'HASH_PATH_ALGORITHM' to generate hashes. + # TODO: Should the TUF spec restrict the repository to one particular + # algorithm? Should we allow the repository to specify in the role + # dictionary the algorithm used for these generated hashed paths? + digest_object = tuf.hash.digest(HASH_PATH_ALGORITHM) + digest_object.update(target_filepath) + target_file_path_hash = digest_object.hexdigest() + try: current_metadata = self.metadata['current'] role_names = ['targets'] # Preorder depth-first traversal of the tree of target delegations. while len(role_names) > 0 and target is None: + # Pop the role name from the top of the stack. role_name = role_names.pop(-1) @@ -1575,20 +1633,49 @@ def target(self, target_filepath): break # Push children in reverse order of appearance onto the stack. + # NOTE: This may be a slow operation if there are many delegated roles + # or bins. for child_role in reversed(child_roles): child_role_name = child_role['name'] - child_role_paths = child_role['paths'] + child_role_paths = child_role.get('paths') + child_role_path_hash_prefix = child_role.get('path_hash_prefix') - # Ensure that we explore only delegated roles trusted with the target. - # We assume conservation of delegated paths in the complete tree of - # delegations. Note that the call to _ensure_all_targets_allowed in - # _update_metadata should already ensure that all targets metadata is - # valid; i.e. that the targets signed by a delegatee is a proper - # subset of the targets delegated to it by the delegator. - # Nevertheless, we check it again here for performance and safety - # reasons. - if target_filepath in child_role_paths: - role_names.append(child_role_name) + if child_role_path_hash_prefix is not None: + if target_file_path_hash.startswith(child_role_path_hash_prefix): + + # Found a matching path hash prefix. The metadata for + # 'child_role_name' will be retrieved on the next iteration + # of the while-loop. + role_names.append(child_role_name) + elif child_role_paths is not None: + + # Ensure that we explore only delegated roles trusted with the target. + # We assume conservation of delegated paths in the complete tree of + # delegations. Note that the call to _ensure_all_targets_allowed in + # _update_metadata should already ensure that all targets metadata is + # valid; i.e. that the targets signed by a delegatee is a proper + # subset of the targets delegated to it by the delegator. + # Nevertheless, we check it again here for performance and safety + # reasons. + for child_role_path in child_role_paths: + + # A child role path may be a filepath or directory. The child + # role 'child_role_name' is added if 'target_filepath' is located + # under 'child_role_path'. Explicit filepaths are also added. + prefix = os.path.commonprefix([target_filepath, child_role_path]) + if prefix == child_role_path: + + # The metadata for 'child_role_name' will be retrieved on the next + # iteration of the while-loop. + role_names.append(child_role_name) + else: + + # 'role_name' should have been validated when it was downloaded. + # The 'paths' or 'path_hash_prefix' fields should not be missing, + # so log a warning if this else clause is reached. + message = repr(child_role)+' unexpectedly did not contain one of '+\ + 'the required fields ("paths" or "path_hash_prefix").' + logger.warn(message) except: raise finally: diff --git a/tuf/log.py b/tuf/log.py index cd61ecf7..f3c018bc 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -12,10 +12,9 @@ See LICENSE for licensing information. - A central location for all logging-related configuration. - This module should be imported once by the main program. - If other modules wish to incorporate 'tuf' logging, they - should do the following: + A central location for all logging-related configuration. This module should + be imported once by the main program. If other modules wish to incorporate + 'tuf' logging, they should do the following: import logging logger = logging.getLogger('tuf') @@ -26,18 +25,28 @@ instance. In this 'log.py' module, we perform the initial setup for the name 'tuf'. The 'log.py' module should only be imported once by the main program. When any other module does a logging.getLogger('tuf'), it is referring to the - same 'tuf' instance and its associated settings we set up here in 'log.py'. - See http://docs.python.org/library/logging.html#logger-objects - for more information. + same 'tuf' instance, and its associated settings, set here in 'log.py'. + See http://docs.python.org/library/logging.html#logger-objects for more + information. We use multiple handlers to process log messages in various ways and to configure each one independently. Instead of using one single manner of processing log messages, we can use two built-in handlers that have already been configured for us. For example, the built-in FileHandler will catch - log message and dump them to a file. If we wanted, we could set this file - handler to only catch CRITICAL (and greater) messages and save them to a - file. The other stream handler would still handle DEBUG-level (and greater) - messages. + log messages and dump them to a file. If we wanted, we could set this file + handler to only catch CRITICAL (and greater) messages and save them to a + file. Other handlers (e.g., StreamHandler) could handle INFO-level + (and greater) messages. + + Logging Levels: + + --Level-- --Value-- + logging.CRITICAL 50 + logging.ERROR 40 + logging.WARNING 30 + logging.INFO 20 + logging.DEBUG 10 + logging.NOTSET 0 """ @@ -45,35 +54,46 @@ import logging import time +import tuf -_DEFAULT_LOG_LEVEL = logging.INFO +# Setting a handler's log level filters only logging messages of that level +# (and above). For example, setting the built-in StreamHandler's log level to +# 'logging.WARNING' will cause the stream handler to only process messages +# of levels: WARNING, ERROR, and CRITICAL. _DEFAULT_LOG_FILENAME = 'tuf.log' +_DEFAULT_LOG_LEVEL = logging.DEBUG +_DEFAULT_CONSOLE_LOG_LEVEL = logging.INFO +_DEFAULT_FILE_LOG_LEVEL = logging.DEBUG # Set the format for logging messages. +# Example format for '_FORMAT_STRING': +# [2013-08-13 15:21:18,068 UTC] [tuf] [INFO][_update_metadata:851@updater.py] _FORMAT_STRING = '[%(asctime)s UTC] [%(name)s] [%(levelname)s]'+\ '[%(funcName)s:%(lineno)s@%(filename)s] %(message)s' + + logging.Formatter.converter = time.gmtime formatter = logging.Formatter(_FORMAT_STRING) -# Set the handlers for the logger. -# The built-in stream handler will log -# messages to 'sys.stderr' and capture -# '_DEFAULT_LOG_LEVEL' messages. -stream_handler = logging.StreamHandler() -stream_handler.setLevel(_DEFAULT_LOG_LEVEL) -stream_handler.setFormatter(formatter) +# Set the handlers for the logger. The console handler is unset by default. A +# module importing 'log.py' should explicitly set the console handler if +# outputting log messages to the screen is needed. Adding a console handler +# can be done with tuf.log.add_console_handler(). Logging messages to a file +# *is* set by default. +console_handler = None -# Set the built-in file handler. Messages -# will be logged to '_DEFAULT_LOG_FILENAME' -# and use the logger's default log level. -# The file will be opened in append mode. +# Set the built-in file handler. Messages will be logged to +# '_DEFAULT_LOG_FILENAME', and only those messages with a log level of +# '_DEFAULT_LOG_LEVEL'. The log level of messages handled by 'file_handler' +# may be modified with 'set_filehandler_log_level()'. '_DEFAULT_LOG_FILENAME' +# will be opened in append mode. file_handler = logging.FileHandler(_DEFAULT_LOG_FILENAME) +file_handler.setLevel(_DEFAULT_LOG_LEVEL) file_handler.setFormatter(formatter) # Set the logger and its settings. logger = logging.getLogger('tuf') logger.setLevel(_DEFAULT_LOG_LEVEL) -logger.addHandler(stream_handler) logger.addHandler(file_handler) # Silently ignore logger exceptions. @@ -83,27 +103,132 @@ -def set_log_level(log_level): +def set_log_level(log_level=_DEFAULT_LOG_LEVEL): """ Allow the default log level to be overridden. log_level: - The log level to set for the logger and handler(s). - E.g., logging.INFO; logging.CRITICAL. + The log level to set for the 'log.py' file handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. None. - Overrides the logging level for the internal - 'logger' and 'handler'. + Overrides the logging level for the 'log.py' file handler. None. """ - + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + logger.setLevel(log_level) - stream_handler.setLevel(log_level) + + + + + +def set_filehandler_log_level(log_level=_DEFAULT_FILE_LOG_LEVEL): + """ + + Allow the default file handler log level to be overridden. + + + log_level: + The log level to set for the 'log.py' file handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + None. + + + Overrides the logging level for the 'log.py' file handler. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + file_handler.setLevel(log_level) + + + + + +def set_console_log_level(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): + """ + + Allow the default log level for console messages to be overridden. + + + log_level: + The log level to set for the console handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + tuf.Error, if the 'log.py' console handler has not been set yet with + add_console_handler(). + + + Overrides the logging level for the console handler. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + if console_handler is not None: + console_handler.setLevel(log_level) + else: + message = 'The console handler has not been set with add_console_handler().' + raise tuf.Error(message) + + + + +def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): + """ + + Add a console handler and set its log level to 'log_level'. + + + log_level: + The log level to set for the console handler. + 'log_level' examples: logging.INFO; logging.CRITICAL. + + + None. + + + Adds a console handler to the 'log.py' logger and sets its logging level to + 'log_level'. + + + None. + + """ + + # Does 'log_level' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.LENGTH_SCHEMA.check_match(log_level) + + # Set the console handler for the logger. The built-in console handler will + # log messages to 'sys.stderr' and capture 'log_level' messages. + console_handler = logging.StreamHandler() + console_handler.setLevel(log_level) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index c2ca98c0..c8fe5afd 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -147,7 +147,7 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): directory_name: The name of the directory containing the key files ('.key'), - conformant to tuf.formats.RELPATH_SCHEMA. + conformant to 'tuf.formats.RELPATH_SCHEMA'. keyids: A list containing the keyids of the signing keys to load. @@ -188,10 +188,8 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): logger.info('Loading private key(s) from '+repr(directory_name)) - # Make sure the directory exists. - if not os.path.exists(directory_name): - logger.warn('...no such directory. Keystore cannot be loaded.') - else: + # Load the private key(s) if 'directory_name' exists, otherwise log a warning. + if os.path.exists(directory_name): # Decrypt the keys we can from those stored in 'keyids'. for keyid in keyids: try: @@ -243,7 +241,11 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): logger.warn(repr(full_filepath)+' contains an invalid key type.') continue + else: + logger.warn('...no such directory. Keystore cannot be loaded.') + logger.info('Done.') + return loaded_keys diff --git a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py index 72937d65..25a258f0 100755 --- a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py +++ b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py @@ -187,4 +187,5 @@ def _write_rogue_metadata(): try: test_extraneous_dependencies_attack() except ExtraneousDependenciesAttackAlert, error: + raise print 'error' diff --git a/tuf/tests/test_keystore.py b/tuf/tests/test_keystore.py index 196bb7d4..816f400a 100755 --- a/tuf/tests/test_keystore.py +++ b/tuf/tests/test_keystore.py @@ -19,12 +19,19 @@ import unittest import shutil import os +import logging import tuf.repo.keystore import tuf.rsa_key import tuf.formats import tuf.util +logger = logging.getLogger('tuf') + +# Disable all logging calls of level CRITICAL and below. +# Comment the line below to enable logging. +logging.disable(logging.CRITICAL) + # We'll need json module for testing '_encrypt()' and '_decrypt()' # internal function. json = tuf.util.import_json() diff --git a/tuf/tests/test_push.py b/tuf/tests/test_push.py old mode 100644 new mode 100755 diff --git a/tuf/tests/test_pushtoolslib.py b/tuf/tests/test_pushtoolslib.py old mode 100644 new mode 100755 From f8d699e62b751be5c5aa49aa7d93981bcd59cc8f Mon Sep 17 00:00:00 2001 From: zanefisher Date: Tue, 13 Aug 2013 18:41:38 -0400 Subject: [PATCH 033/119] Randomized order of aggragate tests. Added teardown code to some unit tests. --- tuf/tests/aggregate_tests.py | 3 +++ tuf/tests/test_signercli.py | 4 ++++ tuf/tests/test_signerlib.py | 4 ++++ tuf/tests/test_updater.py | 4 ++++ 4 files changed, 15 insertions(+) diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index 3ac9ecac..b5dd21b7 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -27,12 +27,15 @@ import tuf.keydb as keydb import tuf.repo.keystore as keystore import tuf.roledb as roledb +import random tests_list = glob.glob('test_*.py') # Remove '.py' from each filename. tests_list = [test[:-3] for test in tests_list] +random.shuffle(tests_list) + suite = unittest.TestLoader().loadTestsFromNames(tests_list) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index aa91d862..ea9fccc5 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -1559,5 +1559,9 @@ def _mock_get_keyids(junk): signercli._get_metadata_directory = original_get_metadata_directory +def tearDownModule(): + unittest_toolbox.Modified_TestCase.clear_toolbox() + + if __name__ == '__main__': unittest.main() diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index dcb47993..8331ea74 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -975,6 +975,10 @@ def _get_signed_role_info(self, role, directory=None): filename) return signed_meta, role_info +def tearDownModule(): + unit_tbox.clear_toolbox() + tuf.repo.keystore.clear_keystore() + if __name__ == '__main__': unittest.main() diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index d32664e7..8386338e 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -1150,5 +1150,9 @@ def test_8_remove_obsolete_targets(self): tuf.download.download_url_to_tempfileobj = original_download +def tearDownModule(): + setup.remove_all_repositories(TestUpdater.repositories['main_repository']) + unittest_toolbox.Modified_TestCase.clear_toolbox() + if __name__ == '__main__': unittest.main() From e1f3cde81854a7d732a7b5899e8b9307a1fce8ae Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 14 Aug 2013 11:06:13 -0400 Subject: [PATCH 034/119] Workaround hashing target paths with Unicode characters. --- tuf/client/updater.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 8b020905..6b269ca0 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1882,7 +1882,16 @@ def _get_target_hash(self, target_filepath, hash_function='sha256'): # 'hash_function' to generate hashes. digest_object = tuf.hash.digest(hash_function) - digest_object.update(target_filepath) + + try: + digest_object.update(target_filepath) + except UnicodeEncodeError: + # Sometimes, there are Unicode characters in target paths. We assume a + # UTF-8 encoding and try to hash that. + digest_object = tuf.hash.digest(hash_function) + encoded_target_filepath = target_filepath.encode('utf-8') + digest_object.update(encoded_target_filepath) + target_filepath_hash = digest_object.hexdigest() return target_filepath_hash From 9255b3d01b0243094c730918ffa029fa4dab6a76 Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 14 Aug 2013 13:22:34 -0400 Subject: [PATCH 035/119] Log to console when tuf.interposition is imported. Reduce logging noise. --- tuf/client/updater.py | 4 ++-- tuf/interposition/utility.py | 1 + tuf/log.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 6b269ca0..54796c28 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1835,8 +1835,8 @@ def _visit_child_role(self, child_role, target_filepath): '"paths" nor "path_hash_prefixes"!') if child_role_is_relevant: - logger.info('Child role '+repr(child_role_name)+' has target '+ - repr(target_filepath)) + logger.debug('Child role '+repr(child_role_name)+' has target '+ + repr(target_filepath)) return child_role_name else: logger.debug('Child role '+repr(child_role_name)+ diff --git a/tuf/interposition/utility.py b/tuf/interposition/utility.py index 8e1c4cd9..b70fd702 100644 --- a/tuf/interposition/utility.py +++ b/tuf/interposition/utility.py @@ -20,6 +20,7 @@ class Logger(object): """A static logging object for tuf.interposition.""" + tuf.log.add_console_handler() __logger = logging.getLogger("tuf.interposition") diff --git a/tuf/log.py b/tuf/log.py index f3c018bc..716cf379 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -55,6 +55,7 @@ import time import tuf +import tuf.formats # Setting a handler's log level filters only logging messages of that level # (and above). For example, setting the built-in StreamHandler's log level to From 3537917015b8fdaea9b25c795cdcba7f94739767 Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 15 Aug 2013 14:33:35 -0400 Subject: [PATCH 036/119] Update the unit tests affected by the design change to log.py --- setup.py | 3 +- tuf/client/basic_client.py | 2 + tuf/client/updater.py | 3 +- tuf/examples/example_client.py | 6 +- tuf/hash.py | 7 ++- tuf/keydb.py | 1 + tuf/log.py | 3 +- tuf/mirrors.py | 1 + tuf/pushtools/push.py | 1 + tuf/pushtools/pushtoolslib.py | 2 +- tuf/pushtools/receivetools/receive.py | 77 ++++++++++++----------- tuf/pushtools/transfer/scp.py | 1 + tuf/repo/quickstart.py | 2 +- tuf/repo/signercli.py | 5 +- tuf/repo/signerlib.py | 1 + tuf/roledb.py | 3 +- tuf/rsa_key.py | 2 + tuf/sig.py | 1 + tuf/tests/system_tests/util_test_tools.py | 6 +- tuf/tests/test_download.py | 14 ++--- tuf/tests/test_formats.py | 1 + tuf/tests/test_hash.py | 5 +- tuf/tests/test_keydb.py | 8 +-- tuf/tests/test_keystore.py | 8 +-- tuf/tests/test_push.py | 9 +-- tuf/tests/test_pushtoolslib.py | 4 ++ tuf/tests/test_quickstart.py | 12 ++-- tuf/tests/test_roledb.py | 8 +-- tuf/tests/test_rsa_key.py | 4 ++ tuf/tests/test_schema.py | 5 ++ tuf/tests/test_sig.py | 8 ++- tuf/tests/test_signercli.py | 11 ++-- tuf/tests/test_signerlib.py | 9 +-- tuf/tests/test_updater.py | 9 ++- tuf/tests/test_util.py | 4 +- tuf/tests/unittest_toolbox.py | 3 +- tuf/util.py | 1 + 37 files changed, 138 insertions(+), 112 deletions(-) diff --git a/setup.py b/setup.py index 4175561b..bbbc3c85 100755 --- a/setup.py +++ b/setup.py @@ -76,7 +76,8 @@ 'tuf.pushtools', 'tuf.pushtools.transfer', 'tuf.repo', - 'tuf.tests' + 'tuf.tests', + 'tuf.tests.system_tests' ], scripts=[ 'tuf/repo/quickstart.py', diff --git a/tuf/client/basic_client.py b/tuf/client/basic_client.py index b690955c..1ba36dbc 100755 --- a/tuf/client/basic_client.py +++ b/tuf/client/basic_client.py @@ -55,6 +55,8 @@ import optparse import logging +import tuf +import tuf.formats import tuf.client.updater import tuf.log diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 83f5f878..f58a0416 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -106,6 +106,7 @@ import shutil import time +import tuf import tuf.conf import tuf.download import tuf.formats @@ -117,7 +118,7 @@ import tuf.sig import tuf.util -logger = logging.getLogger('tuf') +logger = logging.getLogger('tuf.client.updater') class Updater(object): diff --git a/tuf/examples/example_client.py b/tuf/examples/example_client.py index 78b1103f..a1735412 100755 --- a/tuf/examples/example_client.py +++ b/tuf/examples/example_client.py @@ -27,10 +27,12 @@ import logging +import tuf +import tuf.log import tuf.client.updater -# Uncomment the line below to enable printing of debugging information. -#tuf.log.set_log_level(logging.DEBUG) +logger = logging.getLogger('tuf.cient.basic_client') + # Set the local repository directory containing the metadata files. tuf.conf.repository_directory = '.' diff --git a/tuf/hash.py b/tuf/hash.py index 13f12ea0..161a181c 100755 --- a/tuf/hash.py +++ b/tuf/hash.py @@ -24,12 +24,13 @@ """ +import logging + # Import tuf Exceptions. import tuf import tuf.log # Import tuf logger to log warning messages. -import logging logger = logging.getLogger('tuf.hash') # The list of hash libraries imported successfully. @@ -50,7 +51,7 @@ from Crypto.Hash import SHA512 _supported_libraries.append('pycrypto') except ImportError: - logger.warn('Pycrypto hash algorithms could not be imported. ' + logger.debug('Pycrypto hash algorithms could not be imported. ' 'Supported libraries: '+str(_SUPPORTED_LIB_LIST)) pass @@ -61,7 +62,7 @@ import hashlib _supported_libraries.append('hashlib') except ImportError: - logger.warn('Hashlib could not be imported. ' + logger.debug('Hashlib could not be imported. ' 'Supported libraries: '+str(_SUPPORTED_LIB_LIST)) pass diff --git a/tuf/keydb.py b/tuf/keydb.py index e2ee142d..601993b3 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -31,6 +31,7 @@ import logging +import tuf import tuf.formats import tuf.rsa_key diff --git a/tuf/log.py b/tuf/log.py index f3c018bc..51397d1f 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -55,6 +55,7 @@ import time import tuf +import tuf.formats # Setting a handler's log level filters only logging messages of that level # (and above). For example, setting the built-in StreamHandler's log level to @@ -88,7 +89,7 @@ # may be modified with 'set_filehandler_log_level()'. '_DEFAULT_LOG_FILENAME' # will be opened in append mode. file_handler = logging.FileHandler(_DEFAULT_LOG_FILENAME) -file_handler.setLevel(_DEFAULT_LOG_LEVEL) +file_handler.setLevel(_DEFAULT_FILE_LOG_LEVEL) file_handler.setFormatter(formatter) # Set the logger and its settings. diff --git a/tuf/mirrors.py b/tuf/mirrors.py index 5d159021..6a375467 100755 --- a/tuf/mirrors.py +++ b/tuf/mirrors.py @@ -21,6 +21,7 @@ import os import urllib +import tuf import tuf.util import tuf.formats diff --git a/tuf/pushtools/push.py b/tuf/pushtools/push.py index dc100659..17a3b19c 100755 --- a/tuf/pushtools/push.py +++ b/tuf/pushtools/push.py @@ -59,6 +59,7 @@ import sys import optparse +import tuf import tuf.formats import tuf.pushtools.pushtoolslib import tuf.pushtools.transfer.scp diff --git a/tuf/pushtools/pushtoolslib.py b/tuf/pushtools/pushtoolslib.py index ce6ae292..995a2ab2 100755 --- a/tuf/pushtools/pushtoolslib.py +++ b/tuf/pushtools/pushtoolslib.py @@ -151,4 +151,4 @@ def read_config_file(filename, config_type): message = 'Invalid "config_type" argument. Supported: '+repr(CONFIG_TYPES) raise tuf.Error(message) - return config_dict \ No newline at end of file + return config_dict diff --git a/tuf/pushtools/receivetools/receive.py b/tuf/pushtools/receivetools/receive.py index 7734a456..9fe81121 100755 --- a/tuf/pushtools/receivetools/receive.py +++ b/tuf/pushtools/receivetools/receive.py @@ -155,6 +155,7 @@ import logging import optparse +import tuf import tuf.formats import tuf.keydb import tuf.roledb @@ -165,7 +166,7 @@ import tuf.pushtools.pushtoolslib # See 'log.py' to learn how logging is handled in TUF. -logger = logging.getLogger('tuf.receive') +logger = logging.getLogger('tuf.pushtools.receivetools.receive') def receive(config_filepath): @@ -257,7 +258,7 @@ def receive(config_filepath): for directory_name, path in directories_to_check.items(): if not os.path.exists(path): - message = directory_name+' directory does not exist: '+str(path) + message = directory_name+' directory does not exist: '+repr(path) logger.error(message) raise tuf.Error(message) @@ -270,13 +271,13 @@ def receive(config_filepath): # Process all the pushes for each of the pushroots. for pushroot in pushroots: if not os.path.exists(pushroot): - logger.error('The pushroot '+str(pushroot)+' does not exist. Skipping.') + logger.error('The pushroot '+repr(pushroot)+' does not exist. Skipping.') continue # Add the 'processed' and 'processing' directories if not present. # These directories must exist so that we can properly process # a push. - logger.debug('Looking for pushes in pushroot '+str(pushroot)) + logger.debug('Looking for pushes in pushroot '+repr(pushroot)) if not os.path.exists(os.path.join(pushroot, 'processed')): os.mkdir(os.path.join(pushroot, 'processed')) if not os.path.exists(os.path.join(pushroot, 'processing')): @@ -297,7 +298,7 @@ def receive(config_filepath): # Ensure the 'info' file exists. A successful push operation creates # and saves this 'info' file to the push directory. if not os.path.exists(os.path.join(pushpath, 'info')): - message = 'Skipping incomplete push '+str(pushpath)+' (no info file).' + message = 'Skipping incomplete push '+repr(pushpath)+' (no info file).' logger.warn(message) continue @@ -313,7 +314,7 @@ def receive(config_filepath): # Done. Log the result of processing the pushes for 'pushroot'. message = 'Completed processing of all pushes. Push successes = '+\ - str(success_count)+', failures = '+str(failure_count)+'.' + repr(success_count)+', failures = '+repr(failure_count)+'.' logger.info(message) @@ -364,11 +365,11 @@ def _process_new_push(pushroot, pushname, metadata_directory, """ - logger.info('Processing '+str(pushroot)+'/'+str(pushname)) + logger.info('Processing '+repr(pushroot)+'/'+repr(pushname)) # Move the pushed directory to the 'processing' directory. pushpath = os.path.join(pushroot, 'processing', pushname) - logger.debug('Moving push directory to '+str(pushpath)) + logger.debug('Moving push directory to '+repr(pushpath)) if os.path.isdir(pushpath) or os.path.isfile(pushpath): os.remove(pushpath) os.rename(os.path.join(pushroot, pushname), pushpath) @@ -418,14 +419,14 @@ def _process_new_push(pushroot, pushname, metadata_directory, finally: file_object.close() - message = 'Could not process: '+str(pushroot)+'/'+str(pushname) + message = 'Could not process: '+repr(pushroot)+'/'+repr(pushname) logger.exception(message) return False # On success or failure, move 'pushpath' to the processed directory. finally: processedpath = os.path.join(pushroot, 'processed', pushname) - logger.debug('Moving push directory to '+str(processedpath)) + logger.debug('Moving push directory to '+repr(processedpath)) if os.path.isdir(processedpath) or os.path.isfile(processedpath): os.remove(processedpath) os.rename(pushpath, processedpath) @@ -575,17 +576,17 @@ def _process_copied_push(pushpath, metadata_directory, # Allowing equality makes testing/development easier. if formatted_timestamp > new_formatted_timestamp: - message = 'Existing metadata timestamp '+str(timestamp)+' is newer '+\ - 'than the new metadata\'s timestamp '+str(new_timestamp) + message = 'Existing metadata timestamp '+repr(timestamp)+' is newer '+\ + 'than the new metadata\'s timestamp '+repr(new_timestamp) raise tuf.Error(message) else: - message = 'New metadata timestamp is '+str(new_timestamp)+'. '+\ - ' Replacing old metadata with timestamp '+str(timestamp) + message = 'New metadata timestamp is '+repr(new_timestamp)+'. '+\ + ' Replacing old metadata with timestamp '+repr(timestamp) logger.debug(message) # There appears to be no 'targets.txt' metadata file on the repository. else: - message = 'The old targets metadata file '+str(targets_metadatapath)+'. '+\ + message = 'The old targets metadata file '+repr(targets_metadatapath)+'. '+\ 'doesn\'t exist in the repo. Skipping the timestamp check.' logger.warn(message) @@ -594,10 +595,10 @@ def _process_copied_push(pushpath, metadata_directory, formatted_expiration = tuf.formats.parse_time(expiration) if formatted_expiration <= time.time(): - message = 'Pushed metadata expired at '+str(expiration) + message = 'Pushed metadata expired at '+repr(expiration) raise tuf.Error(message) else: - message = 'Metadata will expire at '+str(expiration) + message = 'Metadata will expire at '+repr(expiration) logger.debug(message) # Verify the signatures of the new targets metadata. @@ -608,11 +609,11 @@ def _process_copied_push(pushpath, metadata_directory, # Log the status of the signatures. For example, the number of good, # bad, untrusted, unknown, signatures. status = tuf.sig.get_signature_status(new_targets_signable, 'targets') - logger.debug(str(status)) + logger.debug(repr(status)) # Log the number of targets specified in the new targets metadata file. targets_count = len(new_targets_signable['signed']['targets'].keys()) - message = 'Number of targets specified: '+str(targets_count) + message = 'Number of targets specified: '+repr(targets_count) logger.info(message) # Verify the files of the new targets metadata file. @@ -625,39 +626,39 @@ def _process_copied_push(pushpath, metadata_directory, # Check that the target was provided. if not os.path.exists(targetpath): message = 'The specified target file was not provided: '+\ - str(target_relativepath) + repr(target_relativepath) raise tuf.Error(message) # Check the target's size. A valid size is required of target files. target_size = os.path.getsize(targetpath) if target_size != target_info['length']: - message = 'The size of target file '+str(target_relativepath)+\ - ' is incorrect: was '+str(target_size)+', expected '+\ - str(target_info['length']) + message = 'The size of target file '+repr(target_relativepath)+\ + ' is incorrect: was '+repr(target_size)+', expected '+\ + repr(target_info['length']) raise tuf.Error(message) else: - message = 'Size of target '+str(targetpath)+' is correct '+\ - '('+str(target_size)+' bytes).' + message = 'Size of target '+repr(targetpath)+' is correct '+\ + '('+repr(target_size)+' bytes).' logger.debug(message) # Check hashes. Valid target files is required. hash_count = len(target_info['hashes'].items()) if hash_count == 0: - message = str(targetpath)+' contains an empty hashes dictionary.' + message = repr(targetpath)+' contains an empty hashes dictionary.' raise tuf.Error(message) else: - logger.debug(str(hash_count)+' hash(es) to check.') + logger.debug(repr(hash_count)+' hash(es) to check.') for algorithm, digest in target_info['hashes'].items(): digest_object = tuf.hash.digest_filename(targetpath, algorithm=algorithm) if digest_object.hexdigest() != digest: - message = str(algorithm)+' hash does not match: '+\ - ' was '+str(digest_object.hexdigest())+', expected '+\ - str(digest) + message = repr(algorithm)+' hash does not match: '+\ + ' was '+repr(digest_object.hexdigest())+', expected '+\ + repr(digest) raise tuf.Error(message) else: - message = str(algorithm)+' hash of target '+str(targetpath)+\ - ' is correct ('+str(digest)+').' + message = repr(algorithm)+' hash of target '+repr(targetpath)+\ + ' is correct ('+repr(digest)+').' logger.debug(message) # At this point, the targets metadata and all specified files have been @@ -673,14 +674,14 @@ def _process_copied_push(pushpath, metadata_directory, source_path = os.path.join(push_temporary_directory, targets_basename, target_relativepath) destination_path = os.path.join(targets_directory, target_relativepath) - logger.info('Adding target to repository: '+str(destination_path)) + logger.info('Adding target to repository: '+repr(destination_path)) destination_directory = os.path.dirname(destination_path) if not os.path.exists(destination_directory): os.mkdir(destination_directory) shutil.copy(source_path, destination_path) # Copy the new targets metadata file into place on the repository. - message = 'Adding new targets metadata to repository: '+str(targets_metadatapath) + message = 'Adding new targets metadata to repository: '+repr(targets_metadatapath) logger.info(message) shutil.copy(new_targets_metadatapath, targets_metadatapath) @@ -745,7 +746,7 @@ def _remove_old_files(targets_metadatapath, pushname, for target_relativepath in targets_signable['signed']['targets'].keys(): targetpath = os.path.join(targets_directory, target_relativepath) backup_targetpath = os.path.join(backup_targetsdirectory, target_relativepath) - message = 'Backing up target '+str(targetpath)+' to '+str(backup_targetpath) + message = 'Backing up target '+repr(targetpath)+' to '+repr(backup_targetpath) logger.info(message) # Move the old target file to the backup directory. Create any @@ -760,13 +761,13 @@ def _remove_old_files(targets_metadatapath, pushname, raise tuf.Error(str(e)) os.rename(targetpath, backup_targetpath) else: - message = 'The old target '+str(targetpath)+' doesn\'t exist in the repo.' + message = 'The old target '+repr(targetpath)+' doesn\'t exist in the repo.' logger.warn(message) # Backup the old 'targets.txt' metadata file. backup_targets_metadatafile = os.path.join(backup_destdirectory, 'targets.txt') - message = 'Backing up old metadata '+str(targets_metadatapath)+\ - ' to '+str(backup_targets_metadatafile) + message = 'Backing up old metadata '+repr(targets_metadatapath)+\ + ' to '+repr(backup_targets_metadatafile) logger.info(message) if os.path.isfile(backup_targets_metadatafile): os.remove(backup_targets_metadatafile) diff --git a/tuf/pushtools/transfer/scp.py b/tuf/pushtools/transfer/scp.py index c844f4e9..d4aa7b0d 100755 --- a/tuf/pushtools/transfer/scp.py +++ b/tuf/pushtools/transfer/scp.py @@ -54,6 +54,7 @@ import tempfile import time +import tuf import tuf.formats diff --git a/tuf/repo/quickstart.py b/tuf/repo/quickstart.py index ff67890d..e51373da 100755 --- a/tuf/repo/quickstart.py +++ b/tuf/repo/quickstart.py @@ -102,7 +102,7 @@ import tuf.log # See 'log.py' to learn how logging is handled in TUF. -logger = logging.getLogger('tuf') +logger = logging.getLogger('tuf.quickstart') # Set the default file names for the top-level roles. # For instance: in 'signerlib.py', ROOT_FILENAME = 'root.txt'. diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index d3d7b1d9..a8d9c825 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -58,16 +58,17 @@ import errno import tuf +import tuf.formats import tuf.repo.signerlib import tuf.repo.keystore import tuf.util import tuf.log -json = tuf.util.import_json() - # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.signercli') +json = tuf.util.import_json() + # The maximum number of attempts the user has to enter # valid input. MAX_INPUT_ATTEMPTS = 3 diff --git a/tuf/repo/signerlib.py b/tuf/repo/signerlib.py index 01badeee..3dfd6226 100755 --- a/tuf/repo/signerlib.py +++ b/tuf/repo/signerlib.py @@ -23,6 +23,7 @@ import ConfigParser import logging +import tuf import tuf.formats import tuf.rsa_key import tuf.repo.keystore diff --git a/tuf/roledb.py b/tuf/roledb.py index f6446731..4f114dfc 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -29,9 +29,10 @@ """ -import tuf.formats import logging +import tuf +import tuf.formats import tuf.log # See 'tuf.log' to learn how logging is handled in TUF. diff --git a/tuf/rsa_key.py b/tuf/rsa_key.py index bc1767a0..a86a6138 100755 --- a/tuf/rsa_key.py +++ b/tuf/rsa_key.py @@ -42,6 +42,8 @@ import evpy.signature import evpy.envelope +import tuf + # Digest objects needed to generate hashes. import tuf.hash diff --git a/tuf/sig.py b/tuf/sig.py index 32faae89..ce5b9f56 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -36,6 +36,7 @@ """ +import tuf import tuf.formats import tuf.keydb import tuf.roledb diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/system_tests/util_test_tools.py index 1c263068..969913db 100755 --- a/tuf/tests/system_tests/util_test_tools.py +++ b/tuf/tests/system_tests/util_test_tools.py @@ -137,6 +137,7 @@ import subprocess import tuf +import tuf.formats import tuf.interposition import tuf.util import tuf.client.updater @@ -144,7 +145,7 @@ import tuf.repo.signerlib as signerlib import tuf.repo.keystore as keystore -logger = logging.getLogger('tuf') +logger = logging.getLogger('tuf.tests.system_tests.util_test_tools') # Disable logging for cleaner output. def disable_logging(): @@ -172,7 +173,8 @@ def init_repo(tuf=False, port=None): # Start a simple server pointing to the repository directory. port = random.randint(30000, 45000) command = ['python', '-m', 'SimpleHTTPServer', str(port)] - server_proc = subprocess.Popen(command, stderr=subprocess.PIPE) + server_proc = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) # Tailor url for the repository. In order to download a 'file.txt' # from 'reg_repo' do: url+'reg_repo/file.txt' diff --git a/tuf/tests/test_download.py b/tuf/tests/test_download.py index 96716875..d2e0e38d 100755 --- a/tuf/tests/test_download.py +++ b/tuf/tests/test_download.py @@ -19,12 +19,9 @@ NOTE: Make sure test_download.py is ran in 'tuf/tests/' directory. Otherwise, module that launches simple server would not be found. + """ -import tuf -import tuf.log -import tuf.download as download -import tuf.tests.unittest_toolbox as unittest_toolbox import os import sys @@ -37,11 +34,12 @@ import SocketServer import SimpleHTTPServer -logger = logging.getLogger('tuf') +import tuf +import tuf.log +import tuf.download as download +import tuf.tests.unittest_toolbox as unittest_toolbox -# Disable/Enable logging. Comment-out to Enable logging. -logging.getLogger('tuf') -logging.disable(logging.CRITICAL) +logger = logging.getLogger('tuf.test_download') class TestDownload(unittest_toolbox.Modified_TestCase): diff --git a/tuf/tests/test_formats.py b/tuf/tests/test_formats.py index ecc5338b..1e375ca3 100755 --- a/tuf/tests/test_formats.py +++ b/tuf/tests/test_formats.py @@ -20,6 +20,7 @@ import unittest +import tuf import tuf.formats import tuf.schema diff --git a/tuf/tests/test_hash.py b/tuf/tests/test_hash.py index 0a56c175..8e634a81 100755 --- a/tuf/tests/test_hash.py +++ b/tuf/tests/test_hash.py @@ -23,9 +23,12 @@ import tempfile import unittest +import tuf +import tuf.log import tuf.hash -logger = logging.getLogger('tuf') +logger = logging.getLogger('tuf.test_hash') + if not 'hashlib' in tuf.hash._supported_libraries: logger.warn('Not testing hashlib: could not be imported.') diff --git a/tuf/tests/test_keydb.py b/tuf/tests/test_keydb.py index 7e61f223..c7330875 100755 --- a/tuf/tests/test_keydb.py +++ b/tuf/tests/test_keydb.py @@ -19,15 +19,13 @@ import unittest import logging +import tuf +import tuf.formats import tuf.rsa_key import tuf.keydb import tuf.log -logger = logging.getLogger('tuf') - -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) +logger = logging.getLogger('tuf.test_keydb') # Generate the three keys to use in our test cases. diff --git a/tuf/tests/test_keystore.py b/tuf/tests/test_keystore.py index 816f400a..5302d5f2 100755 --- a/tuf/tests/test_keystore.py +++ b/tuf/tests/test_keystore.py @@ -21,16 +21,14 @@ import os import logging +import tuf import tuf.repo.keystore import tuf.rsa_key import tuf.formats import tuf.util +import tuf.log -logger = logging.getLogger('tuf') - -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) +logger = logging.getLogger('tuf.test_keystore') # We'll need json module for testing '_encrypt()' and '_decrypt()' # internal function. diff --git a/tuf/tests/test_push.py b/tuf/tests/test_push.py index 699881c0..c491b22f 100755 --- a/tuf/tests/test_push.py +++ b/tuf/tests/test_push.py @@ -24,20 +24,17 @@ import ConfigParser import tuf +import tuf.log import tuf.pushtools.push as push import tuf.pushtools.transfer.scp as scp import tuf.pushtools.pushtoolslib as pushtoolslib -import system_tests.util_test_tools as util_test_tools +import tuf.tests.system_tests.util_test_tools as util_test_tools -logger = logging.getLogger('tuf') +logger = logging.getLogger('tuf.test_push') -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) class TestPush(unittest.TestCase): src_push_dict = {} - logger.info(scp.transfer) ORIGINAL_PUSH_CONFIG = pushtoolslib.PUSH_CONFIG diff --git a/tuf/tests/test_pushtoolslib.py b/tuf/tests/test_pushtoolslib.py index 60c4975f..e0cf6694 100755 --- a/tuf/tests/test_pushtoolslib.py +++ b/tuf/tests/test_pushtoolslib.py @@ -20,10 +20,14 @@ import tempfile import unittest import ConfigParser +import logging +import tuf +import tuf.log import tuf.formats import tuf.pushtools.pushtoolslib as pushtoolslib +logger = logging.getLogger('tuf.test_pushtoolslib') class TestPushtoolslib(unittest.TestCase): diff --git a/tuf/tests/test_quickstart.py b/tuf/tests/test_quickstart.py index f0ec3544..a211f905 100755 --- a/tuf/tests/test_quickstart.py +++ b/tuf/tests/test_quickstart.py @@ -25,19 +25,19 @@ import shutil import unittest import logging -import tuf.repo.quickstart as quickstart +import tuf +import tuf.log +import tuf.repo.quickstart as quickstart import tuf.util import tuf.tests.unittest_toolbox - -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) - +logger = logging.getLogger('tuf.test_quickstart') unit_tbox = tuf.tests.unittest_toolbox.Modified_TestCase +logger.info('from test_quickstart') + class TestQuickstart(unit_tbox): def test_1_get_password(self): diff --git a/tuf/tests/test_roledb.py b/tuf/tests/test_roledb.py index 7badf680..6d068054 100755 --- a/tuf/tests/test_roledb.py +++ b/tuf/tests/test_roledb.py @@ -16,20 +16,18 @@ """ + import unittest import logging +import tuf import tuf.formats import tuf.rsa_key import tuf.roledb import tuf.log +logger = logging.getLogger('tuf.test_roledb') -logger = logging.getLogger('tuf') - -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) # Generate the three keys to use in our test cases. KEYS = [] diff --git a/tuf/tests/test_rsa_key.py b/tuf/tests/test_rsa_key.py index d6543378..a7b8708d 100755 --- a/tuf/tests/test_rsa_key.py +++ b/tuf/tests/test_rsa_key.py @@ -23,10 +23,14 @@ """ import unittest +import logging +import tuf +import tuf.log import tuf.formats import tuf.rsa_key +logger = logging.getLogger('tuf.test_rsa_key') RSA_KEY = tuf.rsa_key FORMAT_ERROR_MSG = 'tuf.FormatError was raised! Check object\'s format.' diff --git a/tuf/tests/test_schema.py b/tuf/tests/test_schema.py index f72ae02b..66320a01 100755 --- a/tuf/tests/test_schema.py +++ b/tuf/tests/test_schema.py @@ -17,9 +17,14 @@ """ import unittest +import logging +import tuf +import tuf.log import tuf.schema +logger = logging.getLogger('tuf.test_schema') + class TestSchema(unittest.TestCase): def setUp(self): diff --git a/tuf/tests/test_sig.py b/tuf/tests/test_sig.py index dd302748..efdbe516 100755 --- a/tuf/tests/test_sig.py +++ b/tuf/tests/test_sig.py @@ -17,13 +17,19 @@ """ -import unittest +import unittest +import logging + +import tuf +import tuf.log +import tuf.formats import tuf.keydb import tuf.roledb import tuf.rsa_key import tuf.sig +logger = logging.getLogger('tuf.test_sig') # Setup the keys to use in our test cases. KEYS = [] diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index aa91d862..61ba1ef4 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -42,28 +42,25 @@ class guarantees the order of unit tests. So that, 'test_something_A' import unittest import tuf +import tuf.log import tuf.formats import tuf.util import tuf.repo.keystore as keystore import tuf.repo.signerlib as signerlib + # Module to test: signercli.py import tuf.repo.signercli as signercli + # Helper module unittest_toolbox.py import tuf.tests.unittest_toolbox as unittest_toolbox - -logger = logging.getLogger('tuf') - -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) +logger = logging.getLogger('tuf.test_signercli') # Populating 'rsa_keystore' and 'rsa_passwords' dictionaries. # We will need them when creating keystore directories. unittest_toolbox.Modified_TestCase.bind_keys_to_roles() - class TestSignercli(unittest_toolbox.Modified_TestCase): # SETUP original_prompt = signercli._prompt diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index dcb47993..9008350e 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -54,6 +54,8 @@ import logging import unittest +import tuf +import tuf.log import tuf.util import tuf.formats as formats import tuf.repo.signerlib as signerlib @@ -61,12 +63,7 @@ import tuf.tests.unittest_toolbox - -logger = logging.getLogger('tuf') - -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) +logger = logging.getLogger('tuf.test_signerlib') # 'unittest_toolbox.Modified_TestCase' is too long, I'll set it to 'unit_tbox'. unit_tbox = tuf.tests.unittest_toolbox.Modified_TestCase diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index d32664e7..dc708d62 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -43,6 +43,9 @@ class guarantees the order of unit tests. So that, 'test_something_A' import logging import unittest + +import tuf +import tuf.log import tuf.util import tuf.formats import tuf.repo.keystore as keystore @@ -51,11 +54,7 @@ class guarantees the order of unit tests. So that, 'test_something_A' import tuf.tests.repository_setup as setup import tuf.tests.unittest_toolbox as unittest_toolbox -logger = logging.getLogger('tuf') - -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) +logger = logging.getLogger('tuf.test_updater') # References to roledb and keydb dictionaries (improve readability). roledb = tuf.roledb diff --git a/tuf/tests/test_util.py b/tuf/tests/test_util.py index 9636d606..03da2b93 100755 --- a/tuf/tests/test_util.py +++ b/tuf/tests/test_util.py @@ -30,9 +30,7 @@ import tuf.util as util import tuf.tests.unittest_toolbox as unittest_toolbox -# Disable all logging calls of level CRITICAL and below. -# Comment the line below to enable logging. -logging.disable(logging.CRITICAL) +logger = logging.getLogger('tuf.test_util') class TestUtil(unittest_toolbox.Modified_TestCase): diff --git a/tuf/tests/unittest_toolbox.py b/tuf/tests/unittest_toolbox.py index 5c8b54ff..02247f8f 100755 --- a/tuf/tests/unittest_toolbox.py +++ b/tuf/tests/unittest_toolbox.py @@ -159,10 +159,11 @@ def setUp(self): def tearDown(self): # Removing 'tuf.log' file from current working directory. + """ tuf_log_path = os.path.join(os.getcwd(), 'tuf.log') if os.path.exists(tuf_log_path): os.unlink(tuf_log_path) - + """ for cleanup_function in self._cleanup: # Perform clean up by executing clean-up functions. try: diff --git a/tuf/util.py b/tuf/util.py index 0fb6291d..3c72023a 100755 --- a/tuf/util.py +++ b/tuf/util.py @@ -27,6 +27,7 @@ import logging import tempfile +import tuf import tuf.hash import tuf.conf import tuf.formats From e7018bf96950df6291b8a9fc3fc98f33671f2365 Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 15 Aug 2013 15:48:12 -0400 Subject: [PATCH 037/119] Update system tests following design change to log.py --- tuf/tests/system_tests/slow_retrieval_server.py | 4 +--- .../system_tests/test_arbitrary_package_attack.py | 7 +------ tuf/tests/system_tests/test_endless_data_attack.py | 6 ------ .../test_extraneous_dependencies_attack.py | 9 ++++----- .../system_tests/test_indefinite_freeze_attack.py | 7 +------ tuf/tests/system_tests/test_mix_and_match_attack.py | 7 +------ tuf/tests/system_tests/test_replay_attack.py | 10 ++-------- tuf/tests/system_tests/test_slow_retrieval_attack.py | 12 ++++-------- tuf/tests/system_tests/util_test_tools.py | 6 ------ tuf/tests/unittest_toolbox.py | 6 ------ 10 files changed, 14 insertions(+), 60 deletions(-) diff --git a/tuf/tests/system_tests/slow_retrieval_server.py b/tuf/tests/system_tests/slow_retrieval_server.py index 0887d119..161277b6 100755 --- a/tuf/tests/system_tests/slow_retrieval_server.py +++ b/tuf/tests/system_tests/slow_retrieval_server.py @@ -24,11 +24,9 @@ import random from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer - DELAY = 1 - # HTTP request handler. class Handler(BaseHTTPRequestHandler): @@ -76,4 +74,4 @@ def run(port): else: port = get_random_port() - run(port) \ No newline at end of file + run(port) diff --git a/tuf/tests/system_tests/test_arbitrary_package_attack.py b/tuf/tests/system_tests/test_arbitrary_package_attack.py index 1ddcb053..b5f207ce 100755 --- a/tuf/tests/system_tests/test_arbitrary_package_attack.py +++ b/tuf/tests/system_tests/test_arbitrary_package_attack.py @@ -40,11 +40,6 @@ -# Disable logging. -util_test_tools.disable_logging() - - - class ArbitraryPackageAlert(Exception): pass @@ -149,4 +144,4 @@ def test_arbitrary_package_attack(TUF=False): test_arbitrary_package_attack(TUF=True) except ArbitraryPackageAlert, error: - print error \ No newline at end of file + print error diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 9366accf..4cac5531 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -43,12 +43,6 @@ from tuf.interposition import urllib_tuf - -# Disable logging. -util_test_tools.disable_logging() - - - class EndlessDataAttack(Exception): pass diff --git a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py index 72937d65..de327ea7 100755 --- a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py +++ b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py @@ -27,18 +27,17 @@ import tempfile import time -import util_test_tools +import tuf +import tuf.formats +import tuf.tests.system_tests.util_test_tools import tuf.repo.keystore import tuf.repo.signerlib as signerlib import tuf.repo.signercli as signercli from tuf.interposition import urllib_tuf - -# Disable logging. -util_test_tools.disable_logging() - version = 1 + class ExtraneousDependenciesAttackAlert(Exception): pass diff --git a/tuf/tests/system_tests/test_indefinite_freeze_attack.py b/tuf/tests/system_tests/test_indefinite_freeze_attack.py index 9a5bb1a0..85f457d2 100755 --- a/tuf/tests/system_tests/test_indefinite_freeze_attack.py +++ b/tuf/tests/system_tests/test_indefinite_freeze_attack.py @@ -27,21 +27,16 @@ import tempfile import util_test_tools +import tuf import tuf.formats import tuf.repo.signerlib as signerlib from tuf.interposition import urllib_tuf -# Disable logging. -util_test_tools.disable_logging() - - - class IndefiniteFreezeAttackAlert(Exception): pass - EXPIRATION = 1 # second(s) version = 1 diff --git a/tuf/tests/system_tests/test_mix_and_match_attack.py b/tuf/tests/system_tests/test_mix_and_match_attack.py index 94c40057..29ee9fcf 100755 --- a/tuf/tests/system_tests/test_mix_and_match_attack.py +++ b/tuf/tests/system_tests/test_mix_and_match_attack.py @@ -44,11 +44,6 @@ from tuf.interposition import urllib_tuf -# Disable logging. -util_test_tools.disable_logging() - - - class MixAndMatchAttackAlert(Exception): pass @@ -195,4 +190,4 @@ def test_mix_and_match_attack(TUF=False): try: test_mix_and_match_attack(TUF=True) except MixAndMatchAttackAlert, error: - print error \ No newline at end of file + print error diff --git a/tuf/tests/system_tests/test_replay_attack.py b/tuf/tests/system_tests/test_replay_attack.py index 4250a854..1c190dfa 100755 --- a/tuf/tests/system_tests/test_replay_attack.py +++ b/tuf/tests/system_tests/test_replay_attack.py @@ -38,16 +38,10 @@ import urllib import tempfile -import util_test_tools +import tuf.tests.system_tests.util_test_tools as util_test_tools from tuf.interposition import urllib_tuf - -# Disable logging. -util_test_tools.disable_logging() - - - class TestSetupError(Exception): pass @@ -189,4 +183,4 @@ def test_replay_attack(TUF=False): try: test_replay_attack(TUF=True) except ReplayAttackAlert, error: - print error \ No newline at end of file + print error diff --git a/tuf/tests/system_tests/test_slow_retrieval_attack.py b/tuf/tests/system_tests/test_slow_retrieval_attack.py index d2f9f4d9..7d25a7d2 100755 --- a/tuf/tests/system_tests/test_slow_retrieval_attack.py +++ b/tuf/tests/system_tests/test_slow_retrieval_attack.py @@ -42,15 +42,10 @@ import subprocess from multiprocessing import Process -import util_test_tools +import tuf.tests.system_tests.util_test_tools as util_test_tools from tuf.interposition import urllib_tuf -# Disable logging. -util_test_tools.disable_logging() - - - class SlowRetrievalAttackAlert(Exception): pass @@ -72,7 +67,8 @@ def test_slow_retrieval_attack(TUF=False): # Launch the server. port = random.randint(30000, 45000) command = ['python', 'slow_retrieval_server.py', str(port)] - server_process = subprocess.Popen(command, stderr=subprocess.PIPE) + server_process = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) time.sleep(.1) try: @@ -134,4 +130,4 @@ def test_slow_retrieval_attack(TUF=False): try: test_slow_retrieval_attack(TUF=True) except SlowRetrievalAttackAlert, error: - print error \ No newline at end of file + print error diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/system_tests/util_test_tools.py index 969913db..40024a42 100755 --- a/tuf/tests/system_tests/util_test_tools.py +++ b/tuf/tests/system_tests/util_test_tools.py @@ -147,12 +147,6 @@ logger = logging.getLogger('tuf.tests.system_tests.util_test_tools') -# Disable logging for cleaner output. -def disable_logging(): - logging.getLogger('tuf') - logging.disable(logging.CRITICAL) - - PASSWD = 'test' version = 1 diff --git a/tuf/tests/unittest_toolbox.py b/tuf/tests/unittest_toolbox.py index 02247f8f..673c8ceb 100755 --- a/tuf/tests/unittest_toolbox.py +++ b/tuf/tests/unittest_toolbox.py @@ -158,12 +158,6 @@ def setUp(self): def tearDown(self): - # Removing 'tuf.log' file from current working directory. - """ - tuf_log_path = os.path.join(os.getcwd(), 'tuf.log') - if os.path.exists(tuf_log_path): - os.unlink(tuf_log_path) - """ for cleanup_function in self._cleanup: # Perform clean up by executing clean-up functions. try: From 6f297650fd3895880665d8ef37d0bac636e3c065 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Thu, 15 Aug 2013 17:40:48 -0400 Subject: [PATCH 038/119] Removed unnecessary imports from aggregate_tests.py. --- tuf/tests/aggregate_tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index b5dd21b7..42a1db18 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -24,9 +24,6 @@ import unittest import glob -import tuf.keydb as keydb -import tuf.repo.keystore as keystore -import tuf.roledb as roledb import random tests_list = glob.glob('test_*.py') From 817ad0735b927a10c9cb7588e44a1059af7318da Mon Sep 17 00:00:00 2001 From: zanefisher Date: Mon, 19 Aug 2013 17:10:13 -0400 Subject: [PATCH 039/119] Rewrote extraneous dependencies test, based on arbitrary package test. --- .../test_extraneous_dependencies_attack.py | 236 ++++++++---------- 1 file changed, 109 insertions(+), 127 deletions(-) mode change 100755 => 100644 tuf/tests/system_tests/test_extraneous_dependencies_attack.py diff --git a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py old mode 100755 new mode 100644 index 72937d65..7492a070 --- a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py +++ b/tuf/tests/system_tests/test_extraneous_dependencies_attack.py @@ -1,190 +1,172 @@ -#!/usr/bin/env python - """ test_extraneous_dependencies_attack.py - Konstantin Andrianov + Zane Fisher - February 19, 2012 + August 19, 2013 See LICENSE for licensing information. - Simulate an extraneous dependencies attack. + Simulate an extraneous dependencies attack. The client attempts to download + a file with one legitimate dependency, and one extraneous dependency. - In an extraneous dependencies attack, attacker is able to cause clients to - download software dependencies that are not the intended dependencies. +Note: The interposition provided by 'tuf.interposition' is used to intercept +all calls made by urllib/urillib2 to certain hostnames specified in +the interposition configuration file. Look up interposition.py for more +information and illustration of a sample contents of the interposition +configuration file. Interposition was meant to make TUF integration with an +existing software updater an easy process. This allows for more flexibility +to the existing software updater. However, if you are planning to solely use +TUF there should be no need for interposition, all necessary calls will be +generated from within TUF. + +Note: There is no difference between 'updates' and 'target' files. """ import os -import sys +import shutil import urllib import tempfile -import time - import util_test_tools -import tuf.repo.keystore -import tuf.repo.signerlib as signerlib -import tuf.repo.signercli as signercli + +import tuf from tuf.interposition import urllib_tuf + # Disable logging. util_test_tools.disable_logging() -version = 1 -class ExtraneousDependenciesAttackAlert(Exception): + +class ExtraneousDependencyAlert(Exception): pass -def test_extraneous_dependencies_attack(): +# Interprets the contents of the file it downloads as a list of dependent +# files from the same repository +def _download(url, filename, directory, tuf=False): + destination = os.path.join(directory, filename) + #print 'downloading to '+destination + #print 'from '+url + if tuf: + urllib_tuf.urlretrieve(url, destination) + else: + urllib.urlretrieve(url, destination) + + #print 'DDDDDD '+util_test_tools.read_file_content(destination) + if util_test_tools.read_file_content(destination) != '': + required_files = util_test_tools.read_file_content(destination).split(',') + for required_filename in required_files: + #print 'requires '+required_filename + required_file_url = os.path.dirname(url)+'/'+required_filename + _download(required_file_url, required_filename, directory, tuf) + + + +def test_extraneous_dependency_attack(TUF=False): + """ + + TUF: + If set to 'False' all directories that start with 'tuf_' are ignored, + indicating that tuf is not implemented. + + + Illustrate arbitrary package attack vulnerability. + + """ + + ERROR_MSG = 'Extraneous Dependency Attack was Successful!\n' - ERROR_MSG = '\tExtraneous Dependencies Attack Succeeded!\n\n' try: - # Setup. - root_repo, url, server_proc, keyids = util_test_tools.init_repo(tuf=True) + root_repo, url, server_proc, keyids = util_test_tools.init_repo(tuf=TUF) reg_repo = os.path.join(root_repo, 'reg_repo') tuf_repo = os.path.join(root_repo, 'tuf_repo') - keystore_dir = os.path.join(tuf_repo, 'keystore') - metadata_dir = os.path.join(tuf_repo, 'metadata') - downloads_dir = os.path.join(root_repo, 'downloads') + downloads = os.path.join(root_repo, 'downloads') targets_dir = os.path.join(tuf_repo, 'targets') - # 'roles' holds information about delegated roles. - roles = {'role1':{'password':['pass1']}, - 'role2':{'password':['pass2']}} + # Add files to 'repo' directory: {root_repo} + good_dependency_filepath = util_test_tools.add_file_to_repository(reg_repo, '') + good_dependency_basename = os.path.basename(good_dependency_filepath) - # Add files to 'reg_repo' directory: {root_repo} - role1_path = tempfile.mkdtemp(dir=reg_repo) - roles['role1']['filepath'] = \ - util_test_tools.add_file_to_repository(role1_path, 'Test A') + bad_dependency_filepath = util_test_tools.add_file_to_repository(reg_repo, '') + bad_dependency_basename = os.path.basename(bad_dependency_filepath) - role2_path = tempfile.mkdtemp(dir=reg_repo) - roles['role2']['filepath'] = \ - util_test_tools.add_file_to_repository(role2_path, 'Test B') + # The dependent file lists the good dependency + dependent_filepath = util_test_tools.add_file_to_repository(reg_repo, good_dependency_basename) + dependent_basename = os.path.basename(dependent_filepath) - # Update TUF repository. - util_test_tools.make_targets_meta(root_repo) - util_test_tools.make_release_meta(root_repo) - util_test_tools.make_timestamp_meta(root_repo) + url_to_repo = url+'reg_repo/'+dependent_basename + #downloaded_file = os.path.join(downloads, dependent_basename) + modified_dependency_list = good_dependency_basename+','+bad_dependency_basename - - def _make_delegation(rolename): - expiration_date = tuf.formats.format_time(time.time()+86400) - expiration_date = expiration_date[0:expiration_date.rfind(' UTC')] - # Indicate which file client downloads. - rel_filepath = os.path.relpath(roles[rolename]['filepath'], reg_repo) - roles[rolename]['target_path'] = os.path.join(targets_dir, rel_filepath) - rolepath, file_basename = os.path.split(roles[rolename]['filepath']) - junk, role_relpath = os.path.split(rolepath) - roles[rolename]['targets_dir'] = os.path.join(targets_dir, role_relpath) - roles[rolename]['metadata_dir'] = os.path.join(metadata_dir, 'targets') - - # Create a key to sign a new delegated role. - password = roles[rolename]['password'][0] - key = signerlib.generate_and_save_rsa_key(keystore_dir, password) - roles[rolename]['keyid'] = [key['keyid']] - roles[rolename]['dest_path'] = os.path.join(downloads_dir, file_basename) - - # Create delegation one. - util_test_tools.create_delegation(tuf_repo, - roles[rolename]['targets_dir'], - roles[rolename]['keyid'], password, - 'targets', rolename, expiration_date) - - # Update TUF repository. - # util_test_tools.make_targets_meta(root_repo) - util_test_tools.make_release_meta(root_repo) - util_test_tools.make_timestamp_meta(root_repo) + if TUF: + # Update TUF metadata before attacker modifies anything. + util_test_tools.tuf_refresh_repo(root_repo, keyids) # Modify the url. Remember that the interposition will intercept # urls that have 'localhost:9999' hostname, which was specified in # the json interposition configuration file. Look for 'hostname' # in 'util_test_tools.py'. Further, the 'file_basename' is the target # path relative to 'targets_dir'. - roles[rolename]['url'] = 'http://localhost:9999/'+rel_filepath + url_to_repo = 'http://localhost:9999/'+dependent_basename - # Perform a client download. - urllib_tuf.urlretrieve(roles[rolename]['url'], - roles[rolename]['dest_path']) + # Attacker adds the dependency in the targets repository. + target = os.path.join(targets_dir, dependent_basename) + util_test_tools.modify_file_at_repository(target, modified_dependency_list) + + # Attacker adds the dependency in the regular repository. + util_test_tools.modify_file_at_repository(dependent_filepath, modified_dependency_list) + + # End of Setup. - _make_delegation('role1') - _make_delegation('role2') - - - # The attacks. - - def _write_rogue_metadata(): - global version - version = version+1 - expiration_date = tuf.formats.format_time(time.time()+86400) - # Load the keystore before rebuilding the metadata. - tuf.repo.keystore.load_keystore_from_keyfiles(keystore_dir, - roles['role1']['keyid'], - roles['role1']['password']) - - # Rebuild the delegation role metadata. - signerlib.build_delegated_role_file(roles['role2']['targets_dir'], - roles['role1']['keyid'], metadata_dir, - roles['role1']['metadata_dir'], - 'role1.txt', version, expiration_date) - - # Update release and timestamp metadata. - util_test_tools.make_release_meta(root_repo) - util_test_tools.make_timestamp_meta(root_repo) - - - # Modify a target that was delegated to 'role2'. - util_test_tools.modify_file_at_repository(roles['role2']['target_path'], - 'Test NOT B') - - # Update rogue delegatee metadata. - _write_rogue_metadata() - - # Perform another client download. try: - urllib_tuf.urlretrieve(roles['role2']['url'], roles['role2']['dest_path']) - except tuf.MetadataNotAvailableError, e: + # Client downloads (tries to download) the file. + _download(url=url_to_repo, filename=dependent_basename, directory=downloads, tuf=TUF) + + except tuf.DownloadError: + # If tuf.DownloadError is raised, this means that TUF has prevented + # the download of an unrecognized file. Enable the logging to see, + # what actually happened. pass + else: - raise ExtraneousDependenciesAttackAlert(ERROR_MSG) - - - # Add a target file to the directory delegated to 'role2' but not 'role1'. - util_test_tools.add_file_to_repository(roles['role2']['targets_dir'], 'AAAA') - - # Update rogue delegatee metadata. - _write_rogue_metadata() - - # Perform another client download. - try: - urllib_tuf.urlretrieve(roles['role2']['url'], roles['role2']['dest_path']) - except tuf.MetadataNotAvailableError, e: - pass - else: - raise ExtraneousDependenciesAttackAlert(ERROR_MSG) + # Check if the legitimate dependency was downloaded + if not(os.path.exists(os.path.join(downloads, good_dependency_basename))): + raise tuf.DownloadError + # Check if the extraneous dependency was downloaded + if os.path.exists(os.path.join(downloads, bad_dependency_basename)): + raise ExtraneousDependencyAlert(ERROR_MSG) finally: - server_proc.kill() - #util_test_tools.cleanup(root_repo, server_proc) - - + util_test_tools.cleanup(root_repo, server_proc) +print 'Attempting extraneous dependency attack without TUF:' try: - test_extraneous_dependencies_attack() -except ExtraneousDependenciesAttackAlert, error: - print 'error' + test_extraneous_dependency_attack(TUF=False) + +except ExtraneousDependencyAlert, error: + print error + + + +print 'Attempting extraneous dependency attack with TUF:' +try: + test_extraneous_dependency_attack(TUF=True) + +except ExtraneousDependencyAlert, error: + print error From 0b81cf933c607271ee44d84f301364a8d378585a Mon Sep 17 00:00:00 2001 From: vladdd Date: Tue, 20 Aug 2013 13:17:04 -0400 Subject: [PATCH 040/119] Replace evpy crypto calls in rsa_key.py with PyCrypto's equivalent --- tuf/rsa_key.py | 141 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 92 insertions(+), 49 deletions(-) diff --git a/tuf/rsa_key.py b/tuf/rsa_key.py index a86a6138..ac7b648e 100755 --- a/tuf/rsa_key.py +++ b/tuf/rsa_key.py @@ -12,9 +12,9 @@ See LICENSE for licensing information. - The goal of this module is to support public-key cryptography using - the RSA algorithm. The RSA-related functions provided include - generate(), create_signature(), and verify_signature(). The 'evpy' package + The goal of this module is to support public-key cryptography using the RSA + algorithm. The RSA-related functions provided include generate(), + create_signature(), and verify_signature(). The 'PyCrypto' package used by 'rsa_key.py' generates the actual RSA keys and the functions listed above can be viewed as an easy-to-use public interface. Additional functions contained here include create_in_metadata_format() and @@ -24,10 +24,11 @@ needed of RSA keys, such as public and private keys, keyIDs, and an iden- fier. create_signature() and verify_signature() are supplemental functions used for generating RSA signatures and verifying them. + https://en.wikipedia.org/wiki/RSA_(algorithm) Key IDs are used as identifiers for keys (e.g., RSA key). They are the hexadecimal representation of the hash of key object (specifically, the key - object containing only the public key). See 'rsa_key.py' and the + object containing only the public key). Review 'rsa_key.py' and the '_get_keyid()' function to see precisely how keyids are generated. One may get the keyid of a key object by simply accessing the dictionary's 'keyid' key (i.e., rsakey['keyid']). @@ -38,9 +39,16 @@ # Required for hexadecimal conversions. import binascii -# Needed to generate, sign, and verify RSA keys. -import evpy.signature -import evpy.envelope +# Crypto.PublicKey (i.e., PyCrypto public-key cryptography) provides algorithms +# such as Digital Signature Algorithm (DSA) and the ElGamal encryption system. +# 'Crypto.PublicKey' is needed here to generate, sign, and verify RSA keys. We +# import all public-key cryptography algorithms, even though we only need the +# RSA functions. +import Crypto.PublicKey.RSA + +# PyCrypto requires 'Crypto.Hash' hash objects to generate PKCS#1 v1.5 +# signatures (i.e., Crypto.Signature.PKCS1_v1_5). +import Crypto.Signature.PKCS1_v1_5 import tuf @@ -64,7 +72,7 @@ def generate(bits=_DEFAULT_RSA_KEY_BITS): Generate public and private RSA keys, with modulus length 'bits'. In addition, a keyid used as an identifier for RSA keys is generated. - The object returned conforms to tuf.formats.RSAKEY_SCHEMA and as the form: + The object returned conforms to 'tuf.formats.RSAKEY_SCHEMA' and as the form: {'keytype': 'rsa', 'keyid': keyid, 'keyval': {'public': '-----BEGIN RSA PUBLIC KEY----- ...', @@ -74,15 +82,18 @@ def generate(bits=_DEFAULT_RSA_KEY_BITS): bits: - The key size, or key length, of the RSA key. + The key size, or key length, of the RSA key. 'bits' must be 1024, or + greater, and a multiple of 256. - tuf.CryptoError, if an exception occurs after calling evpy.envelope.keygen(). + ValueError, if an exception occurs after calling the RSA key generation + routine. 'bits' must be 1024, or greater, and a multiple of 256. tuf.FormatError, if 'bits' does not contain the correct format. - The RSA keys are generated by calling evpy.envelope.keygen(). + The RSA keys are generated by calling PyCrypto's + Crypto.PublicKey.RSA.generate(). A dictionary containing the RSA keys and other identifying information. @@ -100,27 +111,29 @@ def generate(bits=_DEFAULT_RSA_KEY_BITS): rsakey_dict = {} keytype = 'rsa' - # Generate the public and private keys. 'public_key' and 'private_key' - # will both be strings containing RSA keys in PEM format. - # The evpy.envelope module performs the actual key generation. The - # evpy.envelope.keygen() function returns a (public, private) tuple. + # Generate the public and private RSA keys. The PyCrypto module performs + # the actual key generation. Raise 'ValueError' if 'bits' is less than 1024 + # or not a multiple of 256. + rsa_key_object = Crypto.PublicKey.RSA.generate(bits) - try: - public_key, private_key = evpy.envelope.keygen(bits, pem=True) - except (evpy.envelope.EnvelopeError, evpy.envelope.KeygenError, MemoryError), e: - raise tuf.CryptoError(e) + # Extract the public & private halves of the RSA key and generate their + # PEM-formatted representations. The dictionary returned contains the + # private and public RSA keys in PEM format, as strings. + private_key_pem = rsa_key_object.exportKey(format='PEM') + rsa_pubkey = rsa_key_object.publickey() + public_key_pem = rsa_pubkey.exportKey(format='PEM') + # Generate the keyid for the RSA key. 'key_value' corresponds to the - # 'keyval' entry of the RSAKEY_SCHEMA dictionary. - key_value = {'public': public_key, + # 'keyval' entry of the 'RSAKEY_SCHEMA' dictionary. The private key + # information is not included in the generation of the 'keyid' identifier. + key_value = {'public': public_key_pem, 'private': ''} - keyid = _get_keyid(key_value) - # Build the 'rsakey_dict' dictionary. - # Update 'key_value' with the RSA private key prior to adding - # 'key_value' to 'rsakey_dict'. - key_value['private'] = private_key + # Build the 'rsakey_dict' dictionary. Update 'key_value' with the RSA + # private key prior to adding 'key_value' to 'rsakey_dict'. + key_value['private'] = private_key_pem rsakey_dict['keytype'] = keytype rsakey_dict['keyid'] = keyid @@ -167,13 +180,13 @@ def create_in_metadata_format(key_value, private=False): tuf.FormatError, if 'key_value' does not conform to - tuf.formats.KEYVAL_SCHEMA. + 'tuf.formats.KEYVAL_SCHEMA'. None. - An KEY_SCHEMA dictionary. + An 'KEY_SCHEMA' dictionary. """ @@ -213,16 +226,21 @@ def create_from_metadata_format(key_metadata): RSA keys as stored in metadata files use a different format, so this function should be called if an RSA key is extracted from one of these metadata files and needs converting. Generate() creates an entirely - new key and returns it in the format appropriate for keydb and keystore. + new key and returns it in the format appropriate for 'keydb.py' and + 'keystore.py'. key_metadata: The RSA key dictionary as stored in Metadata files, conforming to - tuf.formats.KEY_SCHEMA. + 'tuf.formats.KEY_SCHEMA'. It has the form: + + {'keytype': '...', + 'keyval': {'public': '...', + 'private': '...'}} tuf.FormatError, if 'key_metadata' does not conform to - tuf.formats.KEY_SCHEMA. + 'tuf.formats.KEY_SCHEMA'. None. @@ -246,8 +264,7 @@ def create_from_metadata_format(key_metadata): keyid = _get_keyid(key_value) - # We now have all the required key values. - # Build 'rsakey_dict'. + # We now have all the required key values. Build 'rsakey_dict'. rsakey_dict['keytype'] = keytype rsakey_dict['keyid'] = keyid rsakey_dict['keyval'] = key_value @@ -288,7 +305,9 @@ def create_signature(rsakey_dict, data): """ Return a signature dictionary of the form: - {'keyid': keyid, 'method': 'evp', 'sig': sig}. + {'keyid': keyid, + 'method': 'PyCrypto-PKCS#1 v1.5', + 'sig': sig}. The signing process will use the private key rsakey_dict['keyval']['private'] and 'data' to generate the signature. @@ -315,10 +334,11 @@ def create_signature(rsakey_dict, data): 'rsakey_dict' object. - evpy.signature.sign() called to perform the actual signing. + PyCrypto's 'Crypto.Signature.PKCS1_v1_5' called to perform the actual + signing. - A signature dictionary conformat to tuf.format.SIGNATURE_SCHEMA. + A signature dictionary conformat to 'tuf.format.SIGNATURE_SCHEMA'. """ @@ -330,17 +350,26 @@ def create_signature(rsakey_dict, data): tuf.formats.RSAKEY_SCHEMA.check_match(rsakey_dict) # Signing the 'data' object requires a private key. - # The 'evp' (i.e., evpy) signing method is the only method - # currently supported. - signature = {} + # The 'PyCrypto-PKCS#1 v1.5' (i.e., PyCrypto module) signing method is the + # only method currently supported. + signature = {} private_key = rsakey_dict['keyval']['private'] keyid = rsakey_dict['keyid'] - method = 'evp' + method = 'PyCrypto-PKCS#1 v1.5' + sig = None if private_key: - sig = evpy.signature.sign(data, key=private_key) + # Take + try: + rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key) + sha256_object = Crypto.Hash.SHA256.new(data) + pkcs1_signer = Crypto.Signature.PKCS1_v1_5.new(rsa_key_object) + sig = pkcs1_signer.sign(sha256_object) + except (ValueError, IndexError, TypeError), e: + message = 'An RSA signature could not be generated.' + raise tuf.CryptoError(message) else: - raise TypeError('The required private key is not defined for rsakey_dict.') + raise TypeError('The required private key is not defined for "rsakey_dict".') # Build the signature dictionary to be returned. # The hexadecimal representation of 'sig' is stored in the signature. @@ -379,7 +408,7 @@ def verify_signature(rsakey_dict, signature, data): The signature dictionary produced by tuf.rsa_key.create_signature(). 'signature' has the form: {'keyid': keyid, 'method': 'method', 'sig': sig}. Conformant to - tuf.formats.SIGNATURE_SCHEMA. + 'tuf.formats.SIGNATURE_SCHEMA'. data: Data object used by tuf.rsa_key.create_signature() to generate @@ -395,10 +424,10 @@ def verify_signature(rsakey_dict, signature, data): 'signature' must conform to tuf.formats.SIGNATURE_SCHEMA. - evpy.signature_verify() called to do the actual verification. + Crypto.Signature.PKCS1_v1_5.verify() called to do the actual verification. - Boolean. + Boolean. True if the signature is valid, False otherwise. """ @@ -416,12 +445,26 @@ def verify_signature(rsakey_dict, signature, data): # (i.e., rsakey_dict['keyval']['public']), verify whether 'signature' # was produced by rsakey_dict's corresponding private key # rsakey_dict['keyval']['private']. Before returning the Boolean result, - # ensure 'evp' was used as the signing method. - + # ensure 'PyCrypto-PKCS#1 v1.5' was used as the signing method. method = signature['method'] sig = signature['sig'] public_key = rsakey_dict['keyval']['public'] + valid_signature = False - if method != 'evp': + if method == 'PyCrypto-PKCS#1 v1.5': + try: + rsa_key_object = Crypto.PublicKey.RSA.importKey(public_key) + pkcs1_verifier = Crypto.Signature.PKCS1_v1_5.new(rsa_key_object) + sha256_object = Crypto.Hash.SHA256.new(data) + + # The metadata stores signatures in hex. Unhexlify and verify the + # signature. + signature = binascii.unhexlify(sig) + valid_signature = pkcs1_verifier.verify(sha256_object, signature) + except (ValueError, IndexError, TypeError), e: + message = 'The RSA signature could not be verified.' + raise tuf.CryptoError(message) + else: raise tuf.UnknownMethodError(method) - return evpy.signature.verify(data, binascii.unhexlify(sig), key=public_key) + + return valid_signature From 71d4db8a96dac45b82b5d54be7110baf071687df Mon Sep 17 00:00:00 2001 From: zanefisher Date: Mon, 26 Aug 2013 14:18:20 -0400 Subject: [PATCH 041/119] Aggregate tests now passes. Moved various unit test code that ran on import into setup and teardown functions that are called at the appropriate time when run with other tests. --- tuf/tests/repository_setup.py | 18 +++++--- tuf/tests/test_signercli.py | 9 ++-- tuf/tests/test_signerlib.py | 7 +++- tuf/tests/test_updater.py | 78 +++++++++++++++++------------------ 4 files changed, 62 insertions(+), 50 deletions(-) diff --git a/tuf/tests/repository_setup.py b/tuf/tests/repository_setup.py index f355deb5..786889c9 100755 --- a/tuf/tests/repository_setup.py +++ b/tuf/tests/repository_setup.py @@ -31,17 +31,21 @@ import tuf.tests.unittest_toolbox as unittest_toolbox -# Populating 'rsa_keystore' and 'rsa_passwords' dictionaries. -# We will need them in creating keystore directory. -unittest_toolbox.Modified_TestCase.bind_keys_to_roles() - # Role:keyids dictionary. role_keyids = {} -for role in unittest_toolbox.Modified_TestCase.semi_roledict.keys(): - role_keyids[role] = unittest_toolbox.Modified_TestCase.semi_roledict[role]['keyids'] +def _init_role_keyids(): + # Populating 'rsa_keystore' and 'rsa_passwords' dictionaries. + # We will need them in creating keystore directory. + unittest_toolbox.Modified_TestCase.bind_keys_to_roles() + + global role_keyids + + for role in unittest_toolbox.Modified_TestCase.semi_roledict.keys(): + role_keyids[role] = unittest_toolbox.Modified_TestCase.semi_roledict[role]['keyids'] + @@ -54,6 +58,7 @@ def _create_keystore(keystore_directory): _rsa_keystore = unittest_toolbox.Modified_TestCase.rsa_keystore _rsa_passwords = unittest_toolbox.Modified_TestCase.rsa_passwords if not _rsa_keystore or not _rsa_passwords: + import pdb; pdb.set_trace() msg = 'Populate \'rsa_keystore\' and \'rsa_passwords\''+\ ' before invoking this method.' sys.exit(msg) @@ -275,6 +280,7 @@ def create_repositories(): """ + _init_role_keyids() # Make a temporary general repository directory. repository_dir = tempfile.mkdtemp() diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index 78fd467e..dd756d40 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -56,9 +56,6 @@ class guarantees the order of unit tests. So that, 'test_something_A' logger = logging.getLogger('tuf.test_signercli') -# Populating 'rsa_keystore' and 'rsa_passwords' dictionaries. -# We will need them when creating keystore directories. -unittest_toolbox.Modified_TestCase.bind_keys_to_roles() class TestSignercli(unittest_toolbox.Modified_TestCase): @@ -1556,6 +1553,12 @@ def _mock_get_keyids(junk): signercli._get_metadata_directory = original_get_metadata_directory +def setUpModule(): + # Populating 'rsa_keystore' and 'rsa_passwords' dictionaries. + # We will need them when creating keystore directories. + unittest_toolbox.Modified_TestCase.bind_keys_to_roles() + + def tearDownModule(): unittest_toolbox.Modified_TestCase.clear_toolbox() diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index 9bb8d759..4610f4b2 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -68,8 +68,6 @@ # 'unittest_toolbox.Modified_TestCase' is too long, I'll set it to 'unit_tbox'. unit_tbox = tuf.tests.unittest_toolbox.Modified_TestCase -# Generate rsa keys and roles dictionary dictionaries. -unit_tbox.bind_keys_to_roles() class TestSignerlib(unit_tbox): @@ -972,6 +970,11 @@ def _get_signed_role_info(self, role, directory=None): filename) return signed_meta, role_info + +def setUpModule(): + # Generate rsa keys and roles dictionary dictionaries. + unit_tbox.bind_keys_to_roles() + def tearDownModule(): unit_tbox.clear_toolbox() tuf.repo.keystore.clear_keystore() diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index d8d0ad2e..b4db02b0 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -56,10 +56,6 @@ class guarantees the order of unit tests. So that, 'test_something_A' logger = logging.getLogger('tuf.test_updater') -# References to roledb and keydb dictionaries (improve readability). -roledb = tuf.roledb -keydb = tuf.keydb - class TestUpdater_init_(unittest_toolbox.Modified_TestCase): @@ -105,39 +101,43 @@ def test__init__exceptions(self): os.remove(role_filepath) updater.Updater('Repo_Name', self.mirrors) - # Remove all created repositories. + # Remove all created repositories and roles. setup.remove_all_repositories(repositories['main_repository']) + tuf.roledb.clear_roledb() class TestUpdater(unittest_toolbox.Modified_TestCase): - # Create repositories. 'repositories' is a tuple that looks like this: - # (repository_dir, client_repository_dir, server_repository_dir), see - # repository_setup.py odule. - repositories = setup.create_repositories() - # Save references to repository directories and metadata. - # Server side references. - server_repo_dir = repositories['server_repository'] - server_meta_dir = os.path.join(server_repo_dir, 'metadata') - root_filepath = os.path.join(server_meta_dir, 'root.txt') - timestamp_filepath = os.path.join(server_meta_dir, 'timestamp.txt') - targets_filepath = os.path.join(server_meta_dir, 'targets.txt') - release_filepath = os.path.join(server_meta_dir, 'release.txt') + @classmethod + def setUpClass(cls): + # Create repositories. 'repositories' is a tuple that looks like this: + # (repository_dir, client_repository_dir, server_repository_dir), see + # repository_setup.py odule. + cls.repositories = setup.create_repositories() - # References to delegated metadata paths and directories. - delegated_dir1 = os.path.join(server_meta_dir, 'targets') - delegated_filepath1 = os.path.join(delegated_dir1, 'delegated_role1.txt') - delegated_dir2 = os.path.join(delegated_dir1, 'delegated_role1') - delegated_filepath2 = os.path.join(delegated_dir2, 'delegated_role2.txt') - targets_dir = os.path.join(server_repo_dir, 'targets') + # Save references to repository directories and metadata. + # Server side references. + cls.server_repo_dir = cls.repositories['server_repository'] + cls.server_meta_dir = os.path.join(cls.server_repo_dir, 'metadata') + cls.root_filepath = os.path.join(cls.server_meta_dir, 'root.txt') + cls.timestamp_filepath = os.path.join(cls.server_meta_dir, 'timestamp.txt') + cls.targets_filepath = os.path.join(cls.server_meta_dir, 'targets.txt') + cls.release_filepath = os.path.join(cls.server_meta_dir, 'release.txt') - # Client side references. - client_repo_dir = repositories['client_repository'] - client_meta_dir = os.path.join(client_repo_dir, 'metadata') - client_current_dir = os.path.join(client_meta_dir, 'current') - client_previous_dir = os.path.join(client_meta_dir, 'previous') + # References to delegated metadata paths and directories. + cls.delegated_dir1 = os.path.join(cls.server_meta_dir, 'targets') + cls.delegated_filepath1 = os.path.join(cls.delegated_dir1, 'delegated_role1.txt') + cls.delegated_dir2 = os.path.join(cls.delegated_dir1, 'delegated_role1') + cls.delegated_filepath2 = os.path.join(cls.delegated_dir2, 'delegated_role2.txt') + cls.targets_dir = os.path.join(cls.server_repo_dir, 'targets') + + # Client side references. + cls.client_repo_dir = cls.repositories['client_repository'] + cls.client_meta_dir = os.path.join(cls.client_repo_dir, 'metadata') + cls.client_current_dir = os.path.join(cls.client_meta_dir, 'current') + cls.client_previous_dir = os.path.join(cls.client_meta_dir, 'previous') @@ -176,8 +176,8 @@ def tearDown(self): unittest_toolbox.Modified_TestCase.tearDown(self) # Clear roledb and keydb dictionaries. - roledb.clear_roledb() - keydb.clear_keydb() + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() @@ -379,14 +379,14 @@ def test_1__rebuild_key_and_role_db(self): # are populated. 'top_level_role_info' is a unittest_toolbox's dict # that contains top level role information it corresponds to a # ROLEDICT_SCHEMA where roles are keys and role information their values. - self.assertEqual(roledb._roledb_dict, self.top_level_role_info) - self.assertEqual(len(keydb._keydb_dict), 4) + self.assertEqual(tuf.roledb._roledb_dict, self.top_level_role_info) + self.assertEqual(len(tuf.keydb._keydb_dict), 4) # Verify that keydb dictionary was updated. for role in self.role_list: keyids = self.top_level_role_info[role]['keyids'] for keyid in keyids: - self.assertTrue(keyid in keydb._keydb_dict) + self.assertTrue(keyid in tuf.keydb._keydb_dict) @@ -408,22 +408,22 @@ def test_2__import_delegations(self): # Verify that there was no change in roledb and keydb dictionaries # by checking the number of elements in the dictionaries. - self.assertEqual(len(roledb._roledb_dict), 5) - self.assertEqual(len(keydb._keydb_dict), 5) + self.assertEqual(len(tuf.roledb._roledb_dict), 5) + self.assertEqual(len(tuf.keydb._keydb_dict), 5) # Test: normal case, first level delegation. self.Repository._import_delegations('targets/delegated_role1') - self.assertEqual(len(roledb._roledb_dict), 6) - self.assertEqual(len(keydb._keydb_dict), 6) + self.assertEqual(len(tuf.roledb._roledb_dict), 6) + self.assertEqual(len(tuf.keydb._keydb_dict), 6) # Verify that roledb dictionary was updated. - self.assertTrue('targets/delegated_role1' in roledb._roledb_dict) + self.assertTrue('targets/delegated_role1' in tuf.roledb._roledb_dict) # Verify that keydb dictionary was updated. keyids = self.semi_roledict['targets/delegated_role1']['keyids'] for keyid in keyids: - self.assertTrue(keyid in keydb._keydb_dict) + self.assertTrue(keyid in tuf.keydb._keydb_dict) From 4654f9b4a25ac1e249fc9236c7827a31d3929736 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Mon, 26 Aug 2013 14:25:25 -0400 Subject: [PATCH 042/119] Removed stray debug code. --- tuf/tests/repository_setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tuf/tests/repository_setup.py b/tuf/tests/repository_setup.py index 786889c9..759704d1 100755 --- a/tuf/tests/repository_setup.py +++ b/tuf/tests/repository_setup.py @@ -58,7 +58,6 @@ def _create_keystore(keystore_directory): _rsa_keystore = unittest_toolbox.Modified_TestCase.rsa_keystore _rsa_passwords = unittest_toolbox.Modified_TestCase.rsa_passwords if not _rsa_keystore or not _rsa_passwords: - import pdb; pdb.set_trace() msg = 'Populate \'rsa_keystore\' and \'rsa_passwords\''+\ ' before invoking this method.' sys.exit(msg) From 4a85d4bba8a790ee44372a68613d7a05973b5b46 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Mon, 26 Aug 2013 15:35:28 -0400 Subject: [PATCH 043/119] Make aggregate tests randomization optional. --- tuf/tests/aggregate_tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index 42a1db18..a8a56fda 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -19,9 +19,11 @@ Run all the unit tests from every .py file beginning with "test_" in 'tuf/tests'. + Use --random to run the tests in random order. """ +import sys import unittest import glob import random @@ -31,7 +33,8 @@ # Remove '.py' from each filename. tests_list = [test[:-3] for test in tests_list] -random.shuffle(tests_list) +if '--random' in sys.argv: + random.shuffle(tests_list) suite = unittest.TestLoader().loadTestsFromNames(tests_list) From fd39de6665ba8b46dadf553c6f988a4ae32ff172 Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 27 Aug 2013 15:13:55 -0400 Subject: [PATCH 044/119] Import chain-of-trust function from @vladdd. --- tuf/client/basic_client.py | 2 +- tuf/examples/example_client.py | 51 ++++++++--- tuf/examples/example_integration.py | 84 ----------------- tuf/tmp/client/example_integration.py | 90 ------------------- tuf/tmp/client/metadata/current/release.txt | 58 ------------ tuf/tmp/client/metadata/current/root.txt | 70 --------------- tuf/tmp/client/metadata/current/targets.txt | 45 ---------- .../metadata/current/targets/packages.txt | 48 ---------- .../metadata/current/targets/packages/A.txt | 38 -------- .../current/targets/packages/A/Alice.txt | 28 ------ tuf/tmp/client/metadata/current/timestamp.txt | 22 ----- tuf/tmp/client/metadata/previous/release.txt | 28 ------ tuf/tmp/client/metadata/previous/root.txt | 70 --------------- tuf/tmp/client/metadata/previous/targets.txt | 22 ----- .../client/metadata/previous/timestamp.txt | 22 ----- tuf/tmp/client/output.txt | 23 ----- tuf/tmp/client/server-requests.txt | 10 --- .../packages/A/Alice/alice-v2.0.tar.gz | 1 - ...7a998acff6667678f60a72de9491f6df404a22.key | 1 - ...6a959623950923470897b6230ae859bbffad6b.key | 1 - ...79d8e59d7bce66aa21bf1de9b5069c30369dc9.key | 1 - ...95b05fcc77a2bf145ca3d7302ce7868cbaed7b.key | 1 - tuf/tmp/project/helloworld.py | 1 - tuf/tmp/repository/config.cfg | 23 ----- tuf/tmp/repository/metadata/release.txt | 58 ------------ tuf/tmp/repository/metadata/root.txt | 70 --------------- tuf/tmp/repository/metadata/targets.txt | 45 ---------- .../repository/metadata/targets/packages.txt | 48 ---------- .../metadata/targets/packages/A.txt | 38 -------- .../metadata/targets/packages/A/Alice.txt | 28 ------ .../metadata/targets/packages/B.txt | 38 -------- .../metadata/targets/packages/B/Bob.txt | 28 ------ tuf/tmp/repository/metadata/timestamp.txt | 22 ----- tuf/tmp/repository/targets/helloworld.py | 1 - .../packages/A/Alice/alice-v1.0.tar.gz | 1 - .../packages/A/Alice/alice-v2.0.tar.gz | 1 - .../targets/packages/B/Bob/bob-v1.0.tar.gz | 1 - .../targets/packages/B/Bob/bob-v2.0.tar.gz | 1 - 38 files changed, 39 insertions(+), 1081 deletions(-) delete mode 100755 tuf/examples/example_integration.py delete mode 100755 tuf/tmp/client/example_integration.py delete mode 100644 tuf/tmp/client/metadata/current/release.txt delete mode 100644 tuf/tmp/client/metadata/current/root.txt delete mode 100644 tuf/tmp/client/metadata/current/targets.txt delete mode 100644 tuf/tmp/client/metadata/current/targets/packages.txt delete mode 100644 tuf/tmp/client/metadata/current/targets/packages/A.txt delete mode 100644 tuf/tmp/client/metadata/current/targets/packages/A/Alice.txt delete mode 100644 tuf/tmp/client/metadata/current/timestamp.txt delete mode 100644 tuf/tmp/client/metadata/previous/release.txt delete mode 100644 tuf/tmp/client/metadata/previous/root.txt delete mode 100644 tuf/tmp/client/metadata/previous/targets.txt delete mode 100644 tuf/tmp/client/metadata/previous/timestamp.txt delete mode 100644 tuf/tmp/client/output.txt delete mode 100644 tuf/tmp/client/server-requests.txt delete mode 100644 tuf/tmp/client/targets/packages/A/Alice/alice-v2.0.tar.gz delete mode 100644 tuf/tmp/keystore/76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22.key delete mode 100644 tuf/tmp/keystore/845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b.key delete mode 100644 tuf/tmp/keystore/9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9.key delete mode 100644 tuf/tmp/keystore/c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b.key delete mode 100644 tuf/tmp/project/helloworld.py delete mode 100644 tuf/tmp/repository/config.cfg delete mode 100644 tuf/tmp/repository/metadata/release.txt delete mode 100644 tuf/tmp/repository/metadata/root.txt delete mode 100644 tuf/tmp/repository/metadata/targets.txt delete mode 100644 tuf/tmp/repository/metadata/targets/packages.txt delete mode 100644 tuf/tmp/repository/metadata/targets/packages/A.txt delete mode 100644 tuf/tmp/repository/metadata/targets/packages/A/Alice.txt delete mode 100644 tuf/tmp/repository/metadata/targets/packages/B.txt delete mode 100644 tuf/tmp/repository/metadata/targets/packages/B/Bob.txt delete mode 100644 tuf/tmp/repository/metadata/timestamp.txt delete mode 100644 tuf/tmp/repository/targets/helloworld.py delete mode 100644 tuf/tmp/repository/targets/packages/A/Alice/alice-v1.0.tar.gz delete mode 100644 tuf/tmp/repository/targets/packages/A/Alice/alice-v2.0.tar.gz delete mode 100644 tuf/tmp/repository/targets/packages/B/Bob/bob-v1.0.tar.gz delete mode 100644 tuf/tmp/repository/targets/packages/B/Bob/bob-v2.0.tar.gz diff --git a/tuf/client/basic_client.py b/tuf/client/basic_client.py index b6a8de36..1ba36dbc 100755 --- a/tuf/client/basic_client.py +++ b/tuf/client/basic_client.py @@ -131,7 +131,7 @@ def update_client(repository_mirror): # Remove any files from the destination directory that are no longer being # tracked. - #updater.remove_obsolete_targets(destination_directory) + updater.remove_obsolete_targets(destination_directory) diff --git a/tuf/examples/example_client.py b/tuf/examples/example_client.py index faaad672..78b1103f 100755 --- a/tuf/examples/example_client.py +++ b/tuf/examples/example_client.py @@ -1,33 +1,36 @@ """ - simple_pip_integration.py + example_client.py Vladimir Diaz - August 1, 2013 + September 2012 See LICENSE for licensing information. - Example client script demonstrating custom python code one can write for a - PyPI+pip+TUF integration. + Example script demonstrating custom python code a software updater + utilizing The Update Framework may write to securely update files. + The 'basic_client.py' script can be used on the command-line to perform + an update that will download and update all available targets; writing + custom code is not required in this case. - The custom example below demonstrates updating all the targets of a - specified role (i.e., 'targets/ + The custom examples below demonstrate: + (1) updating all targets + (2) updating all the targets of a specified role + (3) updating a specific target explicitely named. """ import logging -import tuf -import tuf.log import tuf.client.updater -logger = logging.getLogger('tuf.cient.basic_client') - +# Uncomment the line below to enable printing of debugging information. +#tuf.log.set_log_level(logging.DEBUG) # Set the local repository directory containing the metadata files. tuf.conf.repository_directory = '.' @@ -51,9 +54,6 @@ # all the targets tracked, and determine which of these targets have been # updated. updater.refresh() - -# -updater.refresh_targets_metadata_chain( all_targets = updater.all_targets() updated_targets = updater.updated_targets(all_targets, destination_directory) @@ -67,3 +67,28 @@ # Remove any files from the destination directory that are no longer being # tracked. updater.remove_obsolete_targets(destination_directory) + + +""" +# Example demonstrating an update that only downloads the targets of +# a specific role (i.e., 'targets/role1') + +updater.refresh() +targets_of_role1 = updater.targets_of_role('targets/role1') +updated_targets = updater.updated_targets(targets_of_role1, destination_directory) + +for target in updated_targets: + updater.download_target(target, destination_directory) +""" + + +""" +# Example demonstrating an update that downloads a specific target. + +updater.refresh() +target = updater.target('LICENSE.txt') +updated_target = updater.updated_targets([target], destination_directory) + +for target in updated_target: + updater.download_target(target, destination_directory) +""" diff --git a/tuf/examples/example_integration.py b/tuf/examples/example_integration.py deleted file mode 100755 index ca86743e..00000000 --- a/tuf/examples/example_integration.py +++ /dev/null @@ -1,84 +0,0 @@ -""" - - example_integration.py - - - Vladimir Diaz - - - August 1, 2013 - - - See LICENSE for licensing information. - - - Example client script outlining custom python code one can write for a - PyPI+pip+TUF integration. It aims to demonstrate efficient retrieval - of a target file and a metadata chain of trust, in a secure manner. - - The custom example below demonstrates updating all the targets of a - specified role (i.e., 'targets/packages/A/Alice.txt'). - -""" - -import logging - -import tuf.client.updater - -# Uncomment the line below to enable printing of debugging information. -tuf.log.set_log_level(logging.DEBUG) - -# Set the local repository directory containing the metadata files. -tuf.conf.repository_directory = '.' - -# Set the repository mirrors. This dictionary is needed by the Updater -# class of updater.py. The client will download metadata and target -# files from any one of these mirrors. -repository_mirrors = {'mirror1': {'url_prefix': 'http://localhost:8001', - 'metadata_path': 'metadata', - 'targets_path': 'targets', - 'confined_target_dirs': ['']}} - -# Create the Upater object using the updater name 'tuf-example' -# and the repository mirrors defined above. -updater = tuf.client.updater.Updater('tuf-example', repository_mirrors) - -# Set the local destination directory to save the target files. -destination_directory = './targets' - -# Refresh the repository's top-level roles, store the target information for -# all the targets of the 'Alice' project, and determine which of these targets -# have been updated. First, refresh top-level roles... -updater.refresh() - -# The 'release.txt' file may be inspected to retreive our desired role, or -# a dictionary that links project names to project roles. -# For example: {'Alice': 'targets/packages/A/Alice'} -alice_role = 'targets/packages/A/Alice' - -# Before we can download the metadata for 'alice_role', the chain of trust -# must be built. At the moment, the client has only downloaded/updated -# the metadata for the top-level roles. -# Download: 'targets/packages.txt', 'targets/packages/A.txt', -# 'targets/packages/A/Alice.txt'. In other words, we only fetch the minimum -# required to get a list of targets that the 'Alice' project -# has signed. Calling updater.all_targets() or updater.target() causes an -# update of all the metadata on the repository, which might be inefficient -# for a repository like PyPI. -updater.refresh_targets_metadata_chain(alice_role) -targets_of_alice = updater.targets_of_role(alice_role) -updated_targets = updater.updated_targets(targets_of_alice, destination_directory) - -# The pip software updater might request multiple targets in one update -# cycle (i.e., -# $ pip install Alice -# fetches 'simple/Alice/index.html', 'alice-v0.1.tar.gz', ...) -# As a simple example here, download a single target file arbitrarily -# chosen, and save it locally. -for updated_target in updated_targets: - if updated_target['filepath'] == 'packages/A/Alice/alice-v0.1.tar.gz': - updater.download_target(updated_target, destination_directory) - -# Remove any files from the destination directory that are no longer being -# tracked. -updater.remove_obsolete_targets(destination_directory) diff --git a/tuf/tmp/client/example_integration.py b/tuf/tmp/client/example_integration.py deleted file mode 100755 index 6c4bcc7e..00000000 --- a/tuf/tmp/client/example_integration.py +++ /dev/null @@ -1,90 +0,0 @@ -""" - - example_integration.py - - - Vladimir Diaz - - - August 1, 2013 - - - See LICENSE for licensing information. - - - Example client script outlining custom python code one can write for a - PyPI+pip+TUF integration. It aims to demonstrate efficient retrieval - of a target file and a metadata chain of trust, in a secure manner. - - The custom example below demonstrates updating all the targets of a - specified role (i.e., 'targets/packages/A/Alice.txt'). - -""" - -import logging - -import tuf.client.updater - -# Uncomment the line below to enable printing of debugging information. -tuf.log.set_log_level(logging.DEBUG) - -# Set the local repository directory containing the metadata files. -tuf.conf.repository_directory = '.' - -# Set the repository mirrors. This dictionary is needed by the Updater -# class of updater.py. The client will download metadata and target -# files from any one of these mirrors. -repository_mirrors = {'mirror1': {'url_prefix': 'http://localhost:8001', - 'metadata_path': 'metadata', - 'targets_path': 'targets', - 'confined_target_dirs': ['']}} - -# Create the Upater object using the updater name 'tuf-example' -# and the repository mirrors defined above. -updater = tuf.client.updater.Updater('tuf-example', repository_mirrors) - -# Set the local destination directory to save the target files. -destination_directory = './targets' - -# The single target file that the client wishes to update/install. -alice_package = 'packages/A/Alice/alice-v2.0.tar.gz' -message = 'Example that updates '+repr(alice_package)+' and downloads the '+\ - 'mimimum metadata to set the required chain of trust.\n' -print message - -# Refresh the repository's top-level roles, store the target information for -# all the targets of the 'Alice' project, and determine which of these targets -# have been updated. First, refresh top-level roles... -updater.refresh() - -# The 'release.txt' file may be inspected to retreive our desired role, or -# a dictionary that links project names to project roles. -# For example: {'Alice': 'targets/packages/A/Alice'} -alice_role = 'targets/packages/A/Alice' - -# Before we can download the metadata for 'alice_role', the chain of trust -# must be built. At the moment, the client has only downloaded/updated -# the metadata for the top-level roles. -# Download: 'targets/packages.txt', 'targets/packages/A.txt', -# 'targets/packages/A/Alice.txt'. In other words, we only fetch the minimum -# required to get a list of targets that the 'Alice' project -# has signed. Calling updater.all_targets() or updater.target() causes an -# update of all the metadata on the repository, which might be inefficient -# for a repository like PyPI. -updater.refresh_targets_metadata_chain(alice_role) -targets_of_alice = updater.targets_of_role(alice_role) -updated_targets = updater.updated_targets(targets_of_alice, destination_directory) - -# The pip software updater might request multiple targets in one update -# cycle (i.e., -# $ pip install Alice -# fetches 'simple/Alice/index.html', 'alice-v0.1.tar.gz', ...) -# As a simple example here, download a single target file arbitrarily -# chosen, and save it locally. -for updated_target in updated_targets: - if updated_target['filepath'] == alice_package: - updater.download_target(updated_target, destination_directory) - -# Remove any files from the destination directory that are no longer being -# tracked. -updater.remove_obsolete_targets(destination_directory) diff --git a/tuf/tmp/client/metadata/current/release.txt b/tuf/tmp/client/metadata/current/release.txt deleted file mode 100644 index 14623ee0..00000000 --- a/tuf/tmp/client/metadata/current/release.txt +++ /dev/null @@ -1,58 +0,0 @@ -{ - "signatures": [ - { - "keyid": "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9", - "method": "evp", - "sig": "adb041550a26327056b17409c59c2294930bcee1dc88008a9b458d828da673e2da4ae3c40257dfa51a25cd2cd23189fd1753546fd441879f275e515b433919e0403478bc2a7b7d9e455283f742fe5d059097be55eb2d705123194f31b13cb7d2a96421e5b7fb09df2f0a5d4245676b71c4630fd20ee29f962b3d327eb3362cd5e2f104b3a036d9c305817df955e19c49f3878cf3e65915c8a542adfd057f62522c1eca75cba513c81adb14994152934ecb4de1fb707d1aca4cc0f2b5ecb09e6645cb6f27f0769c8aeeff7f5728a910af9d310737c17e6b1cd611b07d70ee80de1457b13f54102ec5c58fdcf75470fe4db41c18f93f18a92f9929b8a9693e6e96b6231fc63705f47e05e079259e1eff17234060870685868da555d0bb05546f26d77ff7f091c3bd1a3e77633f2f5505597f8126a2130cacaee9a119c2915b48a0b08ff2152495462119b6a4ca05d302629bb7f7da60346a8cdd12f2820a00af6d1f3debffaf5052c2d31afa9c3fce3f82dbd139fcd0cd5062bede2c77c5e19407" - } - ], - "signed": { - "_type": "Release", - "expires": "2014-08-01 16:19:08 UTC", - "meta": { - "root.txt": { - "hashes": { - "sha256": "2e496d43eb877fc725dd3bd616da0c1e018057982d3c8acf8906a4104680feb1" - }, - "length": 4793 - }, - "targets.txt": { - "hashes": { - "sha256": "c5cbeeaaa617fa9ecf282ed4ed3051ecba2d8f9535f148e14103c6d6ed6bfd39" - }, - "length": 2260 - }, - "targets/packages.txt": { - "hashes": { - "sha256": "324aff11e6488e3619f8a291dc94f82faf60ab00ce67443ba32997bd0d1ad0cb" - }, - "length": 2325 - }, - "targets/packages/A.txt": { - "hashes": { - "sha256": "0af576b49df40cc310ba314a82dc264202ec74d4238eb526d85230aacf9d2282" - }, - "length": 2123 - }, - "targets/packages/A/Alice.txt": { - "hashes": { - "sha256": "49d0adb568d9323161f987087894df88cc0eb45ad2e4b7972b017915899226af" - }, - "length": 1377 - }, - "targets/packages/B.txt": { - "hashes": { - "sha256": "e3618668fe88e9fa99cb305e24d8c78ed3083270bb3de8bbc42dbf4234f2e894" - }, - "length": 2119 - }, - "targets/packages/B/Bob.txt": { - "hashes": { - "sha256": "3fb4ac73a78e66408b6192e28aa3b02e8180793a64fd99e49842b4a4ad45b7e9" - }, - "length": 1369 - } - }, - "version": 2 - } -} diff --git a/tuf/tmp/client/metadata/current/root.txt b/tuf/tmp/client/metadata/current/root.txt deleted file mode 100644 index b7d8e066..00000000 --- a/tuf/tmp/client/metadata/current/root.txt +++ /dev/null @@ -1,70 +0,0 @@ -{ - "signatures": [ - { - "keyid": "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b", - "method": "evp", - "sig": "83448b51098091a562d9074dc14e02af5a6f18fcc18ff0dd0d709d0d1a597fbe9c96d2a077acffdc85be0357f76c46b1d2cbb579ad376458a88b2e00330b4028361c337593c96b7c2eec10dc337c7652d9e83d7c2dca7e59230199d2e6e5d3f0f1a38f09d305b47954f552ecb45df5c247ffa60b3b15198bc17a17d9198e688289ec70e3043cad1c1ac9405ad81d1af5135cf961f1a45d60f08d00c4e1713722711bee4864655a495f6e105020702ffea947fe3288358dffa46fa8d9c4c47fb7b3fd8a47b2371a0ffa78bce885b2ddb36b75ca6f6fea807e236593b882a7b1fc6c0fe43eb4b6709b060e08fb3e6f5a56fea6a5524130d01b461f1d6c2e3b1c2ef8784190245568062ec888af1fee0740b81ca0c99b775396b421f507581257277ba1e8609528ffef0ef5c9c205b63874e2c37b5dabb738cf5597e0c39010b8041a87b00030f69217e41e03376d1899c25d0d66d9e936b9308709eecc9862273a91e42301f6254f501bc10806a8aced547667678c39790598579dcb79b064b074" - } - ], - "signed": { - "_type": "Root", - "expires": "2014-07-31 15:21:53 UTC", - "keys": { - "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAxyi2M6UtfeITuiu4kbWZ\nOU5CRScab9Z8eWy9Weff/IiSYD47uRhnseT84ls50yCDzaUMlrCSG4Kd+AdAfcVl\nmsFCnRC5HHacFF9EEPSDud9L+4LTgqM1fP4vWfv8ZZ1sXI/5Npeo/D3bcC9NmcfR\nBMCzaQl1NEoYK+QVMcqVVImoWeLEtmU52u96Y4FZv06FodhpHPVvKUmJFnXe/ikM\nLe5sTCP0QKLtv+q1bfZ84KBkE50QfDpTF1bqiDeuCtb/3yNb91gkpPspuWjn9icK\nFCc/D5FpYSkVnWCoPzYsuVcQzIPFV2Zr8L1gvSvPOPY7SJulk+jri2T1kZw/rbDe\nA9T/I1yG36/8IXv9JYA8yiMR4cblFqi00gj1Q3NrelBdYc27t8GDNP2D2H4wMw3P\nnTmmfJQ5WpR5ZbjVHy/OU5YCpeJ77sdBITaSVn5IWLYumko+vJohOoordMBfV/jZ\nTVVeaz0OCp9sPs46xKVN+m109/NNejS90gLIKPKDSfPrAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAuvReIoGpKvK0rwmkgccg\nmv8sTtEBolT02THaxO+6+gSPO/509msi/M3UNHsGNda1gccdwRMTsR/C9lCKC9Mp\nZ+JcOUl70SkL1lf4QlttWEDyDgDg1M7RPfcMZnq+eo/BPqGOxmbeNxKS2tqNfpQQ\nOqvSb7MMeII/mitunWo+UbJE/No9dimueFAjgHwQfH3rJCMnjfL5OT15UipHICCV\n/x6Rypc47kyOIfAvGMBnPT+sSapu+tboGPQf4kYwDpQVBPrJuGLYbgFvLAP/JKoi\ndHwXKeOAYP6FSioKcXbOspVd7a6FCUHQtEX8g158WOb28Ggo3eeXJx4Yb6ZW4WhD\no1jHNyELE2n1h42FETUNiESF3WDbfGZX8XINy+PXmVWGsY0YTdFOcSmw5k4fZ2Dp\nD57R7GQanMBBkpB/J5dVYvdP/NLTHeueBRFDJAFM+HRozbTpi+pFTcKV0dITLh43\nQrvSE7ZK2ktW1Zo5aJuhVClxga3teM6N4Hm3wEv1mfqBAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0BLhIsIORHPXuTU1CPSj\n34jQVNs3jXfpEfDIoYQcLPrNoz5icGWlg3H7YFtGSGOIOzJbyMlQR3iyGu7IYSW7\nuyJRX8OJ6rkbLcAg3z4QXZf6Y8isQYQGBiPb/j/LKGpGs6GA0wDqVfcCUJGtz04k\n5P4oPmJZjiQO2uMyQKYkJDWXllAgkY/SkuOUHyk/knE8EHIoNCwqCAVVnKc/gg/O\nL2I6mwkyes6eXQDRdwRK0z1P72ebzAgKdshhU8Jx4S1W3BTdX4CZ0TqBKxiNkhTi\nIWkZHM5hijX7NbCUNTIL4MUSDGga/quqs3kSMCM3lOd37MLiTDXHcCZoF09w6cgy\njsZyZZR7PBgpQjQ4EjgKstrAForp7ph7dF/BAP4Fz3uf9JBhdJ3LaIT/0et0BA/J\n52TxMT84ngzS+yWobqdrOK9xVaNOZTS0j3ScWpBKRCDR1E+llUJlkjphdD6Og8K2\ntFVKwCTYm3qfwHd2ulllVzbOOntTnq/ppcQjtTO2yYSVAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": { - "release": { - "keyids": [ - "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9" - ], - "threshold": 1 - }, - "root": { - "keyids": [ - "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b" - ], - "threshold": 1 - }, - "targets": { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "threshold": 1 - }, - "timestamp": { - "keyids": [ - "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22" - ], - "threshold": 1 - } - }, - "version": 1 - } -} diff --git a/tuf/tmp/client/metadata/current/targets.txt b/tuf/tmp/client/metadata/current/targets.txt deleted file mode 100644 index 7bdb5ae2..00000000 --- a/tuf/tmp/client/metadata/current/targets.txt +++ /dev/null @@ -1,45 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "89a80bef74459020b690187315ff0b3ecae376c68a5305ca08d29151fc3f5047385da8c70d1965d9c47bda9dce4ab8a2c83a8c04792d097491555bc884a8a833e644c0d85b27338154b861c7f829221f3e0d3170b3414a7922ff37cbb5223a7dafd95e8eb5bc4b2bcdcbcb72533751ebe4a6adb441d4389d0f55ad9a68beac98442aac953c0a6e531f45f78891ad15c72e54dda57e673d60d9936278d60f89ababcbc811eda9ba770b1a5cb222ff4e15f18da323b01e49e03ffbdfea207047d2543baa458978fc14644716ce92b9d112e732538d14002d5db5aa7143ee6eddf463b6e96f9504f87b393e8c340bfb5f425c05af454bc67711daabd412e96a295563b9171d7623f08a87a449f8e594e66e68e49f302e639ad523ce1baebe458afe07136030b949c5ba8114f975bcf1462486cc115a50a27263270cb63c0bcbe9e4ebc8171d9453e279086309668ac2d538b665c64888b43806a5bb97207fd91a02f4634c723da81dff84225eec4439c0acdb893410e34fd62343108d7b7055b59e" - } - ], - "signed": { - "_type": "Targets", - "delegations": { - "keys": { - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": [ - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages", - "paths": [ - "packages/" - ], - "threshold": 1 - } - ] - }, - "expires": "2013-10-31 22:49:03 UTC", - "targets": { - "helloworld.py": { - "hashes": { - "sha256": "14d9f7904b16af5b3cd64285eb349bdce11dd3688d6e330ab7da87eb37512941" - }, - "length": 18 - } - }, - "version": 2 - } -} diff --git a/tuf/tmp/client/metadata/current/targets/packages.txt b/tuf/tmp/client/metadata/current/targets/packages.txt deleted file mode 100644 index 65a7e369..00000000 --- a/tuf/tmp/client/metadata/current/targets/packages.txt +++ /dev/null @@ -1,48 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "028e6e6c93e3973bdbc295a5e96fb9d67926dc5a67b3127a77d3dbdd55e1b87bf82183e64f3e0ce133d0cf75b64a0c80f432c91c95bcd583af073473e991c7bc2a12cce0290b17232d82f010268eeb1192a242a14aa992b9b6036fff5dcf3fe5a2fcc0d15d9ca6aee54ca2a053779889968eac11c160fed1056b2ad0092f69deac9d286657b64a92f0b9182bdfee32930117b83baf729bd494b259d60a3ebd54c0a154ba87d710f9f8ab5ef6cfd563dffe346ef6bcb6551c5323f5c68839089a3ea65926c0fa159c43272d1323fd521b403dbe88d7213955e3c121328eb816db3521e059fc37b2e88741f517747344ce9b5520693061848b627077db692ab44afc1cab484270aa826339b1181862b461433b79d066cfb289fdd5f91b4e193bbdf5053d33e93b615e40ade38d7c74d8d3da8ae2df4fbaf4792a867cf08ba182666f465d0a0723058eebbec94b1c9e9f46560e05a45d58fdf98e5b5362077d63f00b0c8cf5ca00f60ef3ef5b2559a1c129eca3422a228aac3fa6882b368bfc233c" - } - ], - "signed": { - "_type": "Targets", - "delegations": { - "keys": { - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": [ - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages/A", - "paths": [ - "packages/A/" - ], - "threshold": 1 - }, - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages/B", - "paths": [ - "packages/B/" - ], - "threshold": 1 - } - ] - }, - "expires": "2014-08-01 15:37:45 UTC", - "targets": {}, - "version": 3 - } -} diff --git a/tuf/tmp/client/metadata/current/targets/packages/A.txt b/tuf/tmp/client/metadata/current/targets/packages/A.txt deleted file mode 100644 index 447153ce..00000000 --- a/tuf/tmp/client/metadata/current/targets/packages/A.txt +++ /dev/null @@ -1,38 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "69be32d77d781bb48f0518dafade5fb0a166d9ad82340a3a20f7630165d69d8ab0f8e753fd490c4c8727968539a4285db94bf73317a83672a177576ebb8091ec8ed34334893a683dad990ddd2ef7f0b1c034ed581b11ff12a30d78e31bb3c16918464a91128b3151eafab427b316134e17106ebaaee9ab78d39673beb4d08fd5aeac506e485e9e71903886ec1adb9a69dd1855b98aec2e7d48e361ec5b92ea728d4d8ba3bb16e84dd36cbef88bfbb8ecb39d9e1b20544a678062af312447b302803592da00f68846d68f6c05dbb5e7419dca5b07e8d43aa5a9b1a3a0e8386c815c665160062c7b4760761d05c683ddf18e398816120cc7860574ac98b9fd3ef74018210b454b765bb4dfe45163b348f44fc1c804ae69fc1a13d7e71a03d0af724838ab959da6828e990e604cb563a00724e6b4deb7e8ca13275bbfc89185d1cd71d3fb2c11692b5785801632bd1e54d60d73b5817dd654217cdc0850df5527f04ae8ba053e9a040a7bd0de740629640354895bf6399ca9672f432d507b4ced82" - } - ], - "signed": { - "_type": "Targets", - "delegations": { - "keys": { - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": [ - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages/A/Alice", - "paths": [ - "packages/A/Alice/" - ], - "threshold": 1 - } - ] - }, - "expires": "2014-08-01 15:45:01 UTC", - "targets": {}, - "version": 2 - } -} diff --git a/tuf/tmp/client/metadata/current/targets/packages/A/Alice.txt b/tuf/tmp/client/metadata/current/targets/packages/A/Alice.txt deleted file mode 100644 index 0b0e365f..00000000 --- a/tuf/tmp/client/metadata/current/targets/packages/A/Alice.txt +++ /dev/null @@ -1,28 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "b55953980cc35017cf6deb2681380880f14b216d261c18e45bf4ebecbda0c6cc07e49607dac81639037008f8cc328ecd7f2c5858d454861b3bfd3f2209c24067164edf0af14c5f4564c9244f7c423825c7e162df618159e3c376f1c6ed4fb56e97b3d7fd3da59724c706e6f86b1ccfae1896ed6f792b76517cb87be92fc6336f892191c2dc3f55511c15cd787157af26489f2e8fc011507dcae5f4f7b314fd1c7a97c7fc8d91559d92e8615bfde318acea99bef2c4906c92c0d6e97ce3ae27c6e7ad5a232809f05fda1f6f5241fea5dfe2b86a00a57859c3b5322ad22cd7ebb5d71c3b8014de5a866068e9eb77ff9d0bdc3599b0de18f0f6f1a3546f03989e02346dc81b36601eda373814401381bf97709a7545ef448c9d3eaf1f80fedf5a959042d700ba7ebd060c4348cac3452258823039d06871d90c5fbf22e2572abda908a1f9160856db4bcd5b152a35ef81dd977f13aabec7d4fa05499a5969e03841e088dd29239795d4a2927616e210200ce3dfa82a4c250775c33c18035e5aa62a" - } - ], - "signed": { - "_type": "Targets", - "expires": "2014-08-01 15:46:53 UTC", - "targets": { - "packages/A/Alice/alice-v1.0.tar.gz": { - "hashes": { - "sha256": "21ecfb59295a055b39003e6b9a63b4aa7b3724458a7c663b00ee6cf69fc09e68" - }, - "length": 15 - }, - "packages/A/Alice/alice-v2.0.tar.gz": { - "hashes": { - "sha256": "21ecfb59295a055b39003e6b9a63b4aa7b3724458a7c663b00ee6cf69fc09e68" - }, - "length": 15 - } - }, - "version": 1 - } -} diff --git a/tuf/tmp/client/metadata/current/timestamp.txt b/tuf/tmp/client/metadata/current/timestamp.txt deleted file mode 100644 index 8cb6f19d..00000000 --- a/tuf/tmp/client/metadata/current/timestamp.txt +++ /dev/null @@ -1,22 +0,0 @@ -{ - "signatures": [ - { - "keyid": "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22", - "method": "evp", - "sig": "9d09a61312b76ab8ec6f16b1bd9800769c306898c92df7a05bdd3a7bfe30747b187dab13427853957c3b21c1dc93519c290564f7703f4b07529fc64ff0e75d91569471bc112ab07d6253475489a07971384bcab10150c69d10c91960b75f7aa04b2c230eecc299b0e174278146bf3ab070841a1ff7ff15c7ad056eb020c29785b3ff432e2029c7bb56cdc36933204ce52e7dd32d5a7b9349b3f00d0da65a3423dadb4d74cf9abb91c29533c6d09b81811b1d3aed04988af0795ffa62fa409ada0a92a72d1f04f89caead224aa734aeb664fdb62a04b6045e8d749e015a5c40df81e275472e681722ed7afcf2556295eeb2463fbb36b055cc8581c9d8457d66c67b400627ab42d4619e8d6f881c66dda74385de451a5cbb967821255f7b211a03e34dc836b42e73d1588ac2ed5395f3e1c0ff281c84a9298d895cbf1b748ad19705a3ee26151eb08bb51df0210acca69b6a358390b223201e588d9c750f60238c6144a264f07e0631305566c1d41a07432800e226caaba2481882ffdeb177c657" - } - ], - "signed": { - "_type": "Timestamp", - "expires": "2014-08-01 16:19:39 UTC", - "meta": { - "release.txt": { - "hashes": { - "sha256": "f22f2f12fad9069b2fb569d9e9273a95b015fe2d8f3937addfed3dc1e48d86de" - }, - "length": 2152 - } - }, - "version": 2 - } -} diff --git a/tuf/tmp/client/metadata/previous/release.txt b/tuf/tmp/client/metadata/previous/release.txt deleted file mode 100644 index 5a36b4d7..00000000 --- a/tuf/tmp/client/metadata/previous/release.txt +++ /dev/null @@ -1,28 +0,0 @@ -{ - "signatures": [ - { - "keyid": "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9", - "method": "evp", - "sig": "30f6cd75b4aab91ef4be4e4c10a46feeed25993b29ebb33f5c792b93dbf9fd03900cee41740876636ee2e415944f318b9197c23eec60d1d722923c7391d5f24669738e86931753177d10d8a712b39d329fa1ed2d8840f58a6297e9d0a6ed3c264f01cafc8ad6ae05f6bb57d848dfdabd67e0f48effece56009d4e93e4ed328b0f431bc2301c839bde6d3aac334957c470e10664249e156faee0a1b34237c775b7e22ce2d240a32e25abe661aa1cba63d933c016fe88b37587cf516dd92dbd8cf836708c2e51aef42e632a333b6b579bde49429771e8b50df160d642c2d05785120b14f2614491426dfee98fb4a4ac87f47daaa05cb3d54b06f7a3de7d0c54643b3dfec72fa89f9c04b20542705a13e6a6c9ffe7b1ad80b8895465f7cd35e8b8a589ccc7a51fda6fa1d2e31942aa3a81b2659d5d799a6ec3e6c68b61df35c47d62a8ba3982886902d61b8d822c709e98102a9f01b65476620dd59bf6027bd11e9db874801268cdc709ebdee69b2f1f5bfd74d95c76f4d8e377ce5b3f17a59f94a" - } - ], - "signed": { - "_type": "Release", - "expires": "2013-08-08 15:21:53 UTC", - "meta": { - "root.txt": { - "hashes": { - "sha256": "2e496d43eb877fc725dd3bd616da0c1e018057982d3c8acf8906a4104680feb1" - }, - "length": 4793 - }, - "targets.txt": { - "hashes": { - "sha256": "44214fdc4a2642e7e929257cbc7f1049120637ee16a8478e8028c110d1350696" - }, - "length": 1183 - } - }, - "version": 1 - } -} diff --git a/tuf/tmp/client/metadata/previous/root.txt b/tuf/tmp/client/metadata/previous/root.txt deleted file mode 100644 index b7d8e066..00000000 --- a/tuf/tmp/client/metadata/previous/root.txt +++ /dev/null @@ -1,70 +0,0 @@ -{ - "signatures": [ - { - "keyid": "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b", - "method": "evp", - "sig": "83448b51098091a562d9074dc14e02af5a6f18fcc18ff0dd0d709d0d1a597fbe9c96d2a077acffdc85be0357f76c46b1d2cbb579ad376458a88b2e00330b4028361c337593c96b7c2eec10dc337c7652d9e83d7c2dca7e59230199d2e6e5d3f0f1a38f09d305b47954f552ecb45df5c247ffa60b3b15198bc17a17d9198e688289ec70e3043cad1c1ac9405ad81d1af5135cf961f1a45d60f08d00c4e1713722711bee4864655a495f6e105020702ffea947fe3288358dffa46fa8d9c4c47fb7b3fd8a47b2371a0ffa78bce885b2ddb36b75ca6f6fea807e236593b882a7b1fc6c0fe43eb4b6709b060e08fb3e6f5a56fea6a5524130d01b461f1d6c2e3b1c2ef8784190245568062ec888af1fee0740b81ca0c99b775396b421f507581257277ba1e8609528ffef0ef5c9c205b63874e2c37b5dabb738cf5597e0c39010b8041a87b00030f69217e41e03376d1899c25d0d66d9e936b9308709eecc9862273a91e42301f6254f501bc10806a8aced547667678c39790598579dcb79b064b074" - } - ], - "signed": { - "_type": "Root", - "expires": "2014-07-31 15:21:53 UTC", - "keys": { - "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAxyi2M6UtfeITuiu4kbWZ\nOU5CRScab9Z8eWy9Weff/IiSYD47uRhnseT84ls50yCDzaUMlrCSG4Kd+AdAfcVl\nmsFCnRC5HHacFF9EEPSDud9L+4LTgqM1fP4vWfv8ZZ1sXI/5Npeo/D3bcC9NmcfR\nBMCzaQl1NEoYK+QVMcqVVImoWeLEtmU52u96Y4FZv06FodhpHPVvKUmJFnXe/ikM\nLe5sTCP0QKLtv+q1bfZ84KBkE50QfDpTF1bqiDeuCtb/3yNb91gkpPspuWjn9icK\nFCc/D5FpYSkVnWCoPzYsuVcQzIPFV2Zr8L1gvSvPOPY7SJulk+jri2T1kZw/rbDe\nA9T/I1yG36/8IXv9JYA8yiMR4cblFqi00gj1Q3NrelBdYc27t8GDNP2D2H4wMw3P\nnTmmfJQ5WpR5ZbjVHy/OU5YCpeJ77sdBITaSVn5IWLYumko+vJohOoordMBfV/jZ\nTVVeaz0OCp9sPs46xKVN+m109/NNejS90gLIKPKDSfPrAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAuvReIoGpKvK0rwmkgccg\nmv8sTtEBolT02THaxO+6+gSPO/509msi/M3UNHsGNda1gccdwRMTsR/C9lCKC9Mp\nZ+JcOUl70SkL1lf4QlttWEDyDgDg1M7RPfcMZnq+eo/BPqGOxmbeNxKS2tqNfpQQ\nOqvSb7MMeII/mitunWo+UbJE/No9dimueFAjgHwQfH3rJCMnjfL5OT15UipHICCV\n/x6Rypc47kyOIfAvGMBnPT+sSapu+tboGPQf4kYwDpQVBPrJuGLYbgFvLAP/JKoi\ndHwXKeOAYP6FSioKcXbOspVd7a6FCUHQtEX8g158WOb28Ggo3eeXJx4Yb6ZW4WhD\no1jHNyELE2n1h42FETUNiESF3WDbfGZX8XINy+PXmVWGsY0YTdFOcSmw5k4fZ2Dp\nD57R7GQanMBBkpB/J5dVYvdP/NLTHeueBRFDJAFM+HRozbTpi+pFTcKV0dITLh43\nQrvSE7ZK2ktW1Zo5aJuhVClxga3teM6N4Hm3wEv1mfqBAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0BLhIsIORHPXuTU1CPSj\n34jQVNs3jXfpEfDIoYQcLPrNoz5icGWlg3H7YFtGSGOIOzJbyMlQR3iyGu7IYSW7\nuyJRX8OJ6rkbLcAg3z4QXZf6Y8isQYQGBiPb/j/LKGpGs6GA0wDqVfcCUJGtz04k\n5P4oPmJZjiQO2uMyQKYkJDWXllAgkY/SkuOUHyk/knE8EHIoNCwqCAVVnKc/gg/O\nL2I6mwkyes6eXQDRdwRK0z1P72ebzAgKdshhU8Jx4S1W3BTdX4CZ0TqBKxiNkhTi\nIWkZHM5hijX7NbCUNTIL4MUSDGga/quqs3kSMCM3lOd37MLiTDXHcCZoF09w6cgy\njsZyZZR7PBgpQjQ4EjgKstrAForp7ph7dF/BAP4Fz3uf9JBhdJ3LaIT/0et0BA/J\n52TxMT84ngzS+yWobqdrOK9xVaNOZTS0j3ScWpBKRCDR1E+llUJlkjphdD6Og8K2\ntFVKwCTYm3qfwHd2ulllVzbOOntTnq/ppcQjtTO2yYSVAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": { - "release": { - "keyids": [ - "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9" - ], - "threshold": 1 - }, - "root": { - "keyids": [ - "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b" - ], - "threshold": 1 - }, - "targets": { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "threshold": 1 - }, - "timestamp": { - "keyids": [ - "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22" - ], - "threshold": 1 - } - }, - "version": 1 - } -} diff --git a/tuf/tmp/client/metadata/previous/targets.txt b/tuf/tmp/client/metadata/previous/targets.txt deleted file mode 100644 index ca6b70cd..00000000 --- a/tuf/tmp/client/metadata/previous/targets.txt +++ /dev/null @@ -1,22 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "131fe09bff13d343f40e22fc735ff9593b73006177791be400fc4efdc0a87dce8767b49e144acb69097b5377afa87628693a70e19b43c1793043805b43fe112a273e72c095e3b0d2d9df16c07f8035879ce549759d353394c3a09cf0587844d2cf2ad6446c4e2bee14c242167d5706541aeead6fad657c81d6b8948d3a5a2537d7016efa5ab5ce595c898ee5f8b807519f01bd70f479763df0fd4d16427959e1ea1dc674dbaeae6c8917bcf42fd0162534610119751729f030f744bdf5734bd0561aae4dc33e1f1166d3aca17a5cb9efd31344ba99c172fffc0b0055f34c77ff3228a87bd3f94f5f31d069ab92ae3f4d22248e11e86d2f4607b7f6b4d0e59855ba2b76eb9ad6fff6ed895f38dd01a204983f9048db4ebfe5e0fa902880ce2a64ccb5d74f98d4a58d515b3873b3e15914be5ef2e7a81d5d95b180de6b8335c994d4e0e4d3ec329b9de37a5e2ffa4b22d4a93250c4d764a353de1dbf48bb5ed4c7ae727901276da7dfc34ab4502fd8898b8d5f713eb0c2057f06a75b35a08a13cb" - } - ], - "signed": { - "_type": "Targets", - "expires": "2013-10-31 22:49:03 UTC", - "targets": { - "helloworld.py": { - "hashes": { - "sha256": "14d9f7904b16af5b3cd64285eb349bdce11dd3688d6e330ab7da87eb37512941" - }, - "length": 18 - } - }, - "version": 1 - } -} diff --git a/tuf/tmp/client/metadata/previous/timestamp.txt b/tuf/tmp/client/metadata/previous/timestamp.txt deleted file mode 100644 index 8cb6f19d..00000000 --- a/tuf/tmp/client/metadata/previous/timestamp.txt +++ /dev/null @@ -1,22 +0,0 @@ -{ - "signatures": [ - { - "keyid": "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22", - "method": "evp", - "sig": "9d09a61312b76ab8ec6f16b1bd9800769c306898c92df7a05bdd3a7bfe30747b187dab13427853957c3b21c1dc93519c290564f7703f4b07529fc64ff0e75d91569471bc112ab07d6253475489a07971384bcab10150c69d10c91960b75f7aa04b2c230eecc299b0e174278146bf3ab070841a1ff7ff15c7ad056eb020c29785b3ff432e2029c7bb56cdc36933204ce52e7dd32d5a7b9349b3f00d0da65a3423dadb4d74cf9abb91c29533c6d09b81811b1d3aed04988af0795ffa62fa409ada0a92a72d1f04f89caead224aa734aeb664fdb62a04b6045e8d749e015a5c40df81e275472e681722ed7afcf2556295eeb2463fbb36b055cc8581c9d8457d66c67b400627ab42d4619e8d6f881c66dda74385de451a5cbb967821255f7b211a03e34dc836b42e73d1588ac2ed5395f3e1c0ff281c84a9298d895cbf1b748ad19705a3ee26151eb08bb51df0210acca69b6a358390b223201e588d9c750f60238c6144a264f07e0631305566c1d41a07432800e226caaba2481882ffdeb177c657" - } - ], - "signed": { - "_type": "Timestamp", - "expires": "2014-08-01 16:19:39 UTC", - "meta": { - "release.txt": { - "hashes": { - "sha256": "f22f2f12fad9069b2fb569d9e9273a95b015fe2d8f3937addfed3dc1e48d86de" - }, - "length": 2152 - } - }, - "version": 2 - } -} diff --git a/tuf/tmp/client/output.txt b/tuf/tmp/client/output.txt deleted file mode 100644 index 32124745..00000000 --- a/tuf/tmp/client/output.txt +++ /dev/null @@ -1,23 +0,0 @@ -$ python -B example_integration.py -Example that updates 'packages/A/Alice/alice-v2.0.tar.gz' and downloads the mimimum metadata to set the required chain of trust. - -[2013-08-01 17:15:45,651 UTC] [tuf.download] [INFO][download_url_to_tempfileobj:362@download.py] Downloading: http://localhost:8001/metadata/timestamp.txt -[2013-08-01 17:15:45,657 UTC] [tuf] [INFO][_update_metadata_if_changed:847@updater.py] Metadata 'release.txt' has changed. -[2013-08-01 17:15:45,657 UTC] [tuf.download] [INFO][download_url_to_tempfileobj:362@download.py] Downloading: http://localhost:8001/metadata/release.txt -[2013-08-01 17:15:45,659 UTC] [tuf.download] [INFO][_check_hashes:221@download.py] The file's sha256 hash is correct: f22f2f12fad9069b2fb569d9e9273a95b015fe2d8f3937addfed3dc1e48d86de -[2013-08-01 17:15:45,662 UTC] [tuf] [INFO][_update_metadata_if_changed:847@updater.py] Metadata 'targets.txt' has changed. -[2013-08-01 17:15:45,662 UTC] [tuf.download] [INFO][download_url_to_tempfileobj:362@download.py] Downloading: http://localhost:8001/metadata/targets.txt -[2013-08-01 17:15:45,664 UTC] [tuf.download] [INFO][_check_hashes:221@download.py] The file's sha256 hash is correct: c5cbeeaaa617fa9ecf282ed4ed3051ecba2d8f9535f148e14103c6d6ed6bfd39 -[2013-08-01 17:15:45,671 UTC] [tuf] [INFO][refresh_targets_metadata_chain:1384@updater.py] Minimum metadata to download to set chain of trust: ['targets', 'targets/packages', 'targets/packages/A']. -[2013-08-01 17:15:45,672 UTC] [tuf] [INFO][_update_metadata_if_changed:847@updater.py] Metadata 'targets/packages.txt' has changed. -[2013-08-01 17:15:45,672 UTC] [tuf.download] [INFO][download_url_to_tempfileobj:362@download.py] Downloading: http://localhost:8001/metadata/targets/packages.txt -[2013-08-01 17:15:45,674 UTC] [tuf.download] [INFO][_check_hashes:221@download.py] The file's sha256 hash is correct: 324aff11e6488e3619f8a291dc94f82faf60ab00ce67443ba32997bd0d1ad0cb -[2013-08-01 17:15:45,678 UTC] [tuf] [INFO][_update_metadata_if_changed:847@updater.py] Metadata 'targets/packages/A.txt' has changed. -[2013-08-01 17:15:45,678 UTC] [tuf.download] [INFO][download_url_to_tempfileobj:362@download.py] Downloading: http://localhost:8001/metadata/targets/packages/A.txt -[2013-08-01 17:15:45,680 UTC] [tuf.download] [INFO][_check_hashes:221@download.py] The file's sha256 hash is correct: 0af576b49df40cc310ba314a82dc264202ec74d4238eb526d85230aacf9d2282 -[2013-08-01 17:15:45,684 UTC] [tuf] [INFO][_update_metadata_if_changed:847@updater.py] Metadata u'targets/packages/A/Alice.txt' has changed. -[2013-08-01 17:15:45,684 UTC] [tuf.download] [INFO][download_url_to_tempfileobj:362@download.py] Downloading: http://localhost:8001/metadata/targets/packages/A/Alice.txt -[2013-08-01 17:15:45,686 UTC] [tuf.download] [INFO][_check_hashes:221@download.py] The file's sha256 hash is correct: 49d0adb568d9323161f987087894df88cc0eb45ad2e4b7972b017915899226af -[2013-08-01 17:15:45,689 UTC] [tuf.download] [INFO][download_url_to_tempfileobj:362@download.py] Downloading: http://localhost:8001/targets/packages/A/Alice/alice-v2.0.tar.gz -[2013-08-01 17:15:45,696 UTC] [tuf.download] [INFO][_check_hashes:221@download.py] The file's sha256 hash is correct: 21ecfb59295a055b39003e6b9a63b4aa7b3724458a7c663b00ee6cf69fc09e68 - diff --git a/tuf/tmp/client/server-requests.txt b/tuf/tmp/client/server-requests.txt deleted file mode 100644 index cdf54025..00000000 --- a/tuf/tmp/client/server-requests.txt +++ /dev/null @@ -1,10 +0,0 @@ -$ python -m SimpleHTTPServer 8001 -Serving HTTP on 0.0.0.0 port 8001 ... -localhost.localdomain - - [01/Aug/2013 13:15:45] "GET /metadata/timestamp.txt HTTP/1.1" 200 - -localhost.localdomain - - [01/Aug/2013 13:15:45] "GET /metadata/release.txt HTTP/1.1" 200 - -localhost.localdomain - - [01/Aug/2013 13:15:45] "GET /metadata/targets.txt HTTP/1.1" 200 - -localhost.localdomain - - [01/Aug/2013 13:15:45] "GET /metadata/targets/packages.txt HTTP/1.1" 200 - -localhost.localdomain - - [01/Aug/2013 13:15:45] "GET /metadata/targets/packages/A.txt HTTP/1.1" 200 - -localhost.localdomain - - [01/Aug/2013 13:15:45] "GET /metadata/targets/packages/A/Alice.txt HTTP/1.1" 200 - -localhost.localdomain - - [01/Aug/2013 13:15:45] "GET /targets/packages/A/Alice/alice-v2.0.tar.gz HTTP/1.1" 200 - - diff --git a/tuf/tmp/client/targets/packages/A/Alice/alice-v2.0.tar.gz b/tuf/tmp/client/targets/packages/A/Alice/alice-v2.0.tar.gz deleted file mode 100644 index 74ff1a79..00000000 --- a/tuf/tmp/client/targets/packages/A/Alice/alice-v2.0.tar.gz +++ /dev/null @@ -1 +0,0 @@ -Alice was here diff --git a/tuf/tmp/keystore/76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22.key b/tuf/tmp/keystore/76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22.key deleted file mode 100644 index 1f77a74c..00000000 --- a/tuf/tmp/keystore/76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22.key +++ /dev/null @@ -1 +0,0 @@ -62bcdb677dd637d1@@@@67900e0676e73b0d6a1cb39dac5d9b56@@@@5a21cb1ee29e9c9a2803e1e583a2f982a7d1787fe00e80808b77d8331807e5d10afc317197e25112be5bfc0fb8467717761d6b8244c33f404bdca82b8f7ca35ce88264f0456b945266c1eb3156f281e105a397404174b4f4d939f4114992e42cc2e2d5fdd3b606292009a6188ac6e8362fcfa075bd5f937473c54e665d000786cf277d5e85ed79a4d4b02c53b3d0549919b0b351cda5be91ed7196660a573b226e666af124e218c979a738983d083df485c02337983cea247805bd2384f7d3ce81a1b7f5ac07684564d4de5a25ad03925d82981a969e661c56b7df1284d23fa316bdd9d3913af26264984f92ee209c79d2116dd022569d1d9ff9379f406f8bccef98fc2b6ca93ab39aad8b0f7a825207fe136e8ee4313ba1baa58bb37dcaa10e744a48b81e2e43d077de7c34021ef9002d930c2bb1412dd4987308d96fe27991a8df947b7bf80e68a8c2c97ad5b089230bb88c6c61d6954044ad1717d54a00c2ff8f0c34b8bcbf5820fa148a07b23a2c37432743592d0cdb413f14a2f03fe25c0da903295d278fbbe0b205b6155280ded5ca6c81560e054875f12fa2634b9933bffc59f14cf60da48de9189d1fc79894175de190e44bba47a59cc9f3ee075870e435824aec894332a920aa287ed17e8951b5baad2cb212105fab4decc3cfd3838a368b70fb6c79fdcd50cca3050e71b2e16b139402dc3baf5f0783db78ebc0235e8524fd66572519d6eef4a9de891e5cf76e8a4adde214ffdaa7a00818671ec911f9082a3d78741a715d4ac756bda84f929adbcf27959ccf2859cd3a2d01aa0827ac90a431641026d92af46192aa14a3362c64d8428936798e16203c1ed86d31d5cc58abdf816e2eb1c87ed58c514bb3b6a8451fa36cdd99598eba5561229a51d35b1f6ed898e34acf80a2d4ec20c8c8058b6d8f13cbf31e55bcaf8be5cfb132c5d592abf0da36684ca7661799c584258607d21aa2647698b64d6d492a1d927c7a65a71bdde4c58cc816492da02461d0d9f2122bb68539fda9114a5612c0297b9b36f6f2f23f774e7e97a063033d8a09c2eb25bd2c6b5c848a864c703329139e3d057f96ed365759a7d30e1809577926bd5c908d29fdb494920b354c8626b01c3193764e467d440c69496f2ff661d4e3a2742046af45f737d05ab26c6006940744768e2bc8f484fada9888ffb8385b249db61332b1f2fe2ebf3e1e0b15efdfbd6bd89a0095862295eb5e88201808e2dc5e08a47d2a9be8c8acd5120c6264a7654a06b89d3d1cbd3daaf9c758da3d000418f04629aa3c7c6792f228b08b7ea26319261c7499182e1e0a242dbb72dbf7546ddde6b75da98440c3560909402d6fbfffb94cb647696fb3f71346ba02872a1f7ac573d0d40627b155f15bdb2aa42a7c68fbcad1821ac1c1af0f53a335c6f127652c2fca8d84c1d16da61e7214b0a838e7782c70144468df9201b4d2b39f4d3349502f0fdef19f60f5083ad03003dbf9b6cbac5d64ec20352572c78c88d2cb7c0b628f81ebef78ad116801766a4d96132f1c13d327aafe32147e782eb30dec6ebccb4be81ea6c1c7f7cff5780a8c818e413d465854871f62cb0fd4b8d43569882f19d21354a3911f42eb1b8da01ecac73ebd3558b7e1103e844c163b8dc6a960deeccdcad6a4201440cd1154dc1d89ecea8680322003eb18a54a59c1165a18d3a739cca28c56597165a6c64aebbc8b00cb9cfca188ca4a81f70a2cd525df0263293eb645ad855e47fee928d9ec3af6062297277ba213bc0770a70aca647be8e950abe783431690a82ce54c869a15a72c688f18b9004ede0778a4a224eab814eb21f4bca2dbb94f113ad8de6dd8ac32c6ae8a8ea87086c8f2f79301506d9bb08873fd94a3d29919e168573ddc43fc870ca8d36310f9101bb5a8342bb0c1dc50aef15244cd3748d28b4b4fa34ec001c51c92073a5a2aa93bdc94aadbf9de95f895154c286ace6edfd259a35f69c009c4b2480f95ec0609c2309a7b9c91119dfb8b3c32f96c2114efcb00cc3789684e917b96014a909f95cd0dcffc29bbc83854f211caf40cee43ed190f8e2754dbb2b34ff00c397fb7cb07273449a42f60062c053af51bcad65ec54b0555c7cb41130f2b422c7766106d73d8923f55bd6f9f4d4c79512844ce6e28b3dee56cf2833c86676bb677d0ab03540cbc7313e563d2145e548d21fa5f315d40cf0d16b19882649fa5198bdd0a3e23bc25fa25343178ba0295a7e0ed324de1de0c47c22a2339a9dcf49303913f84c16f5a279bd6ac0b8f84d38b0d4795c51be5811fda225347e65b3c115687a942b8167eb95550e590c5d0a56fbca3419396a8a9f1840c5806312cb7238630421b63785127e94ef5b8b2007c1705cc9994f75fb5961d3417423d4f068b9b11be323681ce0aa8b1eca0d8179f7d8339c47fb2ef9199d162dd57584a430f090f21fa55f4c27cb859afda6926b111001eb9dcad11a8fc7806ec7f322e9a820831b5dd81c831556f6b09adf3eae634e7f29727175c6db3718f4c1ff5dd360cb02afef96d93515e87db812c690fdb9fee13580db87658d789cea94e21182f80a5256b60c2d25f8992e009399c9f8736d88d572433d9f3428879e4111582341990b059c5d56a447a806c233165e7e518ffcc7c48b6929d280740d3e3ce471ebaa7fdcd724b6f928723f68aec822d08db4ffc83b1a37920f333472ad9bc2462dab9319c6ee6e2bfd2f0dae9b2703cfa3a799e03af14c6b2606cced142ddd5183fc1e7846c0bb963f88e102ef27204200d19e9a3a59ee21f45c25f1aa51ead5c1dd9fde1cb2af43817f075d062483709929981c4f46521471b0677849148d13e96628799f587231613ac6fe1a4df271199767ed3f4992274ea55e747738324212706e018e02f5b2a135ce26fac8656fcb9acf89f8bf74394d4113edabfe8af04de6c59f5acaab5faf0d1559d67838b25465ac80488437d8f255f83846915ec7bc652331334bf2afdbab048f281cdd849325f8bbf79f47c2d5b4ab2885cba527b31d4d8f720ba40a0c66217c191a8c90957712dcf726f202448576ab8267550924ade279ad3a4bab5cb66579aa76b1391d628dbcdce5fa29f069966072accd350c73154a8ebb0e0ad3549ebf9202c727ac68ad12ca3a50c5656e6d739ac3cf0d8846eaee8b0c12bbb8781bc8d9d4aa5e68255aa0bfcecb63e998faeccb37a1ee0b5ab121aaca507c71efa8dd1f52649051d35f12c833830a534c684cd8fc6a50fd68d9db1691985e86e79df7e344cd0a4ee9114613db870ea5cbc2678822b5f229fa2c6e0e858eeb53e9e0fa0cab5c6c3888a9375649fedc448b85b0a1f132825d95a5ffec9c40673159c1d1279deb6e0d5eb1b39c92211d00aafcdaba338a91c9163db622ba4f721b258dd71b1d9201954450820a537c8b72d2fd4a43907fb65ad7633438508e2b72d45181052f69caa398a57ef77b383ce1b4c1aecfa86643009b787b888b060b15eac9d244ec023bc84dc5ef60c7c418dabb873505a6cd8cc1a4fdd33bc6dd62741cdba8f2f954b3683ff19683cfe7fa86010e3c832d770545660562c6294b5f7cc519a512f5fe5bab4dec3957a3ffa7dcb238ddc41c169b7f8d62636d651d1e2b341429674bed893b21c086d1e900ba1bf62ae9da8bbf2625dc34f4199c39a86c565a36227514eed22993f4e43badaf9356c50cb074b27685bb8aee47e80a338c617a0a5d617dc241208264a3ef83caaab5d56b22bc43a88e4d780dd06b55c2c3fe01d70f99b221f54779880be10717bb5e11e5ce99de36b2ec097e5d261b3dda04ffd21aaecccb938f36dfa3fd70c09a41a48dc57d10aabbacc4e4eeabd9defee75cbce3fdb5ca140dc85c0b8fb99c02ad51477685057523780d13f3ca13550944a200973e0afa4904ffa30478e02c6c56049fbd775b2f24c06eb212d1ca14b78dd3b0732c79ee39b297ada9b37a77fa18d4bbb378bcf67b37433df8e1f22373f911a5656e81b5d8234a0c4aacaefef4407e438415a03b54453054243f99f8183a8f8171e4b7b50aad9a6a480353e86643199f362d912093dd7c47cfd68f04951241ef384974a10c6178e8b444afab587d8eae119594ee169e6e257a78340daad16b949fd997ddcf237c33b411bc8fb0192cdcecb4d7e19d05ab4b3bf754e42e82fec502fbe37b7b5b9c4821cd364ca76249ee1b7c500ec7cb99005db2c5507fd5524989c6f38d492c7ea5fa8f18dbfba7aadfbeb2f7427d7d23378a1c9e05bfe05482b6d43af1c5e515ea3ef4cc88b2ff7bf9c5fa34cdeec05b85528cb36b6e81bbd00f4e52de941b19659793fb73845b7bad61a6038eca4f66e8f5f4e754e4f461f85aecaa791020fcd09e49ec11dbaaf2b47348c1b16d4f6ee517821219d4318c796b82017c783f40ba46586e0f093711f1bca6f9e1faa194ccbb59216e892d9ec2246870c9efc976fb56104c6f77c6a3713a656930c021951578159d989afeccfe4fcad5aff016cc58b99ccbc117f14f2e3341466 \ No newline at end of file diff --git a/tuf/tmp/keystore/845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b.key b/tuf/tmp/keystore/845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b.key deleted file mode 100644 index d95de382..00000000 --- a/tuf/tmp/keystore/845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b.key +++ /dev/null @@ -1 +0,0 @@ -22f8e4f768a9a082@@@@d130b5ff4a8a53f7cff7dd9660078c48@@@@f8dfcbe99fec59ac6be10161f17c47fbe0d790159e54c6ea8d8ff78ce93334b56b7771bf1fc4ec80e79512f507e00ed796bcc662d5e7414c5125706e0680ba4dd7181ae064f17f4b21378f4bda0eaea2db6553e3285a4393d04c0f7fa355face363767e54f9e8fbd742675d2b0dec19a7ebb476bcfab08809987623cc6ef2f5bad06d3c7b8b2af36fe8f9adb1bb37461bbc66f07c8ddfc6b2b924ff98e1dd7bf1ea363817651133ed042cc9650466d61a7f0e6ff56414275b95002f2cb633a7b690aec5ca68f24189d906851f42a6ed7d45cfcb854d4cc2c864757f5e639fc52a224825460d43bf6c5226cac21991a0186897ddfcf338f195f7c85d53577a0e5ca2afd2bf1a1e7af0c3132a1cab53d762d3b95f3373b4c6a3fc8a03e6bcefcbb04e8e5a5d8f5faac9e2f91273981a865f7b3b09adf008d328f60ce936f5a7cb177c996ecbaea185c37b464c01bde96882bec80762c25716586194e5a805c8d32bb5a0fe4d381afd44f425e3c216ee62352e9a48a9f3a778de341aab60fdbc10ebddff22256f375700057672826c6b26d6bc2ef640c9a42d9d0635a7fced5092f199d1ad8b21907551ba5b625f9d74c46a9a34caae74a1719a20719cdbe66fb67d6d10536f31c59423cd549afacd08a8bd236ebb2aabc50de472946400146d4d8a6dc1af7bc8312683dae9882dac15fe3fc33768ea00a20611db89964d7721a3d158bd7f60bcf72b47d7016d71ddecf55812ca32ec5865ef93b8f2e4e325456d37b0f5a812979175de384ce5028f8b8f6b2d558c9739dc3bd1afb60f75456e75df47e2798e66347ca35ffdb2bb79c68e6a436f83c430c962217c361bddbfcc3a14017eb3980b8fcc89f11d9daf75b168ac092165a9a84f90601a3277120461018ef124661dcef7281a64e80c3987274fe5963b6ee6f59d18b377d9e137543af12564985af6e4e86b8be70ea66a6597f74ed609e8be4f643481b0e41edfdc8f4b14b71d1d61f707ded696fece952ffb18a53f54ed6e95ac845629f8496bee6e18df1c864ea868583b5a1bb7b774d46c7e6dfcd9cb4fc81dab219f947e5a9fc45dec32b35fb481b936f4fc0ac6e505e08cb8e85a6b975f8805d83e764be04dd4a69baf6f2d384a52d597559ef43351c52e2ffdca70f7348f274d62ab31d5a027e0db0e43eef0dd94b7d354affdee44ffc76628172639442fce49af0361829ef4c1a0948ad9103eeaa5f79597af7f3204d3e57b8956a47e709831c0da8e63bc3135871b1bc391cb2de51edf52b5c2533a32e852e6a892164bdcfe5701aa82fedf199b8d99770e2195967c89f607bcfaf4dcad5e2565cde95c4d072674cf9df9319c248cdb414d09dfa2205f9e358d4c7fa30eff96562b448ca0ca96a2c2199a6a116d8ab0bec82bfcf42cc3f6d4c0a02b4e92717ea29b280cb4f8947a3779cc6fab4a95af4ff40169eb2ae83e0a8e50876d861be4e292f4dabb0261322053e80058a3368b139d9d9f37b38c1bfd84f5a504b70c6766072294e008ddd4e002c9003cd860089279c467bd495237f827626f41bfff6868839afc1986998db70249a71bf0fed15432899ad84a050a1228627f86f4fde6e63e36f43e947915afea5f6d9a8164200672f7ae5512c8cd99e86f7ce83a69aeb349a8f663896cb4f78a55b71e88ea31fc9cc671c677d9b8da4f6b9c5deb36479c2e4e453e5dcd3a3f676be24ca790c0b98ef35ed7ef06acf92714f03bee56617a26c0db643cb0921fcb86b3d048c3377d1f0ed4af0ab16b52f50016242834d48abec485ee942034775b51ba09e1ea9719ac9fb0101256bcaa5e1d691d4a2d2d910c7fa735ab04fc38dc342a86c71ee8a02fd37818442b4cb355a550bc5c43bb273310600b7d6b63e7033dff877280d2ba9b9c0766e6d302a4ab3beaaecd246da64cf59d2ac08cd1182d3c5854c25ec1a98faa1e9ce27a4ec71b8c0db6a82ae787642abfa5b65a8fd8cad2542b11045ac0aa31490258993b2814080520f8551c927c227c0958b0c416c5409a880f3738208c2e5a6a90117af9e58cf98cbdf4cd273577c86a1e75575deacee7e90e26c646e257ea0b394da1a36e75232aa0f8b28cdc584778d8ff011dd5e2070e89ceb70cec47aaca2c34e23b5f36e5cfed495ce84036197015eaa3f50114381021255d1afb180ef2e32ff869a7ed5d1ff63e496c434d5901d5999424288badc25e9b942dd07f236f71f181d0b10179e6c394bdacd57bdad2db850e489772bdaf0143e78fdd4fdf73043d02d19a92e7be6a861cc780756d0df7c8399b724fc746170ca6052ff008552e972ca373c59f88430deba1b37a290bddde5baa36fc938c202cb8c78ff364fc054cf861cb572cbddd2780b45986aad3ff3d2372930d894236891ea9b0f3aec6d7cee3fa4e6f2f23853688d36abf320684d63cf8925b7d8f753e4581ce882d2aa168ea895e797667c6fadc8a73a7856b6a87ed9a246e76f0975a2e28c61dca89a964ff14bdb62916387cb930f8567ba8c1058b6d84adb2ba47ef0fd551bf4ec5f5aabb7220149964a9f6ccac9dd64d00bcdab96caddcf67a292457908956f54817e09b198d0836ee51047ee198548e165ac48c41b6dc37e0c0fce0b3f0c854315c0feaa3d2a3713cf8658139fc024c5da5b5160d83bce2b8207c1e6dca03064c36f1111f0b475d50c9b3e33d277f4e07a073d849849586d95740a24fe78b8642a782f40e0c52dbfc519a648fca3de592ff953677e611bfc565d88743f01977b2f8ed7c29786dbcd16885c326c776b66d336242bbe9a52b2c9e29f5338fba10c4c0170390d7972de50f78ba12c39143956fb6046d0d9586262b839f89f9f0676ee772f4cf975d0b88af8506b1f6a6dc2e84ec3a1aa7c2ea0acd60a67abc66c780e05f110d0d77a50220d4b1874214310145aba522de57dc275b927b47c4d5d7fb9c8295ae52ee93a184523bf6911044bf731816092d32956f6ac3e4a7719bc229a59c8ca9f87dc636eda77c1f920b475ca7eef780bfae6e09a7aa6c1421e725ea5cf69113d698de4e233ebf430ae6cc3c4c137ad98b5b70c64a716e0381132ba44d8d802d01549049fdaa29c6efca4ef5bf94640c293fba8e288148de6048c0e9306fc9c9a0ea0b5191773d3e400907533c2d613758aa5b542e011cc610eadc3d543ed7957e451da815c605d7654c3a387cd3b7bed2d897955debb8cb574c39fab9f53e43559347486564d53dc241bd381714afd3ad159d1a74d2d86d3cb001639e3f3d3132e2072464b0bde29c971aa9bf50df70d9bf78ee6064b410ee150d83198fe3e3c257f5cd805476f68e5a4a22c5dc2855976e8661f946d401754dd8f640ad98358360f5038837229471a1b581bedaec41230040620d4701003d1e39e799777aadc1880d1da44aebec41e1dcd2d63e9c7b432864efa5358150ce25d5b4ec89256a2e538194ba3b796427b26a520bdca2c9eb09b36e3fdbc228b94fb21ceff8025ef3c0ec9623eb56819eb828e5e4ba44628910853ae6edf070bf21c4a6ed121628197bf884ffb46053a0ae20701981193436847004518d8447ced9a149a528c5128e175e83c9922094fb04ffe6a78543590f308d4619824a0359320fdd54312559377e6fc4e4d19baf79602cebd397e03ba3c852886b92954b31d7dc9594430e0b022016c0d93eebcd4f46af644cb93bf0e7f7b26c510d2b71fd9fc8ea8d0783504eec466cc8ec45d966093b77de85a2eb4661f665a07edc79c098e855bbd1b7ed5fa8fa955fe6d4f1afb4e65d739c3b1e16170974907c4796a394bfe4ed26abfaa9a5e352d01931e5f4c5ee117d10186fca58dd098d39c8fca7db059930a86f2bcc722a053575662f04b0b0e3ea254484d5f44562e43b081f7f699941502d709ec7b71002fd6c8f060752baa7c8aee65ea54e2411cb36b7c899ca5cc3bb74d42b40ec823399631789f31e7b72f6a9d66e622ae06f870e846e0c7970b711a2a382da4178ac53e9de6289dd79c26f54a5273f5dede8a2cfae6ff36f494c4e4edc827afbb421321cb063021d9afcd61cdffbc72ef1fc760362b59a53419f4a79cb8a92241136f45c24b0cf2f7141a276357318915f01a9e1f1df40f17aad6229ac2580230e02570e64582ade1d2f212c3fe29c8c06b4f2ee11a44a590899003de90eded1e1635a6b21e173b89215fb80576970307fa752f3e67693e709a52da05ef0d24ee07eb7fa07ff0dd11b205306fc9e33c5f27ddc073824fbf04cb7f12b442d78d1daa77297c8809954f1416ba0ed116a7f251568488b65bf12211cfa8d6abfb944871a40e488c5e3344d3ab5a407f786bb4aa8b8e71777193e0cd2989834dbdb93e546262cdd069caa5700935075f092b2bf5eed24f402ec5e5dd1563364cfadafdb5d7c0ca0dc14c9ed540186dba3f8ac0d3b199a6a0762304e892876890e000502c5583281096475d57b5427aa2cf7a02b889bf75f88cffe6cf7f65986b70fc17c1bf07db8ebf94e28600fb6aedaf166a4c5d \ No newline at end of file diff --git a/tuf/tmp/keystore/9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9.key b/tuf/tmp/keystore/9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9.key deleted file mode 100644 index a4df47f9..00000000 --- a/tuf/tmp/keystore/9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9.key +++ /dev/null @@ -1 +0,0 @@ -ffa1082dfda57293@@@@1807942f12e8643c1902dfa73c62bacc@@@@8b90a0051e0ca57cff8dd4d385d78813bf4f1f3d8a9fc269c09b1f479047aabc66371373ff85b2f94f8154ad2d6d6d2e63779316807b5ce139dc1b8e062d2ffd9e4838e1b01e171ae0474f58042d1e4ba09e496bde5bdf009adaa3971f00b228218992b88a530248fcc03a7c2fd163944ef074ac32b78eddc4d8fc1e572c5bd6435dd925eccdd525fff1625a1a3fd24503318b54d33e141979c8716e072c0d73648caeca7f50acb7c0dbf1d1d5fd345d84ee0d3b0a4d73e3226e17c19ebeb4aa36d171f87422d44dd055b28dd843d1b36b7f612e3c01194d67e5bc66f41629d17915a5edb9c36f559d5cc93bed7f46d481291a5908f80ea1920833f147a33dc890c24ea970a7c4376d7a017098a17019851cfd7b80f031d797f7a713add081f1ec141e2f60340ef86ace184e7de7d25be2a49e18b5581be2d8d31c93b526ea607e3518629c42e4f51a4128ad771e585f0f8780b109d28646fa2744451ec549673807b88f20fb8c76f2c5cf1200d9c6b87c5495d060c6e439c09b0694099d1e00a237484eff9a5353b0a7a5f27836325b6f9e9f4dc3849140b21c813c673b9bf71c41a46eb985d29c4de7fa6bb44915bd29fa6e7fbca67112b962707a177cf78d4dd486a1789c86082f154ed35946009697bc0168330fc4344a50e85278629748ad3087f9647d7a22f4ff058ada50a22500ac58ef79a81473004cbb46fca607c6aa0cf6d51ce97096a1ed8ca53a51ba2d58b187a4dac7ff6d0963019a9dbeded86d31f972cb206ce4c5ff8144083a8e7617a171393328c40db8efc65b6313d334cb6b38109f5a7a5d3504bb5681ab353e6fa26a729debddcb26e7e3714282965800b99d215e1aca2686db147a6fe2f0b2c08590a6f0eec563504b414be09041b312be0c828583c47664231f2097cc6f7f235740ac5f20efbbf9ca94a5399b89f014324b7f63d40c81015a66bb1c21e5f5e9b3b25be78115e3d275d9c589b334b3f6731b65bf138ba62f21cf920d781642a634ebf4d4df0e436c27d8e0313c8c91e0ba3d7f350e8b10bf6d35ba3806a078c0f7d79e959ee4b692235db3840c2a09cf7611eb10f9660c3b255e5a47ae6740cf65bcc5507886b0d9c761e3ef8a07783b27bc3bfbc17ecc5f4cba4b6d2ff21f6ca1f1692f0952fbabf47c40ea7f7203a7daa59d7556445a4f8e9dbc95da03a1d4e811d4da0f70be7925597d8a3a975ccd7ccdf1f654b6737b82c047fcf4b99d96164888501fef2443927b9092cdad5abb78ffcd0c2af9179e340106f27b824f22e19e5d1b5b414a560c245f8247f6223d476a74138c02f9b51cff6ce94a78c1cee1d742aa071fed5b2881a6e0825eb408eea19c958dcd53f18923ca0b78bbae1400ba3bd8a29a652d060eb21020ff86783592192a9e1f463a7ff577830fbfd7d2da0fe9717e369c7c0d913829dfa3cbf8c3993b2a070dc7a97dba189478ac599081056836e6e60260def44674dc7d5e9b19071e865ab53689269e09c4d482729e2a8573e410ca1fbd42cf2697a21f3cdb1b05b7c998f8a0309a65bb74cfc569c17efd4fd01fc145c0dce1d2ca9fe6f307206205d6d4d2e4340eec171846e33f54895e1fbf6e2ded54f96802634c7b6429e81ca06e7b0bf3597a77e6a2cba7cf34000af81b5e0d52cfb5259df0ca8ef01c6d39382be8506d6e64da07ac82d4a28cdadc65e6408c2f9b00d7611cd29ccb4bb46f1a5fd4e336a211d2d05c7840a620c6b1d55ad041e9f8515e1fff24969d406f8d585814efbdcaff708244b64a5e7fba8d53da785dc40e2c20bb20e37834f5c7c1e989f721d6b09ca2c93a71455b2664a8703bea0f60b103d41710ed17164f985a5dbad6d34b18dc0efd4cc4eb45a1bcec40bc1e8c2bebb4cbd4613eb08aab8d3689c3b3813edfc85b320d149b337f7382a3cf0ba86739a7335f5a8495ee80f63b11ce7d1f48e9bfaa45ba9d7fa1ebb68202263611cd5e812ac8f720c8c09444976f83ba1d3b11c51d2275ebab89f0528d8adf1c44e2014c5299e324eaab0ac98da61c5b04ccd5af2a3aea5b28edf4d222b2a1efbf8e61f7e2c15849b25eddff54164380616bd9a1f3533f4e68e600725ef68d119cefea0b7639e5ac34480747e15bed434dfd66b45c41416359d74fa369a4bc5f75ec1e3acd7c9721420d799eca893d4a0d44058dd106d88e187c702431badb282d0d8c466b63ed31e21594b734573795a27c5a05e1412c8b4bc01c7ce656093a03675dc775f3ac83c7a4e5bd6826a37e7dc8ab707b059d9f33ad9dc5ec4d985fc824afbd383ac674858ccc17230a5f1f6a3ddd36b5ce2e5759211359eb9595f4f8859aefaf60c64e5cb73128a36eb75562a2d5e3e0f57ca4ac23cd1a93b64f7e8419b24029d064d31ce8defdd20c61f7649d53750d61de98dc76a5a2feb8cd1f4ea94cf5a0a19716fac3509f0195d5312c145472788f3fcb15f2530c83ade5e1f52b89f88b64f50d526ce347db5cf1b69787cbc4251c9ab358e0a27bdcbbecf4ba45717349f83367564df5555c214045e93d703d34010315806c464ebb0cbbb7e2316a77db218b0a07e83a0adac3eeb47c8e8827d4db18aed99babceac76f43626bcb8449279cca7df6a9a98315fb68bf44527c1cd4080b6db386cfa31f9e4d6bff6a60b552f79694a57d223973e948c4b420e4e28e97d72f1be6b24c47bd1557b0dbf3a8b5796a940ce44a934ef5b505c67fd62c201fb646ff669ca8951621906419b2d6e1edff80a2f3ee2b88cdd7f5fb6571b8e222de4d29d5ef53b78182a1f184b19fac12e6a95588ca61a1530b82911a29d33f96c25347b6433980d6ff5ba861c799f5154d9dc437d8a79bb0268ab0f6ae10b2fe041b1978c056995813b85aff6bbdbc3b05049ec4ab5783ecc014bec0b5d0796fc8f7d24938419cf8d91853c4a3de7180db21384faa7922c993ec1e90d6289a4e120cc5f3bf9abba48607897ffedd62035e0d3a64e5fc50dfd4447e2e08154b0d927b39a2b0d4c2f548f22b050dbd5b57c24f976bbee6e99bcf076ad7752ef112205376fd85a507f3c828ed725d82f98cfd9a48d6d24ea257dd9336217de95a2fe0f6b4522393b92b45aba7bcd1274892f953ff1ac8e1cfb7b34054a380cb698bd6eb3a7223987a8031141bd940aa016c203ed2474f2175da0df51e17ed345a3e3ebe4669daf4463960df9b96558e0e823a17d0bf7bbfca092c61abd2eaa8e752bb1425b8552187f1ed97c9393706ad4ed7b5cce410d2f109f031c4c4d9890dbbc6c949514b53051b929941cf84b6dbb58f4679bdfcd5369df94c676eda6a3a2bec0da14b7799a1d1fdac51908e88160cd3cd00cc35e4399d269511151ab3f6037f71e7891d69c9ad17fe99a8dd7925e7c389b9d7a64d6bdc6da9d1a5754bd28eeb241ac4f5fcec02105a0f87b49b25a123997a80200a91b0b967e574db4a3ead7ca8861528044be1d4c3e0e67adeb1e315e8a6e3a8769b6dfa51d8b0f614f73dc85fc19209e06c61a9363b5094d1525057826db9a220a77b21d6dd07843a38480cfd04aed6588597d2b8e61929db15c1b58cfdd29abb60dc43dc613ae271c7d0e9851b8320ec654fb5a1329f5a7c3061f47adeba485f5efa817a57cb3578d81922aa338541143977cbdbbb571e43c9e2cda533cd3d096fc179d7913534d0558640cfafe2925ce7c1521cf0d88b2d96a877b2848a65cad25b6d5b167a0c4dbe8d5a6b22b1d9dd5f2637038426c1e68e6f3c811c5a311537eae0a120003de51731f43647586fd50b449aebb2717897c0ac15ac1ea652c9f1c5aca59239d3bc73f5f642d0442b414d92e5f5a5943a5577c65303255f89fa3f0d9f3d68a2cbca9eb45d7fcebc0e7384272b3d8eeaf11dd0d2df0023da90e9b12a044fe8e7295990dbfe708b3a1a3e5c35121de6028c7ed9ea30afeeefc9e71f00a441360589d4e88bf2032f5b9493cc4eb0b552cf7bcc809facca67e1a4b2c4df47dd85044d193c99b112d0b1372c8931976fe8f7d22f425960609cbd7453e102b09e6e5832005bc6d507d21a95af7d885a53f96cc8c0631705e4109bf38b77cdf3264677471dc82d42cfb8e1d0fdab72241b1147ba3d9b6828efc43ec885cac44bbde464340f63524339246a683bc846ff66eb7e4629aa6cb8a8d2b84738290e218d3eac9ab11a3d1ac7fa324c54a2c12ee42a1b3bbdb5e278caaeed7827af95c17f42baa9fd8eb1faa422376493dcd3744b959d3afc177d00a47ea7af3cf0ae27438d47bcd6522d61efc380e8644cb6729829f5d84025f4ad1eeec7543cecd9c5d20b511cbd8cc1413630827fa75fde12a8d32d7f405b7ebab98d7703f64cd5d2a8d2b06efa9b1b5539ff5f9ca2d8ff669be4fe78e61674847ebcbe6730808a71f26d9f1e3002e34ea939742eb48d81bde41c3209810257054600bdef0cfd83e93a6eacbc544ba20524f6fe0c4bdc0bb66cd70e3ceb3866994f9c9f6e4a1a3691b0898c0020ea4394bc61114933d7f8d011cce4f46251506c3189f \ No newline at end of file diff --git a/tuf/tmp/keystore/c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b.key b/tuf/tmp/keystore/c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b.key deleted file mode 100644 index 89fac20a..00000000 --- a/tuf/tmp/keystore/c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b.key +++ /dev/null @@ -1 +0,0 @@ -4d55b7a7f568191e@@@@03bf204f88ba4ee6a698552112cf0351@@@@610d3c31cad4e8a2e5c1c1377dcded8755dfa64b98f81a06b115e73fdc2516bc28c8e46c805a85e72c524b4b0e7de8c7dd7a23f712d3e0a5e022e2d09c8208f252e376ed39391d1447d61b5ad0eb5ef8218dc8f4bfd59ed4839001b68dff940e19ee9d84f6e9a75b096df0714bfd77e1d1832af2ad6df6a25210f9c252ad76d0af2ed85b68ff6c683ab21a9114448ebe6fef45c63bc35fdd65abd6eaef7b5aa568f7026bb4d1bc27c1e1888d99759d7d1f04808a102c47f55f78bb1447319c1e6ac14e43dc0755145c22303bad1925d25d8b6ed36cda54cada4f3741425b3519f47a8b2d72d997d63e9d85a79ee94df410b5b1819a599ad5495d9f0d5ccbb53b2c9777520553d5cedb9b3ab205e6fcb1b0803f9232f13778002d867145deb7ba5c848c9c775c4508ff8e8c0f44dccd59659c96e5fb7d92d8bf5c79d93c5ccb2eae7d1ff0cf6d39608207982ec99ba3ad56c748fa84cc318a225ae1d3794d2cc5b17b117c0632a942c03cf020b2db7af37de33792f6e02d82e037ebe1adc2669a4150aa0b864b0f5396feca1a9cd0e022d83d6af15a695cac9448f5a8b0f95014a6655bea667f0fc37242d444cb8089e2d08cf5473da17202dcffffd5a6a4c30440d5d6127e265bbdd3ed446f20739acc881dc84381881565e5e4e1410476d2309e1623c71c76f3ab1b823552b991105967bed40f5f525e1156e9609c89a5c5be26358b210ae78d2b7f89785b4c884acab40d5a125e0952defff288880beba9aeacd1ee30c00bf4137d7f598129ab2c6a65ff4e0f95051459879f045266392be0e9d66e67bb9ecd669446400a140c16441a3b92c178aa85ca4554f9149b83ac556c690acfd8e6e9b399c0225bc5c212ef4acc880a3e7f64d5e3022b30113531743a2beed13389c8aba61830e15443cbd6a390b041ee0de6b30a45219b46e9e0379a14e245ab4d97250e8694ce58e64bc4ca3ad3893bea6a9e814dd7fb10e88133ab3d370dcc0d7f67630537a527387df7b53a0afc40e8484b433bffce5868dedca77b3dbf637181c9d4c4adb78ec3abdb01649685f462f9bc134ff5941b26bc1deb014a54a271a32f8c4e9782d4d91ff6994a14e4b319913503a1aaf05235ac2cd3a6358e6a1db57ba7dfc302d90624d9295126ff61c9c93bc3a04bddd14e744350a2878aa24ab97cb64148db1323139646a5e20c7f50036f6ba7b3300292661e70b532f6f8ec4404e0ce79d94ca9e76dcd5761e601d3aac60f92af558d51f8b0d8cef5fa0fc5c89a291c47e3ff814f1fd51974f5752ec890cdaa3d14dfd0b2d6f0c16c5feea1c93708e6853ac7d4b00611d0b8e0faa321664c043042996c4a30dd7ec9ee3b52eda257f6b8dff5aace60740f28bfc0221b6df8afbbf9b3a238d29a5347c34a2b21e5fe33d81d91a6bc71fd07f3e84a01a15b2af67619a6558e5797dea859fbe92bf63879f37474b4d929740737fa7889c30051976e8ce12a26bb343d4f796d9d7ef45b84b51afb508be5f6a56fef4ba884513b574aa791cf0c959a4291b59de7b7756d12f69464acdb8934904397380d851c65aec521c249b47768cc85f155e1e2e742c0b959aabafaa6ab03928cee4292c64093bab6479c17e830cf3d3fee2e0cd2001a0fe50d0763aa1245a320bb65bf0b760de219170999a0e0843ff771ec78511696f9c6c27293c3e07fd04f83db567d34c7c7bbbd03e806f46fe4c199c17a9897fc584b15a791b530716deffda3180000bf3acc97a2eed0a015b4e58f529e3945bfef27b597dbc880d0784ae23bcf672d6aee3131014af77123675dc335454468d7f317560d23d24dfa5b1390d86792420b81a23c76c4ffefa62d81355e31460e572d5662b996efaa287bc9d8b11e37c5538668f811296f845cfb59ef0d7619ecc089e679fc9847c70f92d401a12195232438a97c23bf2b20ee13a305f1e9911dec8d42c7024c95a76f6edceb0f0c53c1675aea0a1dfeb5d42abea93e83f52d546ba6a9f265d2e09d2ec3a31bfaba4a5a3c5fb7668a8f7e23fafa6565a4496bc50e589ea264582d72366befed568431f4abf9bfd2463caf61a7c94cdc4c78179bc724708b4b2d6ba12d318269234c43fb99b8579bbce41d61b106f980470734774bfcd4a21c5fe6ef2329e593752d0ac7a96bf5a2e3b662dfcf9f66c9a0ff3af9767f72553b8bacf71e17ff211859aaba0de822d13662ac9e3f93274e18c3e4a64d2cb4a7671513f6eeeeeb3b13f0a96671649333594e1f29e7e55571b51f56928c9bb2fa0c5f2f759bd37bd449faf0a6fa645f1dca7d62ac4708d91b9491ff6c63030358381ed4490111894cb8ee4a08660c711f7a31d9629fe60afa8dd3f2cdefa53974c9138d85a8d72a41bd69e21dfe1f1c52d5f99fdf85efcf8e2edb65aeb34eef2884f9d08f4dd8afd85059707d445d5aeba45167d7a788f3042db4f1647c604998df8ecbb98fbbd323db5fc8fdf184d8bf1e8b5029db74e5b9141d6942cec3371ace5b0d272665cf94cdca2c8493a6cd786570adf755f9d332c06b856fd49ebba4b6cf28d01dcb5201469356c89dda87dd40654cb72cf5bba0459f1ebf9958c01901cde507604e2e7f85a1d18efdbaee72344e09de0d36eff6fc0287f4d14feb78d37760e1eb2e210c6692182dd9be2eeee32bd150d5a2653503e5bb89b28e5044ac9fc581833464fb4a52a4ddcf0234b0f93e7330602afee6fdab383759e7d5d538665159a265af980de87bde6af20001db41e51ca386a916fb7c2bf686d3bee0bd7b867e5f053bf4330f8d7bfef75a25983d5ffce285c01f889864d9bbfec641e56ccf6a10baea89173ccc955c8c6689ef5d9e2f6d0450a88797373e4e0e58747a991351d8ae4b528241a70b6c33b70640955d864b13b03238b461e5cf77e275d438b912890cd80fc0e71ed477dda53f70e34bd60574744b863569e08ec73cfe5688080179ddd5518e393ab6580fce579d795671e86b75db2cab874bc2feedfcc700de4d5e91830af75b5e748cd41b6d33cc43574bf1bbde97bfed9fca6969e82d2a91608c911517a11eb2cb902675fc041f72291c11683f64ab8de00db5557eb211a271671bd36c3e64233cf3c9645cc7d119cc965c8dc4f089d12f624fec44e777ffb6f11a94d900435b2b251fd15f66b33af69d84ded0daea86a27eec017ef3f49d5c1985941fa3b5458d0fc62f9a827504615862f9fb8c7bc8b882e589ccd99be657e0dc77106cd923eb017ddfdfc69abd8b7a3e84274e82d59b0548ffcb8d00719a143578c15b6b21385304d5605b173c7c5c39cccc822a3b799e0c92d7acea08d4a8640df9648409553f37472e8475e363960e033e6eea31f5658df847c2ad7f3aa928844076e55861c191a38ebb40b61702837be36d873b69203d729b1fd7c0aee27c2bbb631088b391caaa37e657cd07cf1c5abb0d731e85977e08fa0c9c2ab5595a828d85e40803b5c9c3d136ffe5f8e99c32d97b74ba41dc0734bca5a3752da3cfb703d0270102d1fdca6c05cbea7cae2401252cecd8d25a59948b8b68314f333c080af233c40a9ce985f5a5d408c11e7669232c49c514d3a6fedc8c20123122c24d5391effd446c2e015c3246e41265f43139cf4cc896115f1cab18d8aeb77abef6e18941cf6030f28c11ef2f886694456bebea59a0e8c9804af9d87923272a62f124e8768f7e2bbe2cb1baedab8730a4e41e83fa6f09e74f1fff78e0f84e6856db5d0dafcf59801a2e6fa337331b0c87b214d4b3050913b221f1039f0f208863cb074a7ed91ffe9929c1e7b78f8c8f1a313547d36bcb4ca1ec3aa8307c6683e2c5133c57f33ab3f00eddcee3aa7e945fca4fec091df3cc8c1bf4c9e4f962cff6a8dfd5a889c1ebd6b01c9dcec4dc6e0dacdd01f762283e5a685ffeef908913dd07445262d9cb69043d793741f8730ede9eb901447f7d96501a89a27cf56aadb4f6f8b5a42ad9b544a17479ab1ef4609ef323f8f625252f56ea24c476c1faf1da7db1f150b1dff6f13cee2187447bd0a520a1b93f738d705b3d06104306f28c4d0f9f690e8bc910b34f5795cab5ab4cdadf3d33766471f32af4d43ebab51adecc53cd4e417a1fe05568d841cf3912318c842593b1c08127df99798ae8016e60dd7e72c9f98b592ad147c128cc26bd7ba23c02d2be2a2940cb9253e4255e1668f1bdb02e4e2dfab00e18646c563aa2e76843bda94fdb07c919a427a2c03298467eaf0f22ac637bbef24cbc8de8f713271260cd9babe44f0015fd050b8daa7be497eaecfe743b0869950aaab32868215d79e04ecae97f057adbfe14c6a8aa3117ed77d8083aba08198ee90dbc3d0d818a501c5cc14ee90af041d3560bf3de7a08ff33a41bb110bdcb5ef0642551e0c9e2b882579acbd1b332acada35e757d7715c92b4b6f454db7c774610912e3fe6e72103e399af697b6e539f1bbcb02351ba8660a33ff44f67a5829fcbf8def0bb04df0c50cdaf5c417f19dccfe2ab648f3a11d5502189d6470efe6077ec3af83 \ No newline at end of file diff --git a/tuf/tmp/project/helloworld.py b/tuf/tmp/project/helloworld.py deleted file mode 100644 index c5b49102..00000000 --- a/tuf/tmp/project/helloworld.py +++ /dev/null @@ -1 +0,0 @@ -print hello world diff --git a/tuf/tmp/repository/config.cfg b/tuf/tmp/repository/config.cfg deleted file mode 100644 index 9045b651..00000000 --- a/tuf/tmp/repository/config.cfg +++ /dev/null @@ -1,23 +0,0 @@ -[expiration] -days = 364 -years = 0 -minutes = 0 -hours = 0 -seconds = 0 - -[release] -keyids = 9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9 -threshold = 1 - -[timestamp] -keyids = 76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22 -threshold = 1 - -[root] -keyids = c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b -threshold = 1 - -[targets] -keyids = 845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b -threshold = 1 - diff --git a/tuf/tmp/repository/metadata/release.txt b/tuf/tmp/repository/metadata/release.txt deleted file mode 100644 index 14623ee0..00000000 --- a/tuf/tmp/repository/metadata/release.txt +++ /dev/null @@ -1,58 +0,0 @@ -{ - "signatures": [ - { - "keyid": "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9", - "method": "evp", - "sig": "adb041550a26327056b17409c59c2294930bcee1dc88008a9b458d828da673e2da4ae3c40257dfa51a25cd2cd23189fd1753546fd441879f275e515b433919e0403478bc2a7b7d9e455283f742fe5d059097be55eb2d705123194f31b13cb7d2a96421e5b7fb09df2f0a5d4245676b71c4630fd20ee29f962b3d327eb3362cd5e2f104b3a036d9c305817df955e19c49f3878cf3e65915c8a542adfd057f62522c1eca75cba513c81adb14994152934ecb4de1fb707d1aca4cc0f2b5ecb09e6645cb6f27f0769c8aeeff7f5728a910af9d310737c17e6b1cd611b07d70ee80de1457b13f54102ec5c58fdcf75470fe4db41c18f93f18a92f9929b8a9693e6e96b6231fc63705f47e05e079259e1eff17234060870685868da555d0bb05546f26d77ff7f091c3bd1a3e77633f2f5505597f8126a2130cacaee9a119c2915b48a0b08ff2152495462119b6a4ca05d302629bb7f7da60346a8cdd12f2820a00af6d1f3debffaf5052c2d31afa9c3fce3f82dbd139fcd0cd5062bede2c77c5e19407" - } - ], - "signed": { - "_type": "Release", - "expires": "2014-08-01 16:19:08 UTC", - "meta": { - "root.txt": { - "hashes": { - "sha256": "2e496d43eb877fc725dd3bd616da0c1e018057982d3c8acf8906a4104680feb1" - }, - "length": 4793 - }, - "targets.txt": { - "hashes": { - "sha256": "c5cbeeaaa617fa9ecf282ed4ed3051ecba2d8f9535f148e14103c6d6ed6bfd39" - }, - "length": 2260 - }, - "targets/packages.txt": { - "hashes": { - "sha256": "324aff11e6488e3619f8a291dc94f82faf60ab00ce67443ba32997bd0d1ad0cb" - }, - "length": 2325 - }, - "targets/packages/A.txt": { - "hashes": { - "sha256": "0af576b49df40cc310ba314a82dc264202ec74d4238eb526d85230aacf9d2282" - }, - "length": 2123 - }, - "targets/packages/A/Alice.txt": { - "hashes": { - "sha256": "49d0adb568d9323161f987087894df88cc0eb45ad2e4b7972b017915899226af" - }, - "length": 1377 - }, - "targets/packages/B.txt": { - "hashes": { - "sha256": "e3618668fe88e9fa99cb305e24d8c78ed3083270bb3de8bbc42dbf4234f2e894" - }, - "length": 2119 - }, - "targets/packages/B/Bob.txt": { - "hashes": { - "sha256": "3fb4ac73a78e66408b6192e28aa3b02e8180793a64fd99e49842b4a4ad45b7e9" - }, - "length": 1369 - } - }, - "version": 2 - } -} diff --git a/tuf/tmp/repository/metadata/root.txt b/tuf/tmp/repository/metadata/root.txt deleted file mode 100644 index b7d8e066..00000000 --- a/tuf/tmp/repository/metadata/root.txt +++ /dev/null @@ -1,70 +0,0 @@ -{ - "signatures": [ - { - "keyid": "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b", - "method": "evp", - "sig": "83448b51098091a562d9074dc14e02af5a6f18fcc18ff0dd0d709d0d1a597fbe9c96d2a077acffdc85be0357f76c46b1d2cbb579ad376458a88b2e00330b4028361c337593c96b7c2eec10dc337c7652d9e83d7c2dca7e59230199d2e6e5d3f0f1a38f09d305b47954f552ecb45df5c247ffa60b3b15198bc17a17d9198e688289ec70e3043cad1c1ac9405ad81d1af5135cf961f1a45d60f08d00c4e1713722711bee4864655a495f6e105020702ffea947fe3288358dffa46fa8d9c4c47fb7b3fd8a47b2371a0ffa78bce885b2ddb36b75ca6f6fea807e236593b882a7b1fc6c0fe43eb4b6709b060e08fb3e6f5a56fea6a5524130d01b461f1d6c2e3b1c2ef8784190245568062ec888af1fee0740b81ca0c99b775396b421f507581257277ba1e8609528ffef0ef5c9c205b63874e2c37b5dabb738cf5597e0c39010b8041a87b00030f69217e41e03376d1899c25d0d66d9e936b9308709eecc9862273a91e42301f6254f501bc10806a8aced547667678c39790598579dcb79b064b074" - } - ], - "signed": { - "_type": "Root", - "expires": "2014-07-31 15:21:53 UTC", - "keys": { - "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAxyi2M6UtfeITuiu4kbWZ\nOU5CRScab9Z8eWy9Weff/IiSYD47uRhnseT84ls50yCDzaUMlrCSG4Kd+AdAfcVl\nmsFCnRC5HHacFF9EEPSDud9L+4LTgqM1fP4vWfv8ZZ1sXI/5Npeo/D3bcC9NmcfR\nBMCzaQl1NEoYK+QVMcqVVImoWeLEtmU52u96Y4FZv06FodhpHPVvKUmJFnXe/ikM\nLe5sTCP0QKLtv+q1bfZ84KBkE50QfDpTF1bqiDeuCtb/3yNb91gkpPspuWjn9icK\nFCc/D5FpYSkVnWCoPzYsuVcQzIPFV2Zr8L1gvSvPOPY7SJulk+jri2T1kZw/rbDe\nA9T/I1yG36/8IXv9JYA8yiMR4cblFqi00gj1Q3NrelBdYc27t8GDNP2D2H4wMw3P\nnTmmfJQ5WpR5ZbjVHy/OU5YCpeJ77sdBITaSVn5IWLYumko+vJohOoordMBfV/jZ\nTVVeaz0OCp9sPs46xKVN+m109/NNejS90gLIKPKDSfPrAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAuvReIoGpKvK0rwmkgccg\nmv8sTtEBolT02THaxO+6+gSPO/509msi/M3UNHsGNda1gccdwRMTsR/C9lCKC9Mp\nZ+JcOUl70SkL1lf4QlttWEDyDgDg1M7RPfcMZnq+eo/BPqGOxmbeNxKS2tqNfpQQ\nOqvSb7MMeII/mitunWo+UbJE/No9dimueFAjgHwQfH3rJCMnjfL5OT15UipHICCV\n/x6Rypc47kyOIfAvGMBnPT+sSapu+tboGPQf4kYwDpQVBPrJuGLYbgFvLAP/JKoi\ndHwXKeOAYP6FSioKcXbOspVd7a6FCUHQtEX8g158WOb28Ggo3eeXJx4Yb6ZW4WhD\no1jHNyELE2n1h42FETUNiESF3WDbfGZX8XINy+PXmVWGsY0YTdFOcSmw5k4fZ2Dp\nD57R7GQanMBBkpB/J5dVYvdP/NLTHeueBRFDJAFM+HRozbTpi+pFTcKV0dITLh43\nQrvSE7ZK2ktW1Zo5aJuhVClxga3teM6N4Hm3wEv1mfqBAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - }, - "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0BLhIsIORHPXuTU1CPSj\n34jQVNs3jXfpEfDIoYQcLPrNoz5icGWlg3H7YFtGSGOIOzJbyMlQR3iyGu7IYSW7\nuyJRX8OJ6rkbLcAg3z4QXZf6Y8isQYQGBiPb/j/LKGpGs6GA0wDqVfcCUJGtz04k\n5P4oPmJZjiQO2uMyQKYkJDWXllAgkY/SkuOUHyk/knE8EHIoNCwqCAVVnKc/gg/O\nL2I6mwkyes6eXQDRdwRK0z1P72ebzAgKdshhU8Jx4S1W3BTdX4CZ0TqBKxiNkhTi\nIWkZHM5hijX7NbCUNTIL4MUSDGga/quqs3kSMCM3lOd37MLiTDXHcCZoF09w6cgy\njsZyZZR7PBgpQjQ4EjgKstrAForp7ph7dF/BAP4Fz3uf9JBhdJ3LaIT/0et0BA/J\n52TxMT84ngzS+yWobqdrOK9xVaNOZTS0j3ScWpBKRCDR1E+llUJlkjphdD6Og8K2\ntFVKwCTYm3qfwHd2ulllVzbOOntTnq/ppcQjtTO2yYSVAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": { - "release": { - "keyids": [ - "9a0933bdae0a2c387f086ecd2d79d8e59d7bce66aa21bf1de9b5069c30369dc9" - ], - "threshold": 1 - }, - "root": { - "keyids": [ - "c05b95f47865866ebe22fcb76595b05fcc77a2bf145ca3d7302ce7868cbaed7b" - ], - "threshold": 1 - }, - "targets": { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "threshold": 1 - }, - "timestamp": { - "keyids": [ - "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22" - ], - "threshold": 1 - } - }, - "version": 1 - } -} diff --git a/tuf/tmp/repository/metadata/targets.txt b/tuf/tmp/repository/metadata/targets.txt deleted file mode 100644 index 7bdb5ae2..00000000 --- a/tuf/tmp/repository/metadata/targets.txt +++ /dev/null @@ -1,45 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "89a80bef74459020b690187315ff0b3ecae376c68a5305ca08d29151fc3f5047385da8c70d1965d9c47bda9dce4ab8a2c83a8c04792d097491555bc884a8a833e644c0d85b27338154b861c7f829221f3e0d3170b3414a7922ff37cbb5223a7dafd95e8eb5bc4b2bcdcbcb72533751ebe4a6adb441d4389d0f55ad9a68beac98442aac953c0a6e531f45f78891ad15c72e54dda57e673d60d9936278d60f89ababcbc811eda9ba770b1a5cb222ff4e15f18da323b01e49e03ffbdfea207047d2543baa458978fc14644716ce92b9d112e732538d14002d5db5aa7143ee6eddf463b6e96f9504f87b393e8c340bfb5f425c05af454bc67711daabd412e96a295563b9171d7623f08a87a449f8e594e66e68e49f302e639ad523ce1baebe458afe07136030b949c5ba8114f975bcf1462486cc115a50a27263270cb63c0bcbe9e4ebc8171d9453e279086309668ac2d538b665c64888b43806a5bb97207fd91a02f4634c723da81dff84225eec4439c0acdb893410e34fd62343108d7b7055b59e" - } - ], - "signed": { - "_type": "Targets", - "delegations": { - "keys": { - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": [ - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages", - "paths": [ - "packages/" - ], - "threshold": 1 - } - ] - }, - "expires": "2013-10-31 22:49:03 UTC", - "targets": { - "helloworld.py": { - "hashes": { - "sha256": "14d9f7904b16af5b3cd64285eb349bdce11dd3688d6e330ab7da87eb37512941" - }, - "length": 18 - } - }, - "version": 2 - } -} diff --git a/tuf/tmp/repository/metadata/targets/packages.txt b/tuf/tmp/repository/metadata/targets/packages.txt deleted file mode 100644 index 65a7e369..00000000 --- a/tuf/tmp/repository/metadata/targets/packages.txt +++ /dev/null @@ -1,48 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "028e6e6c93e3973bdbc295a5e96fb9d67926dc5a67b3127a77d3dbdd55e1b87bf82183e64f3e0ce133d0cf75b64a0c80f432c91c95bcd583af073473e991c7bc2a12cce0290b17232d82f010268eeb1192a242a14aa992b9b6036fff5dcf3fe5a2fcc0d15d9ca6aee54ca2a053779889968eac11c160fed1056b2ad0092f69deac9d286657b64a92f0b9182bdfee32930117b83baf729bd494b259d60a3ebd54c0a154ba87d710f9f8ab5ef6cfd563dffe346ef6bcb6551c5323f5c68839089a3ea65926c0fa159c43272d1323fd521b403dbe88d7213955e3c121328eb816db3521e059fc37b2e88741f517747344ce9b5520693061848b627077db692ab44afc1cab484270aa826339b1181862b461433b79d066cfb289fdd5f91b4e193bbdf5053d33e93b615e40ade38d7c74d8d3da8ae2df4fbaf4792a867cf08ba182666f465d0a0723058eebbec94b1c9e9f46560e05a45d58fdf98e5b5362077d63f00b0c8cf5ca00f60ef3ef5b2559a1c129eca3422a228aac3fa6882b368bfc233c" - } - ], - "signed": { - "_type": "Targets", - "delegations": { - "keys": { - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": [ - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages/A", - "paths": [ - "packages/A/" - ], - "threshold": 1 - }, - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages/B", - "paths": [ - "packages/B/" - ], - "threshold": 1 - } - ] - }, - "expires": "2014-08-01 15:37:45 UTC", - "targets": {}, - "version": 3 - } -} diff --git a/tuf/tmp/repository/metadata/targets/packages/A.txt b/tuf/tmp/repository/metadata/targets/packages/A.txt deleted file mode 100644 index 447153ce..00000000 --- a/tuf/tmp/repository/metadata/targets/packages/A.txt +++ /dev/null @@ -1,38 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "69be32d77d781bb48f0518dafade5fb0a166d9ad82340a3a20f7630165d69d8ab0f8e753fd490c4c8727968539a4285db94bf73317a83672a177576ebb8091ec8ed34334893a683dad990ddd2ef7f0b1c034ed581b11ff12a30d78e31bb3c16918464a91128b3151eafab427b316134e17106ebaaee9ab78d39673beb4d08fd5aeac506e485e9e71903886ec1adb9a69dd1855b98aec2e7d48e361ec5b92ea728d4d8ba3bb16e84dd36cbef88bfbb8ecb39d9e1b20544a678062af312447b302803592da00f68846d68f6c05dbb5e7419dca5b07e8d43aa5a9b1a3a0e8386c815c665160062c7b4760761d05c683ddf18e398816120cc7860574ac98b9fd3ef74018210b454b765bb4dfe45163b348f44fc1c804ae69fc1a13d7e71a03d0af724838ab959da6828e990e604cb563a00724e6b4deb7e8ca13275bbfc89185d1cd71d3fb2c11692b5785801632bd1e54d60d73b5817dd654217cdc0850df5527f04ae8ba053e9a040a7bd0de740629640354895bf6399ca9672f432d507b4ced82" - } - ], - "signed": { - "_type": "Targets", - "delegations": { - "keys": { - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": [ - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages/A/Alice", - "paths": [ - "packages/A/Alice/" - ], - "threshold": 1 - } - ] - }, - "expires": "2014-08-01 15:45:01 UTC", - "targets": {}, - "version": 2 - } -} diff --git a/tuf/tmp/repository/metadata/targets/packages/A/Alice.txt b/tuf/tmp/repository/metadata/targets/packages/A/Alice.txt deleted file mode 100644 index 0b0e365f..00000000 --- a/tuf/tmp/repository/metadata/targets/packages/A/Alice.txt +++ /dev/null @@ -1,28 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "b55953980cc35017cf6deb2681380880f14b216d261c18e45bf4ebecbda0c6cc07e49607dac81639037008f8cc328ecd7f2c5858d454861b3bfd3f2209c24067164edf0af14c5f4564c9244f7c423825c7e162df618159e3c376f1c6ed4fb56e97b3d7fd3da59724c706e6f86b1ccfae1896ed6f792b76517cb87be92fc6336f892191c2dc3f55511c15cd787157af26489f2e8fc011507dcae5f4f7b314fd1c7a97c7fc8d91559d92e8615bfde318acea99bef2c4906c92c0d6e97ce3ae27c6e7ad5a232809f05fda1f6f5241fea5dfe2b86a00a57859c3b5322ad22cd7ebb5d71c3b8014de5a866068e9eb77ff9d0bdc3599b0de18f0f6f1a3546f03989e02346dc81b36601eda373814401381bf97709a7545ef448c9d3eaf1f80fedf5a959042d700ba7ebd060c4348cac3452258823039d06871d90c5fbf22e2572abda908a1f9160856db4bcd5b152a35ef81dd977f13aabec7d4fa05499a5969e03841e088dd29239795d4a2927616e210200ce3dfa82a4c250775c33c18035e5aa62a" - } - ], - "signed": { - "_type": "Targets", - "expires": "2014-08-01 15:46:53 UTC", - "targets": { - "packages/A/Alice/alice-v1.0.tar.gz": { - "hashes": { - "sha256": "21ecfb59295a055b39003e6b9a63b4aa7b3724458a7c663b00ee6cf69fc09e68" - }, - "length": 15 - }, - "packages/A/Alice/alice-v2.0.tar.gz": { - "hashes": { - "sha256": "21ecfb59295a055b39003e6b9a63b4aa7b3724458a7c663b00ee6cf69fc09e68" - }, - "length": 15 - } - }, - "version": 1 - } -} diff --git a/tuf/tmp/repository/metadata/targets/packages/B.txt b/tuf/tmp/repository/metadata/targets/packages/B.txt deleted file mode 100644 index bcce3d16..00000000 --- a/tuf/tmp/repository/metadata/targets/packages/B.txt +++ /dev/null @@ -1,38 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "6be3f064c985d60a1ab0ede575b8f86ff55f4bbbe457d30e330ae53ef4377751793282639d914c66028620f7d0076f73ae7ebf816cf1c98d77e765f1c76d64bebe79fb759d7307146458e206725fee8c9371f755cb49a59bd06458a5f214797175dbd6a745c290f968c43d15a8bb993cac6b02860a9e6fc351a083e72ea63a3298f138db25cb0441946ef88f09a3e4b3dfdd88622f79f3d3bdbc9d1f280a160e0e5a6a80dab324a7764b7f8951a0fd75a071f2b0e71a0a2914559bf1c2c7272dba42e5d171d36323afe98bf1eb48773c6a6ea23af941eb79a707be607b3e6a096ad330b6009db637460139e6aef0891455aa8e7b6c8953a409841357a43dd6b93e66c308d96ab80822d37eee2cd24c928f46f353957764dfd3c01a19aadc8c8d61d7f335d3d45cb6c28efae840ce3cc9324afcd4501a254a2283b0beaf4b590cb498ba3615244acf3a64b09fc216601222de8abb22b7c9e042b63e840c52c0cc31482c46e3a31e21d3267676249f0f7468b754748dfc6d83190d05b735343b88" - } - ], - "signed": { - "_type": "Targets", - "delegations": { - "keys": { - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b": { - "keytype": "rsa", - "keyval": { - "private": "", - "public": "-----BEGIN PUBLIC KEY-----\nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA0HjJD27+MwJ6xCPS4TrT\ntehsJQZDz2W10/ZixTWI0x9k6fXwvd18wqGLwz+vS13ow6SrUHBYXwBO/5Pblpgu\nN2mE/CWPzQ59vaVvcT3yb8gKqrtqcaT4tYSumzJzBNocDTik2Tyx2g0VincnDPAa\nVnsrkk9zmh+f1WMffaT4w2cu82NRTCoVVB43bk1UyjHQD0+moDb9UL2lQk2pBdMZ\n+PaChhMtgzf1mlmOicnb/OZgJK5uJfq6umv5oO4Io0tzEF/6xnj+6CsQWwMhiQpK\nZHNsGO+f/WE/SFOXd9C+Ljnu9JsqWIPAYJG9x8PCNY4sZhv+cBYDwqTav/NmcwtZ\nfaw0RTuKQBwD8C2syS2LdU14H5koWY8H6+TQVQp0JnddzTsO2PRrbB/whQpYwN3L\nY9Q+H5/AurrlIjv/HehoPRHceeBrlRNiqF0OKraYQicI/pPcfH/UcHvp/GsGxTMd\nwEIEszVoDK579WtJtDvK1CZPISeJ7RDxspG2Shz6ID8vAgMBAAE=\n-----END PUBLIC KEY-----\n" - } - } - }, - "roles": [ - { - "keyids": [ - "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b" - ], - "name": "targets/packages/B/Bob", - "paths": [ - "packages/B/Bob/" - ], - "threshold": 1 - } - ] - }, - "expires": "2014-08-01 15:48:49 UTC", - "targets": {}, - "version": 2 - } -} diff --git a/tuf/tmp/repository/metadata/targets/packages/B/Bob.txt b/tuf/tmp/repository/metadata/targets/packages/B/Bob.txt deleted file mode 100644 index 80934a8c..00000000 --- a/tuf/tmp/repository/metadata/targets/packages/B/Bob.txt +++ /dev/null @@ -1,28 +0,0 @@ -{ - "signatures": [ - { - "keyid": "845dc45bcd742df57d984dbaa46a959623950923470897b6230ae859bbffad6b", - "method": "evp", - "sig": "a15f0ec1c526228eb47847309571e4b4c3ee69202727420ad3472beec9e1c8dd17143215e9a98b9150eb9aa710e053661a64fc8a21ae2c0860d41eb765f9c5571a922e7a969ec501283fd1e811f15921151a584abba9f7d96a5a80191616e0047c60529fab7bd6d8a8bf963d32981e875cbd1909295abf1aaf84b420ca24b934a78adadb41c5e58b31abc3162a0f4310acca29a582e80c65da54f2a48c585b8c5191311905c6885832f898f995ac1e6594be4cffd9a681fbb09fbac3fefde9fa9ebd44cbcffa183f1c9e7219e031c47cd02edc53dec61a4ada8e9c9e20a57956526fd963bbc22bba4bc3c9b3fd962e8d220e27ae83164348840cd1333817d3931387f6d1a2badb24b88013f50350481bfdd2a61f50af840a2f26d861c2526e92f9607aa13ffec8276c36fb473cad0ef6e431ce6091e4e07df2ba4e2233247466ca0f2c4f0245c72ae153b95b288fd15194cdb836887fa37c62e4ad0eeb4f48367be8662a84ffab639c734f6c5d0c4429042bb43aad3e978806489a3a12bb4c3e" - } - ], - "signed": { - "_type": "Targets", - "expires": "2014-08-01 15:49:51 UTC", - "targets": { - "packages/B/Bob/bob-v1.0.tar.gz": { - "hashes": { - "sha256": "fa0b862231b81ff78cb1f431531c819354519e32f351c9b2109534ac5e9b1f07" - }, - "length": 13 - }, - "packages/B/Bob/bob-v2.0.tar.gz": { - "hashes": { - "sha256": "fa0b862231b81ff78cb1f431531c819354519e32f351c9b2109534ac5e9b1f07" - }, - "length": 13 - } - }, - "version": 1 - } -} diff --git a/tuf/tmp/repository/metadata/timestamp.txt b/tuf/tmp/repository/metadata/timestamp.txt deleted file mode 100644 index 8cb6f19d..00000000 --- a/tuf/tmp/repository/metadata/timestamp.txt +++ /dev/null @@ -1,22 +0,0 @@ -{ - "signatures": [ - { - "keyid": "76caaafb5aabc062ea4b4efaf17a998acff6667678f60a72de9491f6df404a22", - "method": "evp", - "sig": "9d09a61312b76ab8ec6f16b1bd9800769c306898c92df7a05bdd3a7bfe30747b187dab13427853957c3b21c1dc93519c290564f7703f4b07529fc64ff0e75d91569471bc112ab07d6253475489a07971384bcab10150c69d10c91960b75f7aa04b2c230eecc299b0e174278146bf3ab070841a1ff7ff15c7ad056eb020c29785b3ff432e2029c7bb56cdc36933204ce52e7dd32d5a7b9349b3f00d0da65a3423dadb4d74cf9abb91c29533c6d09b81811b1d3aed04988af0795ffa62fa409ada0a92a72d1f04f89caead224aa734aeb664fdb62a04b6045e8d749e015a5c40df81e275472e681722ed7afcf2556295eeb2463fbb36b055cc8581c9d8457d66c67b400627ab42d4619e8d6f881c66dda74385de451a5cbb967821255f7b211a03e34dc836b42e73d1588ac2ed5395f3e1c0ff281c84a9298d895cbf1b748ad19705a3ee26151eb08bb51df0210acca69b6a358390b223201e588d9c750f60238c6144a264f07e0631305566c1d41a07432800e226caaba2481882ffdeb177c657" - } - ], - "signed": { - "_type": "Timestamp", - "expires": "2014-08-01 16:19:39 UTC", - "meta": { - "release.txt": { - "hashes": { - "sha256": "f22f2f12fad9069b2fb569d9e9273a95b015fe2d8f3937addfed3dc1e48d86de" - }, - "length": 2152 - } - }, - "version": 2 - } -} diff --git a/tuf/tmp/repository/targets/helloworld.py b/tuf/tmp/repository/targets/helloworld.py deleted file mode 100644 index c5b49102..00000000 --- a/tuf/tmp/repository/targets/helloworld.py +++ /dev/null @@ -1 +0,0 @@ -print hello world diff --git a/tuf/tmp/repository/targets/packages/A/Alice/alice-v1.0.tar.gz b/tuf/tmp/repository/targets/packages/A/Alice/alice-v1.0.tar.gz deleted file mode 100644 index 74ff1a79..00000000 --- a/tuf/tmp/repository/targets/packages/A/Alice/alice-v1.0.tar.gz +++ /dev/null @@ -1 +0,0 @@ -Alice was here diff --git a/tuf/tmp/repository/targets/packages/A/Alice/alice-v2.0.tar.gz b/tuf/tmp/repository/targets/packages/A/Alice/alice-v2.0.tar.gz deleted file mode 100644 index 74ff1a79..00000000 --- a/tuf/tmp/repository/targets/packages/A/Alice/alice-v2.0.tar.gz +++ /dev/null @@ -1 +0,0 @@ -Alice was here diff --git a/tuf/tmp/repository/targets/packages/B/Bob/bob-v1.0.tar.gz b/tuf/tmp/repository/targets/packages/B/Bob/bob-v1.0.tar.gz deleted file mode 100644 index 139fb28e..00000000 --- a/tuf/tmp/repository/targets/packages/B/Bob/bob-v1.0.tar.gz +++ /dev/null @@ -1 +0,0 @@ -Bob was here diff --git a/tuf/tmp/repository/targets/packages/B/Bob/bob-v2.0.tar.gz b/tuf/tmp/repository/targets/packages/B/Bob/bob-v2.0.tar.gz deleted file mode 100644 index 139fb28e..00000000 --- a/tuf/tmp/repository/targets/packages/B/Bob/bob-v2.0.tar.gz +++ /dev/null @@ -1 +0,0 @@ -Bob was here From fe84ae3fdb1b75bdeb44b75792c1af0d2ada7014 Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 27 Aug 2013 15:22:08 -0400 Subject: [PATCH 045/119] Undo a line in tuf.log. --- tuf/log.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tuf/log.py b/tuf/log.py index 8f3a54f0..51397d1f 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -95,7 +95,6 @@ # Set the logger and its settings. logger = logging.getLogger('tuf') logger.setLevel(_DEFAULT_LOG_LEVEL) -logger.addHandler(stream_handler) logger.addHandler(file_handler) # Silently ignore logger exceptions. From 2048a3eccbf59095d0acb58c0863f49c86f7f9e0 Mon Sep 17 00:00:00 2001 From: ttgump Date: Tue, 27 Aug 2013 17:02:42 -0400 Subject: [PATCH 046/119] Update fix for endless-data-attack test. --- tuf/download.py | 6 +++--- tuf/tests/system_tests/test_endless_data_attack.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tuf/download.py b/tuf/download.py index 561cbae0..861e2966 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -403,10 +403,10 @@ def download_url_to_tempfileobj(url, required_hashes=None, file_length, required_length) - # Does 'total_downloaded' match 'required_length'? - if total_downloaded != required_length: + # Does 'total_downloaded' matches 'required_length'? + if required_length is not None and total_downloaded != required_length: message = 'Total downloaded length '+str(total_downloaded)+ \ - ' bytes doesn\'t match required length '+str(required_length)+' bytes.' + ' bytes doesn\'t match required length '+str(required_length)+' bytes.' raise tuf.DownloadError(message) # We appear to have downloaded the correct amount. Check the hashes. diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 196e4af9..476a134b 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -44,6 +44,7 @@ import logging +logger = logging.getLogger('tuf.test_endless_data_attack') class EndlessDataAttack(Exception): pass From fe27f24eac1ce58c22b2f1176b960a07016e4842 Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 29 Aug 2013 09:28:03 -0400 Subject: [PATCH 047/119] Update rsa_key.py and keystore.py with final PyCrypto major changes rsa_key.py was modified to generate and verify RSASSA-PSS signatures instead of RSASSA-PKCS1-v1_5. Optional functions to read and save passphrase-protected PEM files also added to rsa_key.py. keystore.py was modified to generate encrypted .key files (similar scheme as before) with PyCrypto to support uniform encryption of varied key types. User passwords are no longer temporarily stored, but used to derive a symmetric key with PBKDF2. The derived key is then used with AES-256-Mode-CTR to generate the encrypted key data. Affected unit tests updated. --- tuf/formats.py | 3 + tuf/repo/keystore.py | 237 ++++++++++++++++++++++++++++--------- tuf/rsa_key.py | 227 ++++++++++++++++++++++++++++++----- tuf/tests/test_keystore.py | 59 ++++----- tuf/tests/test_rsa_key.py | 5 +- 5 files changed, 419 insertions(+), 112 deletions(-) diff --git a/tuf/formats.py b/tuf/formats.py index a9ac4db3..60692efd 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -138,6 +138,9 @@ # The minimum number of bits for an RSA key. Must be 2048 bits and greater. RSAKEYBITS_SCHEMA = SCHEMA.Integer(lo=2048) +# An RSA key in PEM format. +PEMRSA_SCHEMA = SCHEMA.AnyString() + # A string representing a password. PASSWORD_SCHEMA = SCHEMA.AnyString() diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index c8fe5afd..479bf8c2 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -36,25 +36,54 @@ import binascii import logging -import evpy.cipher +# Import PyCrypto's Key Derivation Function (KDF) module. 'keystore.py' +# needs this module to derive a secret key according to the Password-Based +# Key Derivation Function 2 specification. The derived key is used as the +# symmetric key to encrypt TUF key information. PyCrypto's implementation: +# Crypto.Protocol.KDF.PBKDF2(). PKCS#5 v2.0 PBKDF2 specification: +# http://tools.ietf.org/html/rfc2898#section-5.2 +import Crypto.Protocol.KDF + +# PyCrypto's AES implementation. AES is a symmetric key algorithm that +# operates on fixed block sizes of 128-bits. +# https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +import Crypto.Cipher.AES + +# 'Crypto.Random' is a cryptographically strong version of Python's standard +# "random" module. Random bits of data are needed for salts and +# initialization vectors suitable for the encryption algorithms used in +# 'keystore.py'. +import Crypto.Random +import Crypto.Util.Counter import tuf.rsa_key import tuf.util + # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.keystore') json = tuf.util.import_json() -# The delimeter symbol used to separate the different sections +# The delimiter symbol used to separate the different sections # of encrypted files (i.e., salt, IV, ciphertext, passphrase). -# This delimeter is arbitrarily chosen and should not occur in +# This delimiter is arbitrarily chosen and should not occur in # the hexadecimal representations of the fields it is separating. -_ENCRYPTION_DELIMETER = '@@@@' +_ENCRYPTION_DELIMITER = '@@@@' + +# AES key size. Default key size = 32 bytes = AES-256. +_AES_KEY_SIZE = 32 + +# Default salt size, in bytes. A 128-bit salt (i.e., a random sequence of data +# to protect against dictionary attacks) is generated for PBKDF2. +_SALT_SIZE = 16 + +# Default PBKDF2 passphrase iterations. +_PBKDF2_ITERATIONS = 90510 # A password is set for each key added to the keystore. # The passwords dict has the form: {keyid: 'password', ...} -_key_passwords = {} +_derived_keys = {} # The keystore database, which has the form: # {keyid: key, ...}. @@ -96,7 +125,7 @@ def add_rsakey(rsakey_dict, password, keyid=None): tuf.KeyAlreadyExistsError, if 'rsakey_dict' is found in the keystore. - The '_keystore' and '_key_passwords' dictionaries are modified. + The '_keystore' and '_derived_keys' dictionaries are modified. None. @@ -115,22 +144,29 @@ def add_rsakey(rsakey_dict, password, keyid=None): # If 'keyid' was passed as an argument, does it # have the correct format? - if keyid: + if keyid is not None: # Raise 'tuf.FormatError' if the check fails. tuf.formats.KEYID_SCHEMA.check_match(keyid) # Check if the keyid found in 'rsakey_dict' matches # the 'keyid' supplied as an argument. if keyid != rsakey_dict['keyid']: - raise tuf.Error('Incorrect keyid '+rsakey_dict['keyid']+' expected '+keyid) + message = 'Incorrect keyid: '+repr(rsakey_dict['keyid'])+'.'+\ + 'Expected: '+repr(keyid) + raise tuf.Error(message) # Check if the keyid belonging to 'rsakey_dict' is not already # available in the key database. keyid = rsakey_dict['keyid'] if keyid in _keystore: - raise tuf.KeyAlreadyExistsError('keyid: '+keyid+' already exists.') - - _key_passwords[keyid] = password + message = 'Keyid: '+repr(keyid)+' already exists.' + raise tuf.KeyAlreadyExistsError(message) + + # The _derived_keys dictionary does not store the user's password. A key + # derivation function is applied to 'password' prior to storing it in + # _derived_keys. + salt, derived_key= _generate_derived_key(password) + _derived_keys[keyid] = {'salt': salt, 'derived_key': derived_key} _keystore[keyid] = rsakey_dict @@ -161,7 +197,7 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): format. - The '_keystore' and '_key_passwords' dictionaries are modified. + The '_keystore' and '_derived_keys' dictionaries are modified. The key files found in 'directory_name' are read. @@ -303,7 +339,7 @@ def save_keystore_to_keyfiles(directory_name): continue # Encrypt 'key_metadata_format' and save it. - encrypted_key = _encrypt(json.dumps(key_metadata_format), _key_passwords[keyid]) + encrypted_key = _encrypt(json.dumps(key_metadata_format), _derived_keys[keyid]) file_object.write(encrypted_key) file_object.close() logger.info(repr(basefilename)+' saved.') @@ -334,7 +370,7 @@ def clear_keystore(): """ _keystore.clear() - _key_passwords.clear() + _derived_keys.clear() @@ -343,7 +379,7 @@ def clear_keystore(): def change_password(keyid, old_password, new_password): """ - Change the password for 'keyid'. + Change the password for 'keyid'. keyid: @@ -370,17 +406,35 @@ def change_password(keyid, old_password, new_password): """ - # Check if 'keyid' is the keystore. - if keyid not in _keystore or keyid not in _key_passwords: - raise tuf.UnknownKeyError(keyid+' not recognized.') + # Does 'keyid' have the correct format? + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.KEYID_SCHEMA.check_match(keyid) + + # Does 'old_password' and 'new_password' have the correct format? + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.PASSWORD_SCHEMA.check_match(old_password) + tuf.formats.PASSWORD_SCHEMA.check_match(new_password) + + # Check if 'keyid' is in the keystore. + if keyid not in _keystore or keyid not in _derived_keys: + message = repr(keyid)+' not recognized.' + raise tuf.UnknownKeyError(message) - # Check if the old password matches. - if _key_passwords[keyid] != old_password: - raise tuf.BadPasswordError('Old password invalid') - # If 'new_password' has the correct format, update '_key_passwords'. - if tuf.formats.PASSWORD_SCHEMA.matches(new_password): - _key_passwords[keyid] = new_password + # Check if the old password matches. The _derived_keys dictionary + # stores derived keys instead of user passwords, according to the + # key derivation function used by _generate_derived_key(). + salt = _derived_keys[keyid]['salt'] + junk, old_derived_key = _generate_derived_key(old_password, salt) + if _derived_keys[keyid]['derived_key'] != old_derived_key: + message = 'Old password invalid.' + raise tuf.BadPasswordError(message) + + # Update '_derived_keys[keyid]' with the new derived key and salt. + salt, new_derived_key = _generate_derived_key(new_password) + _derived_keys[keyid] = {} + _derived_keys[keyid]['salt'] = salt + _derived_keys[keyid]['derived_key'] = new_derived_key @@ -426,12 +480,48 @@ def get_key(keyid): -def _encrypt(key_data, password): +def _generate_derived_key(password, salt=None): """ - Encrypt 'key_data' using the Advanced Encryption Standard algorithm. - 'password' is treated as the symmetric key, strengthened using SHA512. - The key size is 192 bits and AES's mode of operation is set to CBC - (Cipher-Block Chaining). + Generate a derived key by feeding 'password' to the Password-Based Key + Derivation Function (PBKDF2). PyCrypto's PBKDF2 implementation is + currently used. + + """ + + if salt is None: + salt = Crypto.Random.new().read(_SALT_SIZE) + + + def pseudorandom_function(password, salt): + """ + PyCrypto's PBKDF2() expects a callable function for the optional + 'prf' argument. 'prf' is set to HMAC SHA1 by default. + """ + + return Crypto.Hash.HMAC.new(password, salt, Crypto.Hash.SHA256).digest() + + + # 'dkLen' is the desired key length. 'count' is the number of password + # iterations performed by PBKDF2. 'prf' is a pseudorandom function, which + # must be callable. + derived_key = Crypto.Protocol.KDF.PBKDF2(password, salt, + dkLen=_AES_KEY_SIZE, + count=_PBKDF2_ITERATIONS, + prf=pseudorandom_function) + + return salt, derived_key + + + + + +def _encrypt(key_data, derived_key_information): + """ + Encrypt 'key_data' using the Advanced Encryption Standard (AES-256) algorithm. + 'derived_key_information' should have been strengthened by PBKDF2. The key + size is 256 bits and AES's mode of operation is set to CTR (Counter Mode). + The HMAC of the ciphertext is generated to ensure the ciphertext has not been + modified. 'key_data' is the JSON string representation of the key. In the case of RSA keys, this format would be 'tuf.formats.RSAKEY_SCHEMA': @@ -439,50 +529,91 @@ def _encrypt(key_data, password): 'keyval': {'public': '-----BEGIN RSA PUBLIC KEY----- ...', 'private': '-----BEGIN RSA PRIVATE KEY----- ...'}} - tuf.CryptoError raised if the encryption fails. + 'derived_key_information' is a dictionary of the form: + {'salt': '...' + 'derived_key': '...'} + + 'tuf.CryptoError' raised if the encryption fails. """ - # Use AES192 to encrypt 'key_data'. + # Generate a random initialization vector (IV). The 'iv' is treated as the + # initial counter block to a stateful counter block function (i.e., + # PyCrypto's 'Crypto.Util.Counter'. The AES block cipher operates on 128-bit + # blocks, so generate a random 16-byte initialization block. + iv = Crypto.Random.new().read(16) + stateful_counter_128bit_blocks = Crypto.Util.Counter.new(128, + initial_value=long(iv.encode('hex'), 16)) + symmetric_key = derived_key_information['derived_key'] + aes_cipher = Crypto.Cipher.AES.new(symmetric_key, + Crypto.Cipher.AES.MODE_CTR, + counter=stateful_counter_128bit_blocks) + + # Use AES-256 to encrypt 'key_data'. try: - salt, iv, ciphertext = evpy.cipher.encrypt(key_data, password) - except evpy.cipher.CipherError: - raise tuf.CryptoError + ciphertext = aes_cipher.encrypt(key_data) + except: + message = 'The key data could not be encrypted.' + raise tuf.CryptoError(message) - # Return the salt, initialization vector, and ciphertext as a single string. - # These three values are delimited by '_ENCRYPTION_DELIMETER' to make - # extraction easier. This delimeter is arbitrarily chosen and should not + # Generate the hmac of the ciphertext to ensure it has not been modified. + salt = derived_key_information['salt'] + derived_key = derived_key_information['derived_key'] + hmac_object = Crypto.Hash.HMAC.new(derived_key, ciphertext, Crypto.Hash.SHA256) + hmac = hmac_object.hexdigest() + + # Return the hmac, initialization vector, and ciphertext as a single string. + # These three values are delimited by '_ENCRYPTION_DELIMITER' to make + # extraction easier. This delimiter is arbitrarily chosen and should not # occur in the hexadecimal representations of the fields it is separating. - return binascii.hexlify(salt) + _ENCRYPTION_DELIMETER + \ - binascii.hexlify(iv) + _ENCRYPTION_DELIMETER + \ + return binascii.hexlify(salt) + _ENCRYPTION_DELIMITER + \ + binascii.hexlify(hmac) + _ENCRYPTION_DELIMITER + \ + binascii.hexlify(iv) + _ENCRYPTION_DELIMITER + \ binascii.hexlify(ciphertext) -def _decrypt(key_data, password): +def _decrypt(file_contents, password): """ The corresponding decryption routine for _encrypt(). - tuf.CryptoError raised if the decryption fails. + 'tuf.CryptoError' raised if the decryption fails. """ - # Extract the salt, initialization vector, and ciphertext from 'key_data'. - # These three values are delimited by '_ENCRYPTION_DELIMETER'. - # This delimeter is arbitrarily chosen and should not occur in the + # Extract the salt, hmac, initialization vector, and ciphertext from + # 'file_contents'. These three values are delimited by '_ENCRYPTION_DELIMITER'. + # This delimiter is arbitrarily chosen and should not occur in the # hexadecimal representations of the fields it is separating. - salt, iv, ciphertext = key_data.split(_ENCRYPTION_DELIMETER) + salt, hmac, iv, ciphertext = file_contents.split(_ENCRYPTION_DELIMITER) + salt = binascii.unhexlify(salt) + hmac = binascii.unhexlify(hmac) + iv = binascii.unhexlify(iv) + ciphertext = binascii.unhexlify(ciphertext) - # The following decryption routine assumes 'key_data' was encrypted - # using AES192. + # Generate derived key from 'password'. + junk, derived_key = _generate_derived_key(password, salt) + + # Verify the hmac to ensure the ciphertext is valid or has not been altered. + generated_hmac_object = Crypto.Hash.HMAC.new(derived_key, ciphertext, + Crypto.Hash.SHA256) + generated_hmac = generated_hmac_object.hexdigest() + + if generated_hmac != hmac: + raise tuf.CryptoError('Decryption failed.') + + # The following decryption routine assumes 'ciphertext' was encrypted with + # AES-256. + stateful_counter_128bit_blocks = Crypto.Util.Counter.new(128, + initial_value=long(iv.encode('hex'), 16)) + aes_cipher = Crypto.Cipher.AES.new(derived_key, + Crypto.Cipher.AES.MODE_CTR, + counter=stateful_counter_128bit_blocks) try: - key_plaintext = evpy.cipher.decrypt(binascii.unhexlify(salt), - binascii.unhexlify(iv), - binascii.unhexlify(ciphertext), - password) - except evpy.cipher.CipherError: - raise tuf.CryptoError + key_plaintext = aes_cipher.decrypt(ciphertext) + except: + raise tuf.CryptoError('Decryption failed.') return key_plaintext diff --git a/tuf/rsa_key.py b/tuf/rsa_key.py index ac7b648e..634abf71 100755 --- a/tuf/rsa_key.py +++ b/tuf/rsa_key.py @@ -14,16 +14,17 @@ The goal of this module is to support public-key cryptography using the RSA algorithm. The RSA-related functions provided include generate(), - create_signature(), and verify_signature(). The 'PyCrypto' package - used by 'rsa_key.py' generates the actual RSA keys and the functions listed - above can be viewed as an easy-to-use public interface. Additional functions - contained here include create_in_metadata_format() and - create_from_metadata_format(). These last two functions produce or use RSA - keys compatible with the key structures listed in TUF Metadata files. - The generate() function returns a dictionary containing all the information - needed of RSA keys, such as public and private keys, keyIDs, and an iden- - fier. create_signature() and verify_signature() are supplemental functions - used for generating RSA signatures and verifying them. + create_signature(), and verify_signature(). The create_encrypted_pem() and + create_from_encrypted_pem() functions are optional, and may be used save a + generated RSA key to a file. The 'PyCrypto' package used by 'rsa_key.py' + generates the actual RSA keys and the functions listed above can be viewed + as an easy-to-use public interface. Additional functions contained here + include create_in_metadata_format() and create_from_metadata_format(). These + last two functions produce or use RSA keys compatible with the key structures + listed in TUF Metadata files. The generate() function returns a dictionary + containing all the information needed of RSA keys, such as public and private= + keys, keyIDs, and an idenfier. create_signature() and verify_signature() are + supplemental functions used for generating RSA signatures and verifying them. https://en.wikipedia.org/wiki/RSA_(algorithm) Key IDs are used as identifiers for keys (e.g., RSA key). They are the @@ -41,14 +42,14 @@ # Crypto.PublicKey (i.e., PyCrypto public-key cryptography) provides algorithms # such as Digital Signature Algorithm (DSA) and the ElGamal encryption system. -# 'Crypto.PublicKey' is needed here to generate, sign, and verify RSA keys. We -# import all public-key cryptography algorithms, even though we only need the -# RSA functions. +# 'Crypto.PublicKey.RSA' is needed here to generate, sign, and verify RSA keys. import Crypto.PublicKey.RSA -# PyCrypto requires 'Crypto.Hash' hash objects to generate PKCS#1 v1.5 -# signatures (i.e., Crypto.Signature.PKCS1_v1_5). -import Crypto.Signature.PKCS1_v1_5 +# PyCrypto requires 'Crypto.Hash' hash objects to generate PKCS#1 PSS +# signatures (i.e., Crypto.Signature.PKCS1_PSS). +# https://tools.ietf.org/html/rfc3447 +import Crypto.Hash.SHA256 +import Crypto.Signature.PKCS1_PSS import tuf @@ -123,7 +124,6 @@ def generate(bits=_DEFAULT_RSA_KEY_BITS): rsa_pubkey = rsa_key_object.publickey() public_key_pem = rsa_pubkey.exportKey(format='PEM') - # Generate the keyid for the RSA key. 'key_value' corresponds to the # 'keyval' entry of the 'RSAKEY_SCHEMA' dictionary. The private key # information is not included in the generation of the 'keyid' identifier. @@ -306,12 +306,15 @@ def create_signature(rsakey_dict, data): Return a signature dictionary of the form: {'keyid': keyid, - 'method': 'PyCrypto-PKCS#1 v1.5', + 'method': 'PyCrypto-PKCS#1 PPS', 'sig': sig}. The signing process will use the private key rsakey_dict['keyval']['private'] and 'data' to generate the signature. + RFC3447 - RSASSA-PSS + http://www.ietf.org/rfc/rfc3447.txt + rsakey_dict: A dictionary containing the RSA keys and other identifying information. @@ -334,7 +337,7 @@ def create_signature(rsakey_dict, data): 'rsakey_dict' object. - PyCrypto's 'Crypto.Signature.PKCS1_v1_5' called to perform the actual + PyCrypto's 'Crypto.Signature.PKCS1_PSS' called to perform the actual signing. @@ -350,12 +353,12 @@ def create_signature(rsakey_dict, data): tuf.formats.RSAKEY_SCHEMA.check_match(rsakey_dict) # Signing the 'data' object requires a private key. - # The 'PyCrypto-PKCS#1 v1.5' (i.e., PyCrypto module) signing method is the + # The 'PyCrypto-PKCS#1 PSS' (i.e., PyCrypto module) signing method is the # only method currently supported. signature = {} private_key = rsakey_dict['keyval']['private'] keyid = rsakey_dict['keyid'] - method = 'PyCrypto-PKCS#1 v1.5' + method = 'PyCrypto-PKCS#1 PSS' sig = None if private_key: @@ -363,8 +366,8 @@ def create_signature(rsakey_dict, data): try: rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key) sha256_object = Crypto.Hash.SHA256.new(data) - pkcs1_signer = Crypto.Signature.PKCS1_v1_5.new(rsa_key_object) - sig = pkcs1_signer.sign(sha256_object) + pkcs1_pss_signer = Crypto.Signature.PKCS1_PSS.new(rsa_key_object) + sig = pkcs1_pss_signer.sign(sha256_object) except (ValueError, IndexError, TypeError), e: message = 'An RSA signature could not be generated.' raise tuf.CryptoError(message) @@ -420,11 +423,11 @@ def verify_signature(rsakey_dict, signature, data): tuf.FormatError. Raised if either 'rsakey_dict' or 'signature' do not match their respective tuf.formats schema. - 'rsakey_dict' must conform to tuf.formats.RSAKEY_SCHEMA. - 'signature' must conform to tuf.formats.SIGNATURE_SCHEMA. + 'rsakey_dict' must conform to 'tuf.formats.RSAKEY_SCHEMA'. + 'signature' must conform to 'tuf.formats.SIGNATURE_SCHEMA'. - Crypto.Signature.PKCS1_v1_5.verify() called to do the actual verification. + Crypto.Signature.PKCS1_PSS.verify() called to do the actual verification. Boolean. True if the signature is valid, False otherwise. @@ -445,22 +448,22 @@ def verify_signature(rsakey_dict, signature, data): # (i.e., rsakey_dict['keyval']['public']), verify whether 'signature' # was produced by rsakey_dict's corresponding private key # rsakey_dict['keyval']['private']. Before returning the Boolean result, - # ensure 'PyCrypto-PKCS#1 v1.5' was used as the signing method. + # ensure 'PyCrypto-PKCS#1 PSS' was used as the signing method. method = signature['method'] sig = signature['sig'] public_key = rsakey_dict['keyval']['public'] valid_signature = False - if method == 'PyCrypto-PKCS#1 v1.5': + if method == 'PyCrypto-PKCS#1 PSS': try: rsa_key_object = Crypto.PublicKey.RSA.importKey(public_key) - pkcs1_verifier = Crypto.Signature.PKCS1_v1_5.new(rsa_key_object) + pkcs1_pss_verifier = Crypto.Signature.PKCS1_PSS.new(rsa_key_object) sha256_object = Crypto.Hash.SHA256.new(data) # The metadata stores signatures in hex. Unhexlify and verify the # signature. signature = binascii.unhexlify(sig) - valid_signature = pkcs1_verifier.verify(sha256_object, signature) + valid_signature = pkcs1_pss_verifier.verify(sha256_object, signature) except (ValueError, IndexError, TypeError), e: message = 'The RSA signature could not be verified.' raise tuf.CryptoError(message) @@ -468,3 +471,167 @@ def verify_signature(rsakey_dict, signature, data): raise tuf.UnknownMethodError(method) return valid_signature + + + + + +def create_encrypted_pem(rsakey_dict, passphrase): + """ + + Return a string in PEM format, where the private part of the RSA key is + encrypted. The private part of the RSA key is encrypted by the Triple + Data Encryption Algorithm (3DES) and Cipher-block chaining (CBC) for the + mode of operation. Password-Based Key Derivation Function 1 (PBKF1) + MD5 + is used to strengthen 'passphrase'. + + https://en.wikipedia.org/wiki/Triple_DES + https://en.wikipedia.org/wiki/PBKDF2 + + + rsakey_dict: + A dictionary containing the RSA keys and other identifying information. + 'rsakey_dict' has the form: + + {'keytype': 'rsa', + 'keyid': keyid, + 'keyval': {'public': '-----BEGIN RSA PUBLIC KEY----- ...', + 'private': '-----BEGIN RSA PRIVATE KEY----- ...'}} + + The public and private keys are in PEM format and stored as strings. + + passphrase: + The passphrase, or password, to encrypt the private part of the RSA + key. 'passphrase' is not used directly as the encryption key, but used + to derive a stronger encryption key. + + + TypeError, if a private key is not defined for 'rsakey_dict'. + + tuf.FormatError, if an incorrect format is found for the + 'rsakey_dict' object. + + + PyCrypto's 'Crypto.PublicKey.RSA.exportKey()' called to perform the actual + generation of the PEM-formatted output. + + + A string in PEM format, where the private RSA key is encrypted. + + """ + + # Does 'rsakey_dict' have the correct format? + # This check will ensure 'rsakey_dict' has the appropriate number + # of objects and object types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.RSAKEY_SCHEMA.check_match(rsakey_dict) + + # Does 'signature' have the correct format? + tuf.formats.PASSWORD_SCHEMA.check_match(passphrase) + + # Extract the private key from 'rsakey_dict', which is stored in PEM format + # and unencrypted. The extracted key will be imported and converted to + # PyCrypto's RSA key object (i.e., Crypto.PublicKey.RSA).Use PyCrypto's + # exportKey method, with a passphrase specified, to create the string. + # PyCrypto uses PBKDF1+MD5 to strengthen 'passphrase', and 3DES with CBC mode + # for encryption. + private_key = rsakey_dict['keyval']['private'] + try: + rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key) + rsakey_pem_encrypted = rsa_key_object.exportKey(format='PEM', + passphrase=passphrase) + except (ValueError, IndexError, TypeError), e: + message = 'An encrypted RSA key in PEM format could not be generated.' + raise tuf.CryptoError(message) + + return rsakey_pem_encrypted + + + + + +def create_from_encrypted_pem(encrypted_pem, passphrase): + """ + + Return an RSA key in 'tuf.formats.RSAKEY_SCHEMA' format, which has the + form: + {'keytype': 'rsa', + 'keyid': keyid, + 'keyval': {'public': '-----BEGIN RSA PUBLIC KEY----- ...', + 'private': '-----BEGIN RSA PRIVATE KEY----- ...'}} + + The RSAKEY_SCHEMA object is generated from a byte string in PEM format, + where the private part of the RSA key is encrypted. PyCrypto's importKey + method is used, where a passphrase is specified. PyCrypto uses PBKDF1+MD5 + to strengthen 'passphrase', and 3DES with CBC mode for encryption/decryption. + + + encrypted_pem: + A byte string in PEM format, where the private key is encrypted. It has + the form: + + '-----BEGIN RSA PRIVATE KEY-----\n + Proc-Type: 4,ENCRYPTED\nDEK-Info: DES-EDE3-CBC ...' + + passphrase: + The passphrase, or password, to decrypt the private part of the RSA + key. 'passphrase' is not directly used as the encryption key, instead + it is used to derive a stronger symmetric key. + + + TypeError, if a private key is not defined for 'rsakey_dict'. + + tuf.FormatError, if an incorrect format is found for the + 'rsakey_dict' object. + + + PyCrypto's 'Crypto.PublicKey.RSA.importKey()' called to perform the actual + conversion from an encrypted RSA private key. + + + A dictionary in 'tuf.formats.RSAKEY_SCHEMA' format. + + """ + + # Does 'encryped_pem' have the correct format? + # This check will ensure 'encrypted_pem' has the appropriate number + # of objects and object types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.PEMRSA_SCHEMA.check_match(encrypted_pem) + + # Does 'passphrase' have the correct format? + tuf.formats.PASSWORD_SCHEMA.check_match(passphrase) + + keytype = 'rsa' + rsakey_dict = {} + + try: + rsa_key_object = Crypto.PublicKey.RSA.importKey(encrypted_pem, passphrase) + except (ValueError, IndexError, TypeError), e: + message = 'An RSA key object could not be generated from the encrypted'+\ + 'PEM string.' + raise tuf.CryptoError(message) + + # Extract the public & private halves of the RSA key and generate their + # PEM-formatted representations. The dictionary returned contains the + # private and public RSA keys in PEM format, as strings. + private_key_pem = rsa_key_object.exportKey(format='PEM') + rsa_pubkey = rsa_key_object.publickey() + public_key_pem = rsa_pubkey.exportKey(format='PEM') + + # Generate the keyid for the RSA key. 'key_value' corresponds to the + # 'keyval' entry of the 'RSAKEY_SCHEMA' dictionary. The private key + # information is not included in the generation of the 'keyid' identifier. + key_value = {'public': public_key_pem, + 'private': ''} + keyid = _get_keyid(key_value) + + # Build the 'rsakey_dict' dictionary. Update 'key_value' with the RSA + # private key prior to adding 'key_value' to 'rsakey_dict'. + key_value['private'] = private_key_pem + + rsakey_dict['keytype'] = keytype + rsakey_dict['keyid'] = keyid + rsakey_dict['keyval'] = key_value + + return rsakey_dict diff --git a/tuf/tests/test_keystore.py b/tuf/tests/test_keystore.py index 5302d5f2..d00b812b 100755 --- a/tuf/tests/test_keystore.py +++ b/tuf/tests/test_keystore.py @@ -20,6 +20,8 @@ import shutil import os import logging +import Crypto.Random +import Crypto.Protocol.KDF import tuf import tuf.repo.keystore @@ -88,20 +90,20 @@ def tearDown(self): def test_clear_keystore(self): - # Populate KEYSTORE's internal databases '_keystore' and '_key_passwords'. + # Populate KEYSTORE's internal databases '_keystore' and '_derived_keys'. for i in range(3): KEYSTORE.add_rsakey(RSAKEYS[i], PASSWDS[i], RSAKEYS[i]['keyid']) # Verify KEYSTORE's internal databases ARE NOT EMPTY. self.assertTrue(len(KEYSTORE._keystore) > 0) - self.assertTrue(len(KEYSTORE._key_passwords) > 0) + self.assertTrue(len(KEYSTORE._derived_keys) > 0) # Clear KEYSTORE's internal databases. KEYSTORE.clear_keystore() # Verify KEYSTORE's internal databases ARE EMPTY. self.assertFalse(len(KEYSTORE._keystore) > 0) - self.assertFalse(len(KEYSTORE._key_passwords) > 0) + self.assertFalse(len(KEYSTORE._derived_keys) > 0) @@ -113,8 +115,7 @@ def test_add_rsakey(self): self.assertEqual(RSAKEYS[0], KEYSTORE._keystore[RSAKEYS[0]['keyid']], 'Adding an rsa key dict was unsuccessful.') - self.assertEqual(PASSWDS[0], - KEYSTORE._key_passwords[RSAKEYS[0]['keyid']], + self.assertTrue(len(KEYSTORE._derived_keys) == 1, 'Adding a password pertaining to \'_keyid\' was unsuccessful.') # Passing three arguments to the function, i.e. including the 'keyid'. @@ -124,8 +125,7 @@ def test_add_rsakey(self): KEYSTORE._keystore[RSAKEYS[1]['keyid']], 'Adding an rsa key dict was unsuccessful.') - self.assertEqual(PASSWDS[1], - KEYSTORE._key_passwords[RSAKEYS[1]['keyid']], + self.assertTrue(len(KEYSTORE._derived_keys) == 2, 'Adding a password pertaining to \'_keyid\' was unsuccessful.') # Passing a keyid that does not match the keyid in 'rsakey_dict'. @@ -149,7 +149,7 @@ def test_save_keystore_to_keyfiles(self): # Extract and store keyids in '_keyids' list. keyids = [] - # Populate KEYSTORE's internal databases '_keystore' and '_key_passwords'. + # Populate KEYSTORE's internal databases '_keystore' and '_derived_keys'. for i in range(3): KEYSTORE.add_rsakey(RSAKEYS[i], PASSWDS[i], RSAKEYS[i]['keyid']) keyids.append(RSAKEYS[i]['keyid']) @@ -167,7 +167,7 @@ def test_save_keystore_to_keyfiles(self): for keyid in keyids: key_file = os.path.join(_DIR, str(keyid)+'.key') # Checks if key file has been created. - self.assertTrue(os.path.exists(key_file), 'Key file does not exits.') + self.assertTrue(os.path.exists(key_file), 'Key file does not exist.') file_stats = os.stat(key_file) # Checks if key file is not empty. @@ -216,7 +216,7 @@ def test_load_keystore_from_keyfiles(self): # The keystore should not have loaded any keys. self.assertEqual(0, len(KEYSTORE._keystore)) - self.assertEqual(0, len(KEYSTORE._key_passwords)) + self.assertEqual(0, len(KEYSTORE._derived_keys)) # Passing nonexistent 'keyids'. # AS EXPECTED, THIS CALL SHOULDN'T RAISE ANY ERRORS. @@ -225,7 +225,7 @@ def test_load_keystore_from_keyfiles(self): # The keystore should not have loaded any keys. self.assertEqual(0, len(KEYSTORE._keystore)) - self.assertEqual(0, len(KEYSTORE._key_passwords)) + self.assertEqual(0, len(KEYSTORE._derived_keys)) # Passing an invalid 'directory_name' argument - an integer value. self.assertRaises(tuf.FormatError, KEYSTORE.load_keystore_from_keyfiles, @@ -254,6 +254,7 @@ def test_change_password(self): for i in range(2): KEYSTORE.add_rsakey(RSAKEYS[i], PASSWDS[i], RSAKEYS[i]['keyid']) + derived_key_0 = KEYSTORE._derived_keys[RSAKEYS[0]['keyid']] # Create a new password. new_passwd = 'changed_password' @@ -261,13 +262,12 @@ def test_change_password(self): KEYSTORE.change_password(RSAKEYS[0]['keyid'], PASSWDS[0], new_passwd) # Check if password was changed. - self.assertNotEqual(KEYSTORE._key_passwords[RSAKEYS[0]['keyid']], - PASSWDS[0]) - self.assertEqual(KEYSTORE._key_passwords[RSAKEYS[0]['keyid']], - new_passwd) + new_derived_key = KEYSTORE._derived_keys[RSAKEYS[0]['keyid']]['derived_key'] + self.assertNotEqual(new_derived_key, + derived_key_0['derived_key']) - # Passing an invalid keyid i.e. RSAKEY[2] that was not loaded into - # the '_keystore'. + # Passing an invalid keyid (i.e., RSAKEY[2] that was not loaded into + # the '_keystore'). self.assertRaises(tuf.UnknownKeyError, KEYSTORE.change_password, RSAKEYS[2]['keyid'], PASSWDS[1], new_passwd) @@ -295,26 +295,31 @@ def test_get_key(self): def test_internal_encrypt(self): - # Test for valid arguments to '_encrypt()' and a valid return type. - encrypted_key = KEYSTORE._encrypt(json.dumps(RSAKEYS[0]), PASSWDS[0]) + # Test for valid arguments to '_encrypt()' and a valid return type. + salt = Crypto.Random.new().read(16) + derived_key = Crypto.Protocol.KDF.PBKDF2(PASSWDS[0], salt) + derived_key_information = {'salt': salt, 'derived_key': derived_key} + encrypted_key = KEYSTORE._encrypt(json.dumps(RSAKEYS[0]), + derived_key_information) self.assertEqual(type(encrypted_key), str) - # Test for invalid arguments to _encrypt(). - self.assertRaises(tuf.CryptoError, KEYSTORE._encrypt, '', PASSWDS[0]) - self.assertRaises(tuf.CryptoError, KEYSTORE._encrypt, - json.dumps(RSAKEYS[0]), '') - def test_internal_decrypt(self): del RSAKEYS[0]['keyid'] tuf.formats.KEY_SCHEMA.check_match(RSAKEYS[0]) - + + salt = Crypto.Random.new().read(16) + salt, derived_key = tuf.repo.keystore._generate_derived_key(PASSWDS[0], salt) + derived_key_information = {'salt': salt, 'derived_key': derived_key} + # Getting a valid encrypted key using '_encrypt()'. - encrypted_key = KEYSTORE._encrypt(json.dumps(RSAKEYS[0]), PASSWDS[0]) + encrypted_key = KEYSTORE._encrypt(json.dumps(RSAKEYS[0]), + derived_key_information) # Decrypting and decoding (using json's loads()) an encrypted file. - tuf.util.load_json_string(KEYSTORE._decrypt(encrypted_key, PASSWDS[0])) + #tuf.util.load_json_string(KEYSTORE._decrypt(encrypted_key, PASSWDS[0])) + json.dumps(KEYSTORE._decrypt(encrypted_key, PASSWDS[0])) self.assertEqual(RSAKEYS[0], tuf.util.load_json_string( KEYSTORE._decrypt(encrypted_key, PASSWDS[0]))) diff --git a/tuf/tests/test_rsa_key.py b/tuf/tests/test_rsa_key.py index a7b8708d..09a38663 100755 --- a/tuf/tests/test_rsa_key.py +++ b/tuf/tests/test_rsa_key.py @@ -171,9 +171,10 @@ def test_verify_signature(self): self.assertFalse(verified, 'Returned \'True\' on an incorrect signature.') - # Modifying 'signature' to pass an incorrect method since only 'evp' + # Modifying 'signature' to pass an incorrect method since only + # 'PyCrypto-PKCS#1 PSS' # is accepted. - signature['method'] = 'not_evp' + signature['method'] = 'Biff' args = (rsakey_dict, signature, DATA) self.assertRaises(tuf.UnknownMethodError, RSA_KEY.verify_signature, *args) From 19fae3c9088eed77d05e8a44269a286946038beb Mon Sep 17 00:00:00 2001 From: vladdd Date: Fri, 30 Aug 2013 14:58:41 -0400 Subject: [PATCH 048/119] Review Zane's unit test fixes and resolve merge conflicts --- tuf/tests/aggregate_tests.py | 28 +++++++++++++++++++++------- tuf/tests/repository_setup.py | 4 +++- tuf/tests/test_signercli.py | 2 ++ tuf/tests/test_signerlib.py | 2 ++ tuf/tests/test_updater.py | 11 ++++++++--- 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/aggregate_tests.py index a8a56fda..620c54de 100755 --- a/tuf/tests/aggregate_tests.py +++ b/tuf/tests/aggregate_tests.py @@ -11,31 +11,45 @@ January 26, 2013 - August 2013. Modified previous behavior that explicitly imported individual + August 2013. + Modified previous behavior that explicitly imported individual unit tests. -Zane Fisher See LICENSE for licensing information. - Run all the unit tests from every .py file beginning with "test_" in 'tuf/tests'. - Use --random to run the tests in random order. + Run all the unit tests from every .py file beginning with "test_" in + 'tuf/tests'. Use --random to run the tests in random order. """ + import sys import unittest import glob import random + +# Generate a list of pathnames that match a pattern (i.e., that begin with +# 'test_' and end with '.py'. A shell-style wildcard is used with glob() to +# match desired filenames. All the tests matching the pattern will be loaded +# and run in a test suite. tests_list = glob.glob('test_*.py') -# Remove '.py' from each filename. -tests_list = [test[:-3] for test in tests_list] +# Remove '.py' from each filename to allow loadTestsFromNames() (called below) +# to properly load the file as a module. +tests_without_extension = [] +for test in tests_list: + test = test[:-3] + tests_without_extension.append(test) +# Provide command-line option to randomize the order in which the tests run. +# Randomization might catch errors with unit tests that do not properly clean +# up or restore monkey-patched modules. if '--random' in sys.argv: - random.shuffle(tests_list) + random.shuffle(tests_without_extension) -suite = unittest.TestLoader().loadTestsFromNames(tests_list) +suite = unittest.TestLoader().loadTestsFromNames(tests_without_extension) unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tuf/tests/repository_setup.py b/tuf/tests/repository_setup.py index 759704d1..50a8a4e1 100755 --- a/tuf/tests/repository_setup.py +++ b/tuf/tests/repository_setup.py @@ -38,7 +38,7 @@ def _init_role_keyids(): # Populating 'rsa_keystore' and 'rsa_passwords' dictionaries. - # We will need them in creating keystore directory. + # We will need them in creating the keystore directory and metadata files. unittest_toolbox.Modified_TestCase.bind_keys_to_roles() global role_keyids @@ -279,6 +279,8 @@ def create_repositories(): """ + # Ensure the keyids for the required roles are loaded. Role keyids are + # needed for the creation of metadata file and the keystore. _init_role_keyids() # Make a temporary general repository directory. diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index dd756d40..bb4b33ee 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -1554,12 +1554,14 @@ def _mock_get_keyids(junk): def setUpModule(): + # setUpModule() is called before any test cases run. # Populating 'rsa_keystore' and 'rsa_passwords' dictionaries. # We will need them when creating keystore directories. unittest_toolbox.Modified_TestCase.bind_keys_to_roles() def tearDownModule(): + # tearDownModule() is called after all the test cases have run. unittest_toolbox.Modified_TestCase.clear_toolbox() diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index 4610f4b2..38e2f6bf 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -972,10 +972,12 @@ def _get_signed_role_info(self, role, directory=None): def setUpModule(): + # setUpModule() is called before any test cases run. # Generate rsa keys and roles dictionary dictionaries. unit_tbox.bind_keys_to_roles() def tearDownModule(): + # tearDownModule() is called after all the test cases have run. unit_tbox.clear_toolbox() tuf.repo.keystore.clear_keystore() diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index b4db02b0..ba7e234c 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -112,9 +112,10 @@ class TestUpdater(unittest_toolbox.Modified_TestCase): @classmethod def setUpClass(cls): + # setUpClass() is called before tests in an individual class run. # Create repositories. 'repositories' is a tuple that looks like this: # (repository_dir, client_repository_dir, server_repository_dir), see - # repository_setup.py odule. + # 'repository_setup.py' module. cls.repositories = setup.create_repositories() # Save references to repository directories and metadata. @@ -128,9 +129,11 @@ def setUpClass(cls): # References to delegated metadata paths and directories. cls.delegated_dir1 = os.path.join(cls.server_meta_dir, 'targets') - cls.delegated_filepath1 = os.path.join(cls.delegated_dir1, 'delegated_role1.txt') + cls.delegated_filepath1 = os.path.join(cls.delegated_dir1, + 'delegated_role1.txt') cls.delegated_dir2 = os.path.join(cls.delegated_dir1, 'delegated_role1') - cls.delegated_filepath2 = os.path.join(cls.delegated_dir2, 'delegated_role2.txt') + cls.delegated_filepath2 = os.path.join(cls.delegated_dir2, + 'delegated_role2.txt') cls.targets_dir = os.path.join(cls.server_repo_dir, 'targets') # Client side references. @@ -1150,6 +1153,8 @@ def test_8_remove_obsolete_targets(self): def tearDownModule(): + # tearDownModule() is called after all the tests have run. + # http://docs.python.org/2/library/unittest.html#class-and-module-fixtures setup.remove_all_repositories(TestUpdater.repositories['main_repository']) unittest_toolbox.Modified_TestCase.clear_toolbox() From 7c7349a9c8249954bbdeec2d0cebbf77c7ce2960 Mon Sep 17 00:00:00 2001 From: vladdd Date: Tue, 3 Sep 2013 12:48:26 -0400 Subject: [PATCH 049/119] Add tests for create_encrypted_pem() and create_from_encrypted_pem() --- tuf/rsa_key.py | 11 +++---- tuf/tests/test_rsa_key.py | 69 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/tuf/rsa_key.py b/tuf/rsa_key.py index 634abf71..1b825640 100755 --- a/tuf/rsa_key.py +++ b/tuf/rsa_key.py @@ -502,17 +502,16 @@ def create_encrypted_pem(rsakey_dict, passphrase): passphrase: The passphrase, or password, to encrypt the private part of the RSA - key. 'passphrase' is not used directly as the encryption key, but used - to derive a stronger encryption key. + key. 'passphrase' is not used directly as the encryption key, a stronger + encryption key is derived from it. TypeError, if a private key is not defined for 'rsakey_dict'. - tuf.FormatError, if an incorrect format is found for the - 'rsakey_dict' object. + tuf.FormatError, if an incorrect format is found for 'rsakey_dict'. - PyCrypto's 'Crypto.PublicKey.RSA.exportKey()' called to perform the actual + PyCrypto's Crypto.PublicKey.RSA.exportKey() called to perform the actual generation of the PEM-formatted output. @@ -608,7 +607,7 @@ def create_from_encrypted_pem(encrypted_pem, passphrase): try: rsa_key_object = Crypto.PublicKey.RSA.importKey(encrypted_pem, passphrase) except (ValueError, IndexError, TypeError), e: - message = 'An RSA key object could not be generated from the encrypted'+\ + message = 'An RSA key object could not be generated from the encrypted '+\ 'PEM string.' raise tuf.CryptoError(message) diff --git a/tuf/tests/test_rsa_key.py b/tuf/tests/test_rsa_key.py index 09a38663..881ee1d7 100755 --- a/tuf/tests/test_rsa_key.py +++ b/tuf/tests/test_rsa_key.py @@ -183,6 +183,75 @@ def test_verify_signature(self): self.assertRaises(TypeError,RSA_KEY.verify_signature) + def test_create_encrypted_pem(self): + passphrase = 'pw' + + # Check format of 'rsakey_dict'. + self.assertEqual(None, tuf.formats.RSAKEY_SCHEMA.check_match(rsakey_dict), + FORMAT_ERROR_MSG) + + # Check format of 'passphrase'. + self.assertEqual(None, tuf.formats.PASSWORD_SCHEMA.check_match(passphrase), + FORMAT_ERROR_MSG) + + # Generate the encrypted PEM string of 'rsakey_dict'. + pem_rsakey = tuf.rsa_key.create_encrypted_pem(rsakey_dict, passphrase) + + # Check for invalid arguments. + self.assertRaises(tuf.FormatError, + tuf.rsa_key.create_encrypted_pem, 'Biff', passphrase) + self.assertRaises(tuf.FormatError, + tuf.rsa_key.create_encrypted_pem, rsakey_dict, ['pw']) + + + + def test_create_from_encrypted_pem(self): + passphrase = 'pw' + + # Check format of 'rsakey_dict'. + self.assertEqual(None, tuf.formats.RSAKEY_SCHEMA.check_match(rsakey_dict), + FORMAT_ERROR_MSG) + + # Check format of 'passphrase'. + self.assertEqual(None, tuf.formats.PASSWORD_SCHEMA.check_match(passphrase), + FORMAT_ERROR_MSG) + + # Generate the encrypted PEM string of 'rsakey_dict'. + pem_rsakey = tuf.rsa_key.create_encrypted_pem(rsakey_dict, passphrase) + + # Decrypt 'pem_rsakey' and verify the decrypted object is properly + # formatted. + decrypted_rsakey = tuf.rsa_key.create_from_encrypted_pem(pem_rsakey, + passphrase) + self.assertEqual(None, tuf.formats.RSAKEY_SCHEMA.check_match(decrypted_rsakey), + FORMAT_ERROR_MSG) + + # Does 'decrypted_rsakey' match the original 'rsakey_dict'. + self.assertEqual(rsakey_dict, decrypted_rsakey) + + # Attempt decryption of 'pem_rsakey' using an incorrect passphrase. + self.assertRaises(tuf.CryptoError, + tuf.rsa_key.create_from_encrypted_pem, pem_rsakey, + 'bad_pw') + # Check for non-encrypted PEM string. create_from_encrypted_pem()/PyCrypto + # returns a tuf.formats.RSAKEY_SCHEMA object if PEM formatted string is + # not actually encrypted but still a valid PEM string. + non_encrypted_private_key = rsakey_dict['keyval']['private'] + decrypted_non_encrypted = tuf.rsa_key.create_from_encrypted_pem( + non_encrypted_private_key, passphrase) + self.assertEqual(None, tuf.formats.RSAKEY_SCHEMA.check_match( + decrypted_non_encrypted), FORMAT_ERROR_MSG) + + # Check for invalid arguments. + self.assertRaises(tuf.FormatError, + tuf.rsa_key.create_from_encrypted_pem, 123, passphrase) + self.assertRaises(tuf.FormatError, + tuf.rsa_key.create_from_encrypted_pem, pem_rsakey, ['pw']) + self.assertRaises(tuf.CryptoError, + tuf.rsa_key.create_from_encrypted_pem, 'invalid_pem', + passphrase) + + # Run the unit tests. if __name__ == '__main__': From 7497502087440be1b62fe4ba85537684d5660746 Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 3 Sep 2013 18:11:00 -0400 Subject: [PATCH 050/119] Check out my cool code. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index bbbc3c85..e3ff9ded 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,6 @@ author_email='info@updateframework.com', url='https://www.updateframework.com', packages=[ - 'evpy', 'tuf', 'tuf.client', 'tuf.compatibility', From 9ff22ddd4e5d49f3d00d05a4a585cb1043fe7619 Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 3 Sep 2013 18:11:48 -0400 Subject: [PATCH 051/119] Revert "Check out my cool code." This reverts commit 7497502087440be1b62fe4ba85537684d5660746. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e3ff9ded..bbbc3c85 100755 --- a/setup.py +++ b/setup.py @@ -68,6 +68,7 @@ author_email='info@updateframework.com', url='https://www.updateframework.com', packages=[ + 'evpy', 'tuf', 'tuf.client', 'tuf.compatibility', From 16d971da0e5d8c98282cf73e645306504cca8fa0 Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 4 Sep 2013 16:48:38 -0400 Subject: [PATCH 052/119] Add some exceptions for more fine-grained exception handling. --- tuf/__init__.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index deac6f50..c3933ca6 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -53,6 +53,14 @@ class FormatError(Error): +class InvalidMetadataJSONError(FormatError): + """Indicate that some metadata file is not valid JSON.""" + pass + + + + + class UnsupportedAlgorithmError(Error): """Indicate an error while trying to identify a user-specified algorithm.""" pass @@ -93,6 +101,22 @@ class RepositoryError(Error): +class ForbiddenTargetError(RepositoryError): + """Indicate that a role signed for a target that it was not delegated to.""" + pass + + + + + +class ReplayError(RepositoryError): + """Indicate that some metadata has been replayed to the client.""" + pass + + + + + class ExpiredMetadataError(Error): """Indicate that a TUF Metadata file has expired.""" pass @@ -117,8 +141,8 @@ class CryptoError(Error): -class UnsupportedLibraryError(Error): - """Indicate that a supported library could not be located or imported.""" +class BadSignatureError(CryptoError): + """Indicate that some metadata file had a bad signature.""" pass @@ -133,6 +157,22 @@ class UnknownMethodError(CryptoError): +class UnsupportedLibraryError(Error): + """Indicate that a supported library could not be located or imported.""" + pass + + + + + +class DecompressionError(Error): + """Indicate that some error happened while decompressing a file.""" + pass + + + + + class DownloadError(Error): """Indicate an error occurred while attempting to download a file.""" pass @@ -141,6 +181,14 @@ class DownloadError(Error): +class DownloadLengthMismatchError(DownloadError): + """Indicate that a mismatch of lengths was seen while downloading a file.""" + pass + + + + + class SlowRetrievalError(DownloadError): """"Indicate that downloading a file took an unreasonably long time.""" @@ -182,4 +230,18 @@ class InvalidNameError(Error): +class UpdateError(Error): + """An updater will throw this exception in case it could not download a + metadata or target file. + + A dictionary of Exception instances indexed by every mirror URL will also be + provided.""" + + def __init__(self, mirror_errors): + # Dictionary of URL strings to Exception instances + self.mirror_errors = mirror_errors + + + + From da5b9e0999d3b4ceb79f881153196bbcbf5fbe56 Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 4 Sep 2013 17:29:06 -0400 Subject: [PATCH 053/119] Replace generic exception with a specific one. --- tuf/download.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tuf/download.py b/tuf/download.py index ed80c370..0321a72d 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -336,7 +336,7 @@ def _open_connection(url): URL string (e.g., 'http://...' or 'ftp://...' or 'file://...') - tuf.DownloadError + None. Opens a connection to a remote server. @@ -590,8 +590,8 @@ def _check_downloaded_length(total_downloaded, required_length, None. - tuf.DownloadError, if STRICT_REQUIRED_LENGTH is True and total_downloaded - is not equal required_length. + tuf.DownloadLengthMismatchError, if STRICT_REQUIRED_LENGTH is True and + total_downloaded is not equal required_length. None. @@ -612,7 +612,7 @@ def _check_downloaded_length(total_downloaded, required_length, if STRICT_REQUIRED_LENGTH: # This must be due to a programming error, and must never happen! logger.error(message) - raise tuf.DownloadError(message) + raise tuf.DownloadLengthMismatchError(message) else: # We specifically disabled strict checking of required length, but we # will log a warning anyway. This is useful when we wish to download the @@ -660,12 +660,11 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, 'url'. - tuf.DownloadError, if there was an error while downloading the file. + tuf.DownloadLengthMismatchError, if there was a mismatch of observed vs + expected lengths while downloading the file. tuf.FormatError, if any of the arguments are improperly formatted. - tuf.BadHashError, if the hashes don't match. - Any other unforeseen runtime exception. From 3db934e81a8949742d6ecf5e7f765e84cebcf979 Mon Sep 17 00:00:00 2001 From: zanefisher Date: Wed, 4 Sep 2013 20:12:51 -0400 Subject: [PATCH 054/119] Refactor client.updater, removing verification from download.py. --- tuf/client/updater.py | 262 ++++++++++++++++++++++++++---------------- tuf/download.py | 75 ++---------- 2 files changed, 174 insertions(+), 163 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 1ce51548..3b583e23 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -582,8 +582,7 @@ def refresh(self): # Use default but sane information for timestamp metadata, and do not # require strict checks on its required length. - self._update_metadata('timestamp', DEFAULT_TIMESTAMP_FILEINFO, - STRICT_REQUIRED_LENGTH=False) + self._update_metadata('timestamp', DEFAULT_TIMESTAMP_FILEINFO) self._update_metadata_if_changed('release', referenced_metadata='timestamp') @@ -601,8 +600,158 @@ def refresh(self): - def _update_metadata(self, metadata_role, fileinfo, compression=None, - STRICT_REQUIRED_LENGTH=True): +def _check_hashes(input_file, trusted_hashes=None): + """ + + A helper function that verifies multiple secure hashes of the downloaded + file. If any of these fail it raises an exception. This is to conform + with the TUF specs, which support clients with different hashing + algorithms. The 'hash.py' module is used to compute the hashes of the + 'input_file'. + + + input_file: + A file-like object. + + trusted_hashes: + A dictionary with hash-algorithm names as keys and hashes as dict values. + The hashes should be in the hexdigest format. + + + tuf.BadHashError, if the hashes don't match. + + + Hash digest object is created using the 'tuf.hash' module. + + + None. + + """ + + if trusted_hashes: + # Verify each trusted hash of 'trusted_hashes'. Raise exception if + # any of the hashes are incorrect and return if all are correct. + for algorithm, trusted_hash in trusted_hashes.items(): + digest_object = tuf.hash.digest(algorithm) + digest_object.update(input_file.read()) + computed_hash = digest_object.hexdigest() + if trusted_hash != computed_hash: + raise tuf.BadHashError('Hashes do not match! Expected '+ + trusted_hash+' got '+computed_hash) + else: + logger.info('The file\'s '+algorithm+' hash is correct: '+trusted_hash) + else: + logger.warn('No trusted hashes supplied to verify file at: '+ + str(input_file)) + + + + + + def get_target_file(self, target_filepath, file_length, file_hashes): + + def verify_target_file(self, target_file_object): + self._check_hashes(target_file_object, file_hashes) + + return self.__get_file(target_filepath, verify_target_file, 'target', + file_length, download_safely=True) + + + + + + def __verify_metadata_file(self, metadata_file_object, metadata_role, file_hashes): + # FIXME: Decompression should be handled elsewhere. + #if compression: + # metadata_file_object.decompress_temp_file_object(compression) + + self._check_hashes(metadata_file_object, file_hashes) + + # Read and load the downloaded file. + try: + metadata_signable = \ + tuf.util.load_json_string(metadata_file_object.read()) + except: + logger.exception('Invalid metadata from '+mirror_url+'.') + raise + else: + # Verify the signature on the downloaded metadata object. + try: + valid = tuf.sig.verify(metadata_signable, metadata_role) + except: + message = 'Unable to verify '+metadata_filename + logger.exception(message) + raise + else: + if valid: + logger.debug('Good signature on '+mirror_url+'.') + else: + raise tuf.BadSignatureError('Bad signature on '+mirror_url+'.') + + + + + + def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, file_length): + + def unsafely_verify_metadata_file(metadata_file_object): + self.__verify_metadata_file(metadata_file_object, metadata_role, None) + + return self.__get_file(metadata_filepath, unsafely_verify_metadata_file, + 'meta', file_length, download_safely=False) + + + def safely_get_metadata_file(self, metadata_role, metadata_filepath, file_length, file_hashes): + + def safely_verify_metadata_file(metadata_file_object): + self.__verify_metadata_file(metadata_file_object, metadata_role, file_hashes) + + return self.__get_file(metadata_filepath, _verify_metadata_file, + 'meta', file_length, download_safely=True) + + + + + + def __get_file(self, filepath, verify_file, reference_metadata, trusted_length, download_safely): + file_mirrors = tuf.mirrors.get_list_of_mirrors(reference_metadata, filepath, + self.mirrors) + # file_mirror (URL): error (Exception) + file_mirror_errors = {} + target_file_object = None + + for file_mirror in file_mirrors: + try: + if (download_safely): + target_file_object = tuf.download.safe_download(file_mirror, trusted_length) + else: + target_file_object = tuf.download.unsafe_download(file_mirror, trusted_length) + + except Exception, e: + # Remember the error from this mirror, and "reset" the target file. + logger.exception('Download failed from '+file_mirror+'.') + file_mirror_errors[file_mirror] = e + target_file_object = None + else: + try: + verify_file(file_object) + except Exception, e: + file_mirror_errors[file_mirror] = e + target_file_object = None + else: + break + + if target_file_object: + return target_file_object + else: + # TODO: wrap file_mirror_errors in an Exception + raise tuf.UpdateError(file_mirror_errors) + + + + + + def _update_metadata(self, metadata_role, fileinfo, compression=None): """ Download, verify, and 'install' the metadata belonging to 'metadata_role'. @@ -659,15 +808,15 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None, metadata_filename = metadata_filename + '.gz' # Reference to the 'get_list_of_mirrors' function. - get_mirrors = tuf.mirrors.get_list_of_mirrors + # get_mirrors = tuf.mirrors.get_list_of_mirrors # Reference to the 'download_url_to_tempfileobj' function. download_file = tuf.download.download_url_to_tempfileobj # Extract file length and file hashes. They will be passed as arguments # to 'download_file' function. - file_length=fileinfo['length'] - file_hashes=fileinfo['hashes'] + file_length = fileinfo['length'] + file_hashes = fileinfo['hashes'] # A dictionary to keep the error from every mirror that we try. mirror_errors = {} @@ -681,63 +830,15 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None, # 'tuf.formats.SIGNABLE_SCHEMA'. metadata_file_object = None metadata_signable = None - - for mirror_url in get_mirrors('meta', - metadata_filename.encode("utf-8"), - self.mirrors): - try: + if metadata_role == 'timestamp': metadata_file_object = \ - download_file(mirror_url, file_length, file_hashes, - STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH) - except: - logger.exception('Download failed from '+mirror_url+'.') - mirror_errors[mirror_url] = traceback.format_exc(1) - else: - # FIXME: mirror_errors for a mirror_url must not be overwritten! + self.unsafely_get_metadata_file(metadata_role, metadata_filename, file_length) + else: + metadata_file_object = \ + self.safely_get_metadata_file(metadata_role, metadata_filename, file_length, file_hashes) - # FIXME: Another point of failure which we should handle. - if compression: - metadata_file_object.decompress_temp_file_object(compression) - - # Read and load the downloaded file. - try: - metadata_signable = \ - tuf.util.load_json_string(metadata_file_object.read()) - except: - logger.exception('Invalid metadata from '+mirror_url+'.') - mirror_errors[mirror_url] = traceback.format_exc(1) - metadata_signable = None - else: - # Verify the signature on the downloaded metadata object. - try: - valid = tuf.sig.verify(metadata_signable, metadata_role) - except (tuf.UnknownRoleError, tuf.FormatError, tuf.Error), e: - # FIXME: Exception.message is deprecated in 2.6, and gone in 3.0, - # but this is a workaround for Unicode messages. We need a - # long-term solution with #61. - # http://bugs.python.org/issue2517 - message = 'Unable to verify '+metadata_filename+':'+\ - e.message.encode("utf-8") - logger.exception(message) - mirror_errors[mirror_url] = message - metadata_signable = None - else: - if valid: - logger.debug('Good signature on '+mirror_url+'.') - break - else: - message = 'Bad signature on '+mirror_url+'.' - logger.warn(message) - mirror_errors[mirror_url] = message - metadata_signable = None - - # Raise an exception if a valid metadata signable could not be downloaded - # from any of the mirrors. - if metadata_signable is None: - message = 'Unable to update '+str(metadata_filename)+\ - ' from all known mirrors: '+str(mirror_errors) - logger.error(message) - raise tuf.RepositoryError(message) + # Read and load the downloaded file. + metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) # Ensure the loaded 'metadata_signable' is properly formatted. try: @@ -1885,45 +1986,14 @@ def download_target(self, target, destination_directory): tuf.formats.TARGETFILE_SCHEMA.check_match(target) tuf.formats.PATH_SCHEMA.check_match(destination_directory) - # Reference to the 'get_list_of_mirrors' function. - get_mirrors = tuf.mirrors.get_list_of_mirrors - - # Reference to the 'download_url_to_tempfileobj' function. - download_file = tuf.download.download_url_to_tempfileobj - # Extract the target file information. target_filepath = target['filepath'] trusted_length = target['fileinfo']['length'] trusted_hashes = target['fileinfo']['hashes'] - # A dictionary to keep the error from every mirror that we try. - mirror_errors = {} - - # Wherein the downloaded target file will be stored. - target_file_object = None - - logger.info('Trying to download: '+str(target_filepath)) - - # Iterate through the repositority mirrors until we successfully - # download a target. - for mirror_url in get_mirrors('target', target_filepath, self.mirrors): - try: - target_file_object = download_file(mirror_url, trusted_length, - trusted_hashes) - break - except: - # Remember the error from this mirror, and "reset" the target file. - logger.exception('Download failed from '+mirror_url+'.') - mirror_errors[mirror_url] = traceback.format_exc(1) - target_file_object = None - continue - - # We have gone through all the mirrors. Did we get a target file object? - if target_file_object == None: - raise tuf.DownloadError('Failed to download target '+\ - str(target_filepath)+\ - ' from all known mirrors: '+\ - str(mirror_errors)) + # get_target_file checks every mirror and returns the first target + # that passes verification. + target_file_object = get_target_file(target_filepath, trusted_length, trusted_hashes) # We acquired a target file object from a mirror. Move the file into # place (i.e., locally to 'destination_directory'). diff --git a/tuf/download.py b/tuf/download.py index 0321a72d..5fe0bc3e 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -363,54 +363,6 @@ def _open_connection(url): -def _check_hashes(input_file, trusted_hashes=None): - """ - - A helper function that verifies multiple secure hashes of the downloaded - file. If any of these fail it raises an exception. This is to conform - with the TUF specs, which support clients with different hashing - algorithms. The 'hash.py' module is used to compute the hashes of the - 'input_file'. - - - input_file: - A file-like object. - - trusted_hashes: - A dictionary with hash-algorithm names as keys and hashes as dict values. - The hashes should be in the hexdigest format. - - - tuf.BadHashError, if the hashes don't match. - - - Hash digest object is created using the 'tuf.hash' module. - - - None. - - """ - - if trusted_hashes: - # Verify each trusted hash of 'trusted_hashes'. Raise exception if - # any of the hashes are incorrect and return if all are correct. - for algorithm, trusted_hash in trusted_hashes.items(): - digest_object = tuf.hash.digest(algorithm) - digest_object.update(input_file.read()) - computed_hash = digest_object.hexdigest() - if trusted_hash != computed_hash: - raise tuf.BadHashError('Hashes do not match! Expected '+ - trusted_hash+' got '+computed_hash) - else: - logger.info('The file\'s '+algorithm+' hash is correct: '+trusted_hash) - else: - logger.warn('No trusted hashes supplied to verify file at: '+ - str(input_file)) - - - - - def _download_fixed_amount_of_data(connection, temp_file, required_length): """ @@ -624,8 +576,7 @@ def _check_downloaded_length(total_downloaded, required_length, -def download_url_to_tempfileobj(url, required_length, required_hashes=None, - STRICT_REQUIRED_LENGTH=True): +def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): """ Given the url, hashes and length of the desired file, this function @@ -641,13 +592,6 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, required_length: An integer value representing the length of the file. - - required_hashes: - A dictionary, where the keys represent the hashing algorithm used to - hash the file and the dict values the hexdigest. - - For instance, a hash pair might look something like this: - {'md5': '37544f383be1fc1a32f42801c9c4b4d6'} STRICT_REQUIRED_LENGTH: A Boolean indicator used to signal whether we should perform strict @@ -678,13 +622,6 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, tuf.formats.URL_SCHEMA.check_match(url) tuf.formats.LENGTH_SCHEMA.check_match(required_length) - # FIXME: This function should only download files up to an expected length, - # and not check hashes; that is the job of the updater. - if required_hashes: - tuf.formats.HASHDICT_SCHEMA.check_match(required_hashes) - else: - logger.warn('Missing hashes for: '+str(url)) - # 'url.replace()' is for compatibility with Windows-based systems because # they might put back-slashes in place of forward-slashes. This converts it # to the common format. @@ -725,9 +662,6 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, _check_downloaded_length(total_downloaded, required_length, STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH) - # Finally, check the hashes expected of the file. - _check_hashes(temp_file, trusted_hashes=required_hashes) - except: # Close 'temp_file'; any written data is lost. temp_file.close_temp_file() @@ -744,6 +678,13 @@ def download_url_to_tempfileobj(url, required_length, required_hashes=None, socket.setdefaulttimeout(previous_socket_timeout) +def safe_download(url, required_length): + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True) + + +def unsafe_download(url, required_length): + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=False) + From 2d58aeee43f279931186557b5d03d5d2ead41ad9 Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 4 Sep 2013 23:45:08 -0400 Subject: [PATCH 055/119] Merge with updater/download refactoring from @zanefisher. Update download unit test to work after refactoring, but it is a little incomplete (in particular, the unsafe_download function needs more testing). --- tuf/client/updater.py | 131 +++++++++++++++++++------------------ tuf/download.py | 25 ++++--- tuf/tests/test_download.py | 67 +++++-------------- tuf/util.py | 11 +++- 4 files changed, 111 insertions(+), 123 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 3b583e23..7091ccea 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -105,7 +105,6 @@ import os import shutil import time -import traceback import tuf import tuf.conf @@ -600,35 +599,34 @@ def refresh(self): -def _check_hashes(input_file, trusted_hashes=None): - """ - - A helper function that verifies multiple secure hashes of the downloaded - file. If any of these fail it raises an exception. This is to conform - with the TUF specs, which support clients with different hashing - algorithms. The 'hash.py' module is used to compute the hashes of the - 'input_file'. + def __check_hashes(self, input_file, trusted_hashes): + """ + + A helper function that verifies multiple secure hashes of the downloaded + file. If any of these fail it raises an exception. This is to conform + with the TUF specs, which support clients with different hashing + algorithms. The 'hash.py' module is used to compute the hashes of the + 'input_file'. - - input_file: - A file-like object. - - trusted_hashes: - A dictionary with hash-algorithm names as keys and hashes as dict values. - The hashes should be in the hexdigest format. - - - tuf.BadHashError, if the hashes don't match. - - - Hash digest object is created using the 'tuf.hash' module. - - - None. + + input_file: + A file-like object. - """ + trusted_hashes: + A dictionary with hash-algorithm names as keys and hashes as dict values. + The hashes should be in the hexdigest format. + + + tuf.BadHashError, if the hashes don't match. + + + Hash digest object is created using the 'tuf.hash' module. + + + None. + + """ - if trusted_hashes: # Verify each trusted hash of 'trusted_hashes'. Raise exception if # any of the hashes are incorrect and return if all are correct. for algorithm, trusted_hash in trusted_hashes.items(): @@ -640,9 +638,6 @@ def _check_hashes(input_file, trusted_hashes=None): trusted_hash+' got '+computed_hash) else: logger.info('The file\'s '+algorithm+' hash is correct: '+trusted_hash) - else: - logger.warn('No trusted hashes supplied to verify file at: '+ - str(input_file)) @@ -651,21 +646,19 @@ def _check_hashes(input_file, trusted_hashes=None): def get_target_file(self, target_filepath, file_length, file_hashes): def verify_target_file(self, target_file_object): - self._check_hashes(target_file_object, file_hashes) + self.__check_hashes(target_file_object, file_hashes) return self.__get_file(target_filepath, verify_target_file, 'target', - file_length, download_safely=True) + file_length, download_safely=True, compression=None) - def __verify_metadata_file(self, metadata_file_object, metadata_role, file_hashes): - # FIXME: Decompression should be handled elsewhere. - #if compression: - # metadata_file_object.decompress_temp_file_object(compression) + def __verify_metadata_file(self, metadata_file_object, metadata_role, + file_hashes): - self._check_hashes(metadata_file_object, file_hashes) + self.__check_hashes(metadata_file_object, file_hashes) # Read and load the downloaded file. try: @@ -692,59 +685,72 @@ def __verify_metadata_file(self, metadata_file_object, metadata_role, file_hashe - def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, file_length): + def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, + file_length): def unsafely_verify_metadata_file(metadata_file_object): - self.__verify_metadata_file(metadata_file_object, metadata_role, None) + self.__verify_metadata_file(metadata_file_object, metadata_role, + file_hashes=None) return self.__get_file(metadata_filepath, unsafely_verify_metadata_file, - 'meta', file_length, download_safely=False) + 'meta', file_length, download_safely=False, + compression=None) - def safely_get_metadata_file(self, metadata_role, metadata_filepath, file_length, file_hashes): + + + + def safely_get_metadata_file(self, metadata_role, metadata_filepath, + file_length, file_hashes, compression): def safely_verify_metadata_file(metadata_file_object): - self.__verify_metadata_file(metadata_file_object, metadata_role, file_hashes) - + self.__verify_metadata_file(metadata_file_object, metadata_role, + file_hashes=file_hashes) + return self.__get_file(metadata_filepath, _verify_metadata_file, - 'meta', file_length, download_safely=True) + 'meta', file_length, download_safely=True, + compression=compression) - def __get_file(self, filepath, verify_file, reference_metadata, trusted_length, download_safely): - file_mirrors = tuf.mirrors.get_list_of_mirrors(reference_metadata, filepath, - self.mirrors) + def __get_file(self, filepath, verify_file, reference_metadata, + trusted_length, download_safely, compression): + file_mirrors = tuf.mirrors.get_list_of_mirrors(reference_metadata, + filepath, self.mirrors) # file_mirror (URL): error (Exception) file_mirror_errors = {} - target_file_object = None + file_object = None for file_mirror in file_mirrors: try: - if (download_safely): - target_file_object = tuf.download.safe_download(file_mirror, trusted_length) + if download_safely: + file_object = tuf.download.safe_download(file_mirror, trusted_length) else: - target_file_object = tuf.download.unsafe_download(file_mirror, trusted_length) + file_object = tuf.download.unsafe_download(file_mirror, + trusted_length) + + if compression: + file_object.decompress_temp_file_object(compression) except Exception, e: # Remember the error from this mirror, and "reset" the target file. logger.exception('Download failed from '+file_mirror+'.') file_mirror_errors[file_mirror] = e - target_file_object = None + file_object = None else: try: verify_file(file_object) except Exception, e: file_mirror_errors[file_mirror] = e - target_file_object = None + file_object = None else: break - if target_file_object: - return target_file_object + if file_object: + return file_object else: - # TODO: wrap file_mirror_errors in an Exception raise tuf.UpdateError(file_mirror_errors) @@ -807,12 +813,6 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): if compression == 'gzip': metadata_filename = metadata_filename + '.gz' - # Reference to the 'get_list_of_mirrors' function. - # get_mirrors = tuf.mirrors.get_list_of_mirrors - - # Reference to the 'download_url_to_tempfileobj' function. - download_file = tuf.download.download_url_to_tempfileobj - # Extract file length and file hashes. They will be passed as arguments # to 'download_file' function. file_length = fileinfo['length'] @@ -832,10 +832,13 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): metadata_signable = None if metadata_role == 'timestamp': metadata_file_object = \ - self.unsafely_get_metadata_file(metadata_role, metadata_filename, file_length) + self.unsafely_get_metadata_file(metadata_role, metadata_filename, + file_length) else: metadata_file_object = \ - self.safely_get_metadata_file(metadata_role, metadata_filename, file_length, file_hashes) + self.safely_get_metadata_file(metadata_role, metadata_filename, + file_length, file_hashes, + compression=compression) # Read and load the downloaded file. metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) diff --git a/tuf/download.py b/tuf/download.py index 5fe0bc3e..2771b460 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -18,7 +18,7 @@ supplied by the metadata of that file. The downloaded file is technically a file-like object that will automatically destroys itself once closed. Note that the file-like object, 'tuf.util.TempFile', is returned by the - 'download_url_to_tempfileobj()' function. + '_download_file()' function. """ @@ -576,6 +576,20 @@ def _check_downloaded_length(total_downloaded, required_length, +def safe_download(url, required_length): + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True) + + + + + +def unsafe_download(url, required_length): + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=False) + + + + + def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): """ @@ -676,14 +690,7 @@ def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): # Restore previously saved values or functions. httplib.HTTPConnection.response_class = previous_http_response_class socket.setdefaulttimeout(previous_socket_timeout) - - -def safe_download(url, required_length): - return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True) - - -def unsafe_download(url, required_length): - return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=False) + diff --git a/tuf/tests/test_download.py b/tuf/tests/test_download.py index c5a3e903..77d0d5fd 100755 --- a/tuf/tests/test_download.py +++ b/tuf/tests/test_download.py @@ -30,6 +30,7 @@ import subprocess import time import unittest +import urllib2 import tuf @@ -69,7 +70,7 @@ def setUp(self): # NOTE: Following error is raised if delay is not applied: # - time.sleep(.1) + time.sleep(1) # Computing hash of target file data. m = hashlib.md5() @@ -90,47 +91,24 @@ def tearDown(self): # Test: Normal case. def test_download_url_to_tempfileobj(self): - download_file = download.download_url_to_tempfileobj + download_file = download.safe_download - temp_fileobj = download_file(self.url, self.target_data_length, - required_hashes=self.target_hash) - self.assertEquals(self.target_data, temp_fileobj.read()) - self.assertEquals(self.target_data_length, len(temp_fileobj.read())) - temp_fileobj.close_temp_file() - - - # Test: Incorrect hashes. - def test_download_url_to_tempfileobj_and_hashes(self): - - download_file = download.download_url_to_tempfileobj - - # Test: Normal cases without supplying hash arguments. temp_fileobj = download_file(self.url, self.target_data_length) self.assertEquals(self.target_data, temp_fileobj.read()) self.assertEquals(self.target_data_length, len(temp_fileobj.read())) temp_fileobj.close_temp_file() - # What happens when we pass bad hashes to check the downloaded file? - self.assertRaises(tuf.BadHashError, - download_file, self.url, self.target_data_length, - required_hashes={'md5':self.random_string()}) - # Test: Incorrect lengths. def test_download_url_to_tempfileobj_and_lengths(self): - download_file = download.download_url_to_tempfileobj - # NOTE: We catch tuf.BadHashError here because the file, shorter by a byte, # would not match the expected hashes. We log a warning when we find that # the server-reported length of the file does not match our # required_length. We also see that STRICT_REQUIRED_LENGTH does not change # the outcome of the previous test. - for strict_required_length in (True, False): - self.assertRaises(tuf.BadHashError, - download_file, self.url, self.target_data_length - 1, - required_hashes=self.target_hash, - STRICT_REQUIRED_LENGTH=strict_required_length) + download.safe_download(self.url, self.target_data_length - 1) + download.unsafe_download(self.url, self.target_data_length - 1) # NOTE: We catch tuf.DownloadError here because the STRICT_REQUIRED_LENGTH, # which is True by default, mandates that we must download exactly what is @@ -140,14 +118,12 @@ def test_download_url_to_tempfileobj_and_lengths(self): str(self.target_data_length+1)+\ ' bytes. There is a difference of 1 bytes!' self.assertRaisesRegexp(tuf.DownloadError, exception_message, - download_file, self.url, self.target_data_length + 1, - required_hashes=self.target_hash) + download.safe_download, self.url, + self.target_data_length + 1) # NOTE: However, we do not catch a tuf.DownloadError here for the same test # as the previous one because we have disabled STRICT_REQUIRED_LENGTH. - temp_fileobj = download_file(self.url, self.target_data_length + 1, - required_hashes=self.target_hash, - STRICT_REQUIRED_LENGTH=False) + temp_fileobj = download.unsafe_download(self.url, self.target_data_length + 1) self.assertEquals(self.target_data, temp_fileobj.read()) self.assertEquals(self.target_data_length, len(temp_fileobj.read())) temp_fileobj.close_temp_file() @@ -155,17 +131,14 @@ def test_download_url_to_tempfileobj_and_lengths(self): def test_download_url_to_tempfileobj_and_performance(self): - download_file = download.download_url_to_tempfileobj - """ # Measuring performance of 'auto_flush = False' vs. 'auto_flush = True' - # in download_url_to_tempfileobj() during write. No change was observed. + # in download._download_file() during write. No change was observed. star_cpu = time.clock() star_real = time.time() temp_fileobj = download_file(self.url, - self.target_data_length, - required_hashes=self.target_hash) + self.target_data_length) end_cpu = time.clock() end_real = time.time() @@ -184,28 +157,24 @@ def test_download_url_to_tempfileobj_and_performance(self): # Test: Incorrect/Unreachable URLs. def test_download_url_to_tempfileobj_and_urls(self): - download_file = download.download_url_to_tempfileobj + download_file = download.safe_download self.assertRaises(tuf.FormatError, - download_file, None, self.target_data_length, - required_hashes=self.target_hash) + download_file, None, self.target_data_length) - self.assertRaises(tuf.DownloadError, + self.assertRaises(ValueError, download_file, - self.random_string(), self.target_data_length, - required_hashes=self.target_hash) + self.random_string(), self.target_data_length) - self.assertRaises(tuf.DownloadError, + self.assertRaises(urllib2.HTTPError, download_file, 'http://localhost:'+str(self.PORT)+'/'+self.random_string(), - self.target_data_length, - required_hashes=self.target_hash) + self.target_data_length) - self.assertRaises(tuf.DownloadError, + self.assertRaises(urllib2.URLError, download_file, 'http://localhost:'+str(self.PORT+1)+'/'+self.random_string(), - self.target_data_length, - required_hashes=self.target_hash) + self.target_data_length) # Run unit test. diff --git a/tuf/util.py b/tuf/util.py index 3c72023a..aea3b4f6 100755 --- a/tuf/util.py +++ b/tuf/util.py @@ -249,6 +249,8 @@ def decompress_temp_file_object(self, compression): tuf.Error: If an invalid compression is given. + tuf.DecompressionError: If the compression failed for any reason. + 'self._orig_file' is used to store the original data of 'temporary_file'. @@ -266,10 +268,17 @@ def decompress_temp_file_object(self, compression): if compression != 'gzip': raise tuf.Error('Only gzip compression is supported.') + self.seek(0) self._compression = compression self._orig_file = self.temporary_file - self.temporary_file = gzip.GzipFile(fileobj=self.temporary_file, mode='rb') + + try: + self.temporary_file = gzip.GzipFile(fileobj=self.temporary_file, mode='rb') + except: + raise tuf.DecompressionError(self.temporary_file) + + From 9acc102977d1230637a6a755f87bcb9e56b2eea7 Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 5 Sep 2013 00:23:23 -0400 Subject: [PATCH 056/119] Adapt the updater unit test against latest changes. --- tuf/client/updater.py | 3 ++- tuf/download.py | 4 ++-- tuf/tests/test_updater.py | 46 +++++++++++++++++++-------------------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 7091ccea..ac648720 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1996,7 +1996,8 @@ def download_target(self, target, destination_directory): # get_target_file checks every mirror and returns the first target # that passes verification. - target_file_object = get_target_file(target_filepath, trusted_length, trusted_hashes) + target_file_object = self.get_target_file(target_filepath, trusted_length, + trusted_hashes) # We acquired a target file object from a mirror. Move the file into # place (i.e., locally to 'destination_directory'). diff --git a/tuf/download.py b/tuf/download.py index 2771b460..d3cbb1ee 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -111,8 +111,8 @@ def read(self, size): """ - # We should never try to specify a nonpositive size. - assert size > 0 + # We should never try to specify a negative size. + assert size >= 0 # Use max, disallow tiny reads in a loop as they are very inefficient. # We never leave read() with any leftover data from a new recv() call diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index 917d2aee..acd536c7 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -213,7 +213,7 @@ def _mock_download_url_to_tempfileobj(self, output): """ - def _mock_download(url, length, hashes=None, STRICT_REQUIRED_LENGTH=True): + def _mock_download(url, length): if isinstance(output, (str, unicode)): file_path = output elif isinstance(output, list): @@ -223,8 +223,8 @@ def _mock_download(url, length, hashes=None, STRICT_REQUIRED_LENGTH=True): temp_fileobj.write(file_obj.read()) return temp_fileobj - # Patch tuf.download.download_url_to_tempfileobj(). - tuf.download.download_url_to_tempfileobj = _mock_download + # Patch tuf.download.safe_download(). + tuf.download.safe_download = _mock_download @@ -490,7 +490,7 @@ def test_3__update_metadata(self): """ # Setup - original_download = tuf.download.download_url_to_tempfileobj + original_download = tuf.download.safe_download # Since client's '.../metadata/current' will need to have separate # gzipped metadata file in order to test compressed file handling, @@ -554,7 +554,7 @@ def test_3__update_metadata(self): self._remove_target_from_targets_dir(added_target_1) # RESTORE - tuf.download.download_url_to_tempfileobj = original_download + tuf.download.safe_download = original_download @@ -616,7 +616,7 @@ def test_3__update_metadata_if_changed(self): """ # Setup - original_download = tuf.download.download_url_to_tempfileobj + original_download = tuf.download.safe_download # To test updater._update_metadata_if_changed, 'targets' metadata file is # going to be modified at the server's repository. @@ -707,7 +707,7 @@ def test_3__update_metadata_if_changed(self): self._remove_target_from_targets_dir(added_target_1) # RESTORE - tuf.download.download_url_to_tempfileobj = original_download + tuf.download.safe_download = original_download @@ -766,7 +766,7 @@ def test_2__ensure_not_expired(self): def test_4_refresh(self): # Setup. - original_download = tuf.download.download_url_to_tempfileobj + original_download = tuf.download.safe_download # This unit test is based on adding an extra target file to the # server and rebuilding all server-side metadata. When 'refresh' @@ -799,7 +799,7 @@ def test_4_refresh(self): setup.build_server_repository(self.server_repo_dir, self.targets_dir) # RESTORE - tuf.download.download_url_to_tempfileobj = original_download + tuf.download.safe_download = original_download @@ -807,7 +807,7 @@ def test_4_refresh(self): def test_4__refresh_targets_metadata(self): # Setup - original_download = tuf.download.download_url_to_tempfileobj + original_download = tuf.download.safe_download # To test this method a target file would be added to a delegated role, # and metadata on the server side would be rebuilt. @@ -864,7 +864,7 @@ def test_4__refresh_targets_metadata(self): setup.build_server_repository(self.server_repo_dir, self.targets_dir) # RESTORE - tuf.download.download_url_to_tempfileobj = original_download + tuf.download.safe_download = original_download @@ -894,10 +894,10 @@ def test_3__targets_of_role(self): def test_5_all_targets(self): # Setup - original_download = tuf.download.download_url_to_tempfileobj + original_download = tuf.download.safe_download # As with '_refresh_targets_metadata()', tuf.roledb._roledb_dict - # has to be populated. The 'tuf.download.download_url_to_tempfileobj' method + # has to be populated. The 'tuf.download.safe_download' method # should be patched. The 'self.all_role_paths' argument is passed so that # the top-level roles and delegations may be all "downloaded" when # Repository.refresh() is called below. '_mock_download_url_to_tempfileobj' @@ -925,7 +925,7 @@ def test_5_all_targets(self): self.assertTrue(len(all_targets) is 6) # RESTORE - tuf.download.download_url_to_tempfileobj = original_download + tuf.download.safe_download = original_download @@ -954,7 +954,7 @@ def test_5_targets_of_role(self): def test_6_target(self): # Requirements: make sure roledb_dict is populated and - # tuf.download.download_url_to_tempfileobj function is patched. + # tuf.download.safe_download function is patched. # Setup targets_dir_content = os.listdir(self.targets_dir) @@ -985,9 +985,9 @@ def test_6_target(self): def test_6_download_target(self): # Setup: - original_download = tuf.download.download_url_to_tempfileobj + original_download = tuf.download.safe_download - # 'tuf.download.download_url_to_tempfileobj' method should be patched. + # 'tuf.download.safe_download' method should be patched. target_rel_paths_src = self._get_list_of_target_paths(self.targets_dir) # Create temporary directory that will be passed as an argument to the @@ -1032,7 +1032,7 @@ def test_6_download_target(self): mirrors[mirror_name]['confined_target_dirs'] = [''] # RESTORE - tuf.download.download_url_to_tempfileobj = original_download + tuf.download.safe_download = original_download @@ -1040,11 +1040,11 @@ def test_6_download_target(self): def test_7_updated_targets(self): # Setup: - original_download = tuf.download.download_url_to_tempfileobj + original_download = tuf.download.safe_download # In this test, client will have two target files. Server will modify # one of them. As with 'all_targets' function, tuf.roledb._roledb_dict - # has to be populated. 'tuf.download.download_url_to_tempfileobj' method + # has to be populated. 'tuf.download.safe_download' method # should be patched. target_rel_paths_src = self._get_list_of_target_paths(self.targets_dir) @@ -1103,7 +1103,7 @@ def test_7_updated_targets(self): self.fail(msg) # RESTORE - tuf.download.download_url_to_tempfileobj = original_download + tuf.download.safe_download = original_download @@ -1111,7 +1111,7 @@ def test_7_updated_targets(self): def test_8_remove_obsolete_targets(self): # Setup: - original_download = tuf.download.download_url_to_tempfileobj + original_download = tuf.download.safe_download # This unit test should be last, because it removes target files from the # server's targets directory. It is done to avoid adding files, rebuilding @@ -1162,7 +1162,7 @@ def test_8_remove_obsolete_targets(self): self.assertTrue(os.listdir(dest_dir), 2) # RESTORE - tuf.download.download_url_to_tempfileobj = original_download + tuf.download.safe_download = original_download def tearDownModule(): From 1eb806460cd4df45bde5e1d3ae0f115ce02edc3e Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 5 Sep 2013 11:50:15 -0400 Subject: [PATCH 057/119] Continue updating unit tests affected by PyCrypto changes --- tuf/repo/keystore.py | 2 +- tuf/repo/signercli.py | 6 +++--- tuf/tests/repository_setup.py | 12 ++++++------ tuf/tests/test_signercli.py | 2 +- tuf/tests/unittest_toolbox.py | 21 +++++++++++++++------ 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index 479bf8c2..68019de8 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -165,7 +165,7 @@ def add_rsakey(rsakey_dict, password, keyid=None): # The _derived_keys dictionary does not store the user's password. A key # derivation function is applied to 'password' prior to storing it in # _derived_keys. - salt, derived_key= _generate_derived_key(password) + salt, derived_key = _generate_derived_key(password) _derived_keys[keyid] = {'salt': salt, 'derived_key': derived_key} _keystore[keyid] = rsakey_dict diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index a8d9c825..48182e33 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -316,12 +316,12 @@ def _get_all_config_keyids(config_filepath, keystore_directory): loaded_keyids[key].append(keyid) break if keyid not in loaded_keyids[key]: - raise tuf.Error('Could not load a required top-level role key') + raise tuf.Error('Could not load a required top-level role key.') # Ensure we loaded keys for the required top-level roles. for key in ['root', 'targets', 'release', 'timestamp']: if key not in loaded_keyids: - message = 'The configuration file did not contain the required roles' + message = 'The configuration file did not contain the required roles.' raise tuf.Error(message) return loaded_keyids @@ -366,7 +366,7 @@ def _get_role_config_keyids(config_filepath, keystore_directory, role): # Ensure we loaded all the keyids. for keyid in value['keyids']: if keyid not in role_keyids: - raise tuf.Error('Could not load a required role key') + raise tuf.Error('Could not load a required role key.') if not role_keyids: raise tuf.Error('Could not load the required keys for '+role) diff --git a/tuf/tests/repository_setup.py b/tuf/tests/repository_setup.py index 50a8a4e1..066ca61e 100755 --- a/tuf/tests/repository_setup.py +++ b/tuf/tests/repository_setup.py @@ -56,14 +56,14 @@ def _create_keystore(keystore_directory): """ _rsa_keystore = unittest_toolbox.Modified_TestCase.rsa_keystore - _rsa_passwords = unittest_toolbox.Modified_TestCase.rsa_passwords - if not _rsa_keystore or not _rsa_passwords: + _rsa_derived_keys = unittest_toolbox.Modified_TestCase.rsa_derived_keys + if not _rsa_keystore or not _rsa_derived_keys: msg = 'Populate \'rsa_keystore\' and \'rsa_passwords\''+\ ' before invoking this method.' sys.exit(msg) keystore._keystore = _rsa_keystore - keystore._key_passwords = _rsa_passwords + keystore._derived_keys = _rsa_derived_keys keystore.save_keystore_to_keyfiles(keystore_directory) @@ -195,7 +195,7 @@ def _mock_get_keyids(junk): # Clear kestore's dictionaries, by detaching them from unittest_toolbox's # dictionaries. keystore._keystore = {} - keystore._key_passwords = {} + keystore._derived_keys = {} # Make first level delegation. signercli.make_delegation(keystore_dir) @@ -215,7 +215,7 @@ def _mock_get_keyids(junk): keystore._keystore = unittest_toolbox.Modified_TestCase.rsa_keystore - keystore._key_passwords = unittest_toolbox.Modified_TestCase.rsa_passwords + keystore._derived_keys = unittest_toolbox.Modified_TestCase.rsa_passwords # Build release file. signerlib.build_release_file(role_keyids['release'], server_metadata_dir, @@ -226,7 +226,7 @@ def _mock_get_keyids(junk): version, expiration_date+' UTC') keystore._keystore = {} - keystore._key_passwords = {} + keystore._derived_keys = {} # RESTORE signercli._get_metadata_directory = original_get_metadata diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index bb4b33ee..765035e0 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -479,7 +479,7 @@ def test_2__get_role_config_keyids(self): # TESTS for role in self.role_list: # Test: normal cases. - keystore.clear_keystore() + #keystore.clear_keystore() signercli._get_role_config_keyids(config_filepath, keystore_dir, role) # Test: incorrect passwords. diff --git a/tuf/tests/unittest_toolbox.py b/tuf/tests/unittest_toolbox.py index 673c8ceb..e7ca2894 100755 --- a/tuf/tests/unittest_toolbox.py +++ b/tuf/tests/unittest_toolbox.py @@ -114,9 +114,13 @@ def setUp(): # {keyid : {-- rsa key --}, ...} rsa_keystore = {} - # 'rsa_passwords' stores passwords for all created rsa keys. + # 'rsa_passwords' stores the passwords for all created rsa keys. rsa_passwords = {} + # 'derived_keys' stores the salt and derived keys (e.g., PBKDF2) for the + # RSA keys. + rsa_derived_keys = {} + # 'semi_roledict' because it lacks an item that a fully pledged # ROLEDICT_SCHEMA dictionary would have i.e. 'path' key is absent. semi_roledict = {} @@ -380,8 +384,13 @@ def generate_rsakey(): rsakey = rsa_key.generate() keyid = rsakey['keyid'] Modified_TestCase.rsa_keyids.append(keyid) - Modified_TestCase.rsa_passwords[keyid] = Modified_TestCase.random_string() + password = Modified_TestCase.random_string() + Modified_TestCase.rsa_passwords[keyid] = password + salt, derived_key = keystore._generate_derived_key(password) + Modified_TestCase.rsa_derived_keys[keyid] = {'salt': salt, + 'derived_key': derived_key} Modified_TestCase.rsa_keystore[keyid] = rsakey + return keyid @@ -390,19 +399,18 @@ def generate_rsakey(): def create_temp_keystore_directory(self, keystore_dicts=False): - if not self.rsa_keystore or not self.rsa_passwords: + if not self.rsa_keystore or not self.rsa_derived_keys: msg = 'Populate \'rsa_keystore\' and \'rsa_passwords\''+\ ' before invoking this method.' sys.exit(msg) temp_keystore_directory = self.make_temp_directory() keystore._keystore = self.rsa_keystore - keystore._key_passwords = self.rsa_passwords + keystore._derived_keys = self.rsa_derived_keys keystore.save_keystore_to_keyfiles(temp_keystore_directory) if not keystore_dicts: keystore._keystore={} - keystore._key_passwords={} - #keystore.clear_keystore() + keystore._derived_keys={} return temp_keystore_directory @@ -487,5 +495,6 @@ def clear_toolbox(): Modified_TestCase.rsa_keyids = [] Modified_TestCase.rsa_keystore.clear() Modified_TestCase.rsa_passwords.clear() + Modified_TestCase.rsa_derived_keys.clear() Modified_TestCase.semi_roledict.clear() Modified_TestCase.top_level_role_info.clear() From 168bd001302e3676b32f270e6ae01cbdf4c7daa9 Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 5 Sep 2013 15:44:29 -0400 Subject: [PATCH 058/119] WIP on refactoring the updater and downloader. --- tuf/client/updater.py | 51 ++++++++++++++++--------------------------- tuf/download.py | 2 +- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index ac648720..fc3b1a0e 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -353,10 +353,6 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): not end in '.txt'. Examples: 'root', 'targets', 'targets/linux/x86'. - tuf.RepositoryError: - If the metadata could not be loaded or the extracted data is not a - valid metadata object. - tuf.FormatError: If role information belonging to a delegated role of 'metadata_role' is improperly formatted. @@ -391,11 +387,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # 'tuf.formats.SIGNABLE_SCHEMA'. metadata_signable = tuf.util.load_json_file(metadata_filepath) - # Ensure the loaded json object is properly formatted. - try: - tuf.formats.check_signable_object_format(metadata_signable) - except tuf.FormatError, e: - raise tuf.RepositoryError('Invalid format: '+repr(metadata_filepath)+'.') + tuf.formats.check_signable_object_format(metadata_signable) # Extract the 'signed' role object from 'metadata_signable'. metadata_object = metadata_signable['signed'] @@ -551,7 +543,7 @@ def refresh(self): None. - tuf.RepositoryError: + tuf.UpdateError: If the metadata for any of the top-level roles cannot be updated. tuf.ExpiredMetadataError: @@ -577,7 +569,7 @@ def refresh(self): # Update the top-level metadata. The _update_metadata_if_changed() and # _update_metadata() calls below do NOT perform an update if there # is insufficient trusted signatures for the specified metadata. - # Raise 'tuf.RepositoryError' if an update fails. + # Raise 'tuf.UpdateError' if an update fails. # Use default but sane information for timestamp metadata, and do not # require strict checks on its required length. @@ -645,7 +637,7 @@ def __check_hashes(self, input_file, trusted_hashes): def get_target_file(self, target_filepath, file_length, file_hashes): - def verify_target_file(self, target_file_object): + def verify_target_file(target_file_object): self.__check_hashes(target_file_object, file_hashes) return self.__get_file(target_filepath, verify_target_file, 'target', @@ -655,11 +647,7 @@ def verify_target_file(self, target_file_object): - def __verify_metadata_file(self, metadata_file_object, metadata_role, - file_hashes): - - self.__check_hashes(metadata_file_object, file_hashes) - + def __verify_metadata_file(self, metadata_file_object, metadata_role): # Read and load the downloaded file. try: metadata_signable = \ @@ -676,10 +664,8 @@ def __verify_metadata_file(self, metadata_file_object, metadata_role, logger.exception(message) raise else: - if valid: - logger.debug('Good signature on '+mirror_url+'.') - else: - raise tuf.BadSignatureError('Bad signature on '+mirror_url+'.') + if not valid: + raise tuf.BadSignatureError() @@ -689,8 +675,7 @@ def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, file_length): def unsafely_verify_metadata_file(metadata_file_object): - self.__verify_metadata_file(metadata_file_object, metadata_role, - file_hashes=None) + self.__verify_metadata_file(metadata_file_object, metadata_role) return self.__get_file(metadata_filepath, unsafely_verify_metadata_file, 'meta', file_length, download_safely=False, @@ -704,10 +689,10 @@ def safely_get_metadata_file(self, metadata_role, metadata_filepath, file_length, file_hashes, compression): def safely_verify_metadata_file(metadata_file_object): - self.__verify_metadata_file(metadata_file_object, metadata_role, - file_hashes=file_hashes) + self.__check_hashes(metadata_file_object, file_hashes) + self.__verify_metadata_file(metadata_file_object, metadata_role) - return self.__get_file(metadata_filepath, _verify_metadata_file, + return self.__get_file(metadata_filepath, safely_verify_metadata_file, 'meta', file_length, download_safely=True, compression=compression) @@ -751,6 +736,8 @@ def __get_file(self, filepath, verify_file, reference_metadata, if file_object: return file_object else: + logger.exception('Failed to download {0}: {1}'.format(filepath, + file_mirror_errors)) raise tuf.UpdateError(file_mirror_errors) @@ -790,7 +777,7 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): are considered. Any other string is ignored. - tuf.RepositoryError: + tuf.UpdateError: The metadata could not be updated. This is not specific to a single failure but rather indicates that all possible ways to update the metadata have been tried and failed. @@ -860,7 +847,7 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): current_version = current_metadata_role['version'] downloaded_version = metadata_signable['signed']['version'] if downloaded_version < current_version: - message = repr(mirror_url)+' is older than the version currently '+\ + message = str(current_metadata_role)+' is older than the version currently '+\ 'installed.\nDownloaded version: '+repr(downloaded_version)+'\n'+\ 'Current version: '+repr(current_version) raise tuf.RepositoryError(message) @@ -1013,7 +1000,7 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas try: self._update_metadata(metadata_role, fileinfo=new_fileinfo, compression=compression) - except tuf.RepositoryError, e: + except: # The current metadata we have is not current but we couldn't # get new metadata. We shouldn't use the old metadata anymore. # This will get rid of in-memory knowledge of the role and @@ -1023,8 +1010,8 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas # We shouldn't need to, but we need to check the trust # implications of the current implementation. self._delete_metadata(metadata_role) - message = 'Metadata for '+repr(metadata_role)+' could not be updated: ' - raise tuf.MetadataNotAvailableError(message+str(e)) + logger.error('Metadata for '+str(metadata_role)+' could not be updated') + raise else: # We need to remove delegated roles because the delegated roles # may not be trusted anymore. @@ -1970,7 +1957,7 @@ def download_target(self, target, destination_directory): tuf.FormatError: If 'target' is not properly formatted. - tuf.DownloadError: + tuf.UpdateError: If a target could not be downloaded from any of the mirrors. diff --git a/tuf/download.py b/tuf/download.py index d3cbb1ee..0aec7e2f 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -177,7 +177,7 @@ def read(self, size): # We assume that 'size' is accurate w.r.t. to the overall file length; # otherwise, we will miscount the number of truly slow chunks. self.__number_of_slow_chunks += 1 - logger.warn('slow chunk {0}'.format(self.__number_of_slow_chunks)) + logger.warn('slow chunk {0}: {1} <= {2}'.format(self.__number_of_slow_chunks, n, left)) else: # Since we saw more than a tolerable number of slow chunks, we flag this # as a possible slow-retrieval attack. This threshold will determine our From c47f9e628359d73327df579780aa4912ee454c2a Mon Sep 17 00:00:00 2001 From: dachshund Date: Thu, 5 Sep 2013 18:13:11 -0400 Subject: [PATCH 059/119] Some WIP hacks for updater. --- tuf/client/updater.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index d84659b0..2f8a0083 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -880,7 +880,6 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): uncompressed_metadata_filename) current_uncompressed_filepath = os.path.abspath(current_uncompressed_filepath) metadata_file_object.move(current_uncompressed_filepath) - compressed_file_object.move(current_filepath) else: metadata_file_object.move(current_filepath) @@ -995,8 +994,14 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas if gzip_metadata_filename in self.metadata['current'] \ [referenced_metadata]['meta']: compression = 'gzip' + # FIXME: Get the hash of the uncompressed file, because we will be + # checking the hash of the uncompressed file, not the compressed file. + previous_hashes = new_fileinfo['hashes'] new_fileinfo = self.metadata['current'][referenced_metadata] \ ['meta'][gzip_metadata_filename] + # FIXME: Replace the hashes to point to the uncompressed file ones, not + # the compressed file ones. + new_fileinfo['hashes'] = previous_hashes metadata_filename = gzip_metadata_filename else: message = 'Compressed version of '+repr(metadata_filename)+\ From 7ab75fa531d66e278e6a74e628b49f81d4e05bc9 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 11:17:33 -0400 Subject: [PATCH 060/119] Remove MetadataNotAvailableError. --- tuf/__init__.py | 8 -------- tuf/client/updater.py | 11 +++++------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index c3933ca6..69d01792 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -125,14 +125,6 @@ class ExpiredMetadataError(Error): -class MetadataNotAvailableError(Error): - """Indicate an error locating a Metadata file for a specified target/role.""" - pass - - - - - class CryptoError(Error): """Indicate any cryptography-related errors.""" pass diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 2f8a0083..45f333ba 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -936,7 +936,7 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas is 'timestamp'. See refresh(). - tuf.MetadataNotAvailableError: + tuf.UpdateError: If 'metadata_role' could not be downloaded after determining that it had changed. @@ -1868,11 +1868,10 @@ def _preorder_depth_first_walk(self, target_filepath): role_names = ['targets'] # Ensure the client has the most up-to-date version of 'targets.txt'. - # Raise 'tuf.MetadataNotAvailableError' if the changed metadata - # cannot be successfully downloaded and 'tuf.RepositoryError' if the - # referenced metadata is missing. Target methods such as this one - # are called after the top-level metadata have been refreshed (i.e., - # updater.refresh()). + # Raise 'tuf.UpdateError' if the changed metadata cannot be successfully + # downloaded and 'tuf.RepositoryError' if the referenced metadata is + # missing. Target methods such as this one are called after the top-level + # metadata have been refreshed (i.e., updater.refresh()). self._update_metadata_if_changed('targets') # Preorder depth-first traversal of the tree of target delegations. From 6b58670d28d6f223f0e4f82b10542e543b17c1ec Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 13:25:46 -0400 Subject: [PATCH 061/119] Better metadata verification. --- tuf/__init__.py | 34 ++++++----- tuf/client/updater.py | 133 +++++++++++++++++++++--------------------- 2 files changed, 83 insertions(+), 84 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index 69d01792..a4e47c3f 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -53,14 +53,6 @@ class FormatError(Error): -class InvalidMetadataJSONError(FormatError): - """Indicate that some metadata file is not valid JSON.""" - pass - - - - - class UnsupportedAlgorithmError(Error): """Indicate an error while trying to identify a user-specified algorithm.""" pass @@ -109,14 +101,6 @@ class ForbiddenTargetError(RepositoryError): -class ReplayError(RepositoryError): - """Indicate that some metadata has been replayed to the client.""" - pass - - - - - class ExpiredMetadataError(Error): """Indicate that a TUF Metadata file has expired.""" pass @@ -125,6 +109,24 @@ class ExpiredMetadataError(Error): +class ReplayedMetadataError(RepositoryError): + """Indicate that some metadata has been replayed to the client.""" + + def __init__(self, metadata_role, previous_version, current_version): + self.metadata_role = metadata_role + self.previous_version = previous_version + self.current_version = current_version + + + def __str__(self): + return str(self.metadata_role)+' is older than the version currently'+\ + 'installed.\nDownloaded version: '+repr(self.previous_version)+'\n'+\ + 'Current version: '+repr(self.current_version) + + + + + class CryptoError(Error): """Indicate any cryptography-related errors.""" pass diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 45f333ba..8ba75dbd 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -649,24 +649,33 @@ def verify_target_file(target_file_object): def __verify_metadata_file(self, metadata_file_object, metadata_role): - # Read and load the downloaded file. - try: - metadata_signable = \ - tuf.util.load_json_string(metadata_file_object.read()) - except: - logger.exception('Invalid metadata from '+mirror_url+'.') - raise - else: - # Verify the signature on the downloaded metadata object. - try: - valid = tuf.sig.verify(metadata_signable, metadata_role) - except: - message = 'Unable to verify '+metadata_filename - logger.exception(message) - raise - else: - if not valid: - raise tuf.BadSignatureError() + # Ensure the loaded 'metadata_signable' is properly formatted. + metadata_signable = \ + tuf.util.load_json_string(metadata_file_object.read()) + tuf.formats.check_signable_object_format(metadata_signable) + + # Is 'metadata_signable' newer than the currently installed + # version? + current_metadata_role = self.metadata['current'].get(metadata_role) + + # Compare metadata version numbers. Ensure there is a current + # version of the metadata role to be updated. + if current_metadata_role is not None: + current_version = current_metadata_role['version'] + downloaded_version = metadata_signable['signed']['version'] + if downloaded_version < current_version: + raise tuf.ReplayedMetadataError(metadata_role, downloaded_version, + current_version) + + # Reject the metadata if any specified targets are not allowed. + if metadata_signable['signed']['_type'] == 'Targets': + self._ensure_all_targets_allowed(metadata_role, + metadata_signable['signed']) + + # Verify the signature on the downloaded metadata object. + valid = tuf.sig.verify(metadata_signable, metadata_role) + if not valid: + raise tuf.BadSignatureError() @@ -720,16 +729,16 @@ def __get_file(self, filepath, verify_file, reference_metadata, if compression: file_object.decompress_temp_file_object(compression) - except Exception, e: + except Exception, exception: # Remember the error from this mirror, and "reset" the target file. logger.exception('Download failed from '+file_mirror+'.') - file_mirror_errors[file_mirror] = e + file_mirror_errors[file_mirror] = exception file_object = None else: try: verify_file(file_object) - except Exception, e: - file_mirror_errors[file_mirror] = e + except Exception, exception: + file_mirror_errors[file_mirror] = exception file_object = None else: break @@ -792,7 +801,7 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): None. """ - + # Construct the metadata filename as expected by the download/mirror modules. metadata_filename = metadata_role + '.txt' uncompressed_metadata_filename = metadata_filename @@ -807,9 +816,6 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): file_length = fileinfo['length'] file_hashes = fileinfo['hashes'] - # A dictionary to keep the error from every mirror that we try. - mirror_errors = {} - # Attempt a file download from each mirror until the file is downloaded and # verified. If the signature of the downloaded file is valid, proceed, # otherwise log a warning and try the next mirror. 'metadata_file_object' @@ -817,44 +823,28 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): # is the object extracted from 'metadata_file_object'. Metadata saved to # files are regarded as 'signable' objects, conformant to # 'tuf.formats.SIGNABLE_SCHEMA'. + # + # Some metadata (presently timestamp) will be downloaded "unsafely", in the + # sense that we can only estimate its true length and know nothing about + # its hashes. This is because not all metadata will have other metadata + # for it; otherwise we will have an infinite regress of metadata signing + # for each other. In this case, we will download the metadata up to the + # best length we can get for it, not check its hashes, but perform the rest + # of the checks (e.g signature verification). + # + # Note also that we presently support decompression of only "safe" + # metadata, but this is easily extend to "unsafe" metadata as well as + # "safe" targets. + if metadata_role == 'timestamp': - metadata_file_object = \ - self.unsafely_get_metadata_file(metadata_role, metadata_filename, - file_length) + metadata_file_object = \ + self.unsafely_get_metadata_file(metadata_role, metadata_filename, + file_length) else: - metadata_file_object = \ - self.safely_get_metadata_file(metadata_role, metadata_filename, - file_length, file_hashes, - compression=compression) - - # Read and load the downloaded file. - metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) - - # Ensure the loaded 'metadata_signable' is properly formatted. - try: - tuf.formats.check_signable_object_format(metadata_signable) - except tuf.FormatError, e: - message = 'Unable to load '+repr(metadata_filename)+' after update: '+str(e) - raise tuf.RepositoryError(message) - - # Is 'metadata_signable' newer than the currently installed - # version? - current_metadata_role = self.metadata['current'].get(metadata_role) - - # Compare metadata version numbers. Ensure there is a current - # version of the metadata role to be updated. - if current_metadata_role is not None: - current_version = current_metadata_role['version'] - downloaded_version = metadata_signable['signed']['version'] - if downloaded_version < current_version: - message = str(current_metadata_role)+' is older than the version currently '+\ - 'installed.\nDownloaded version: '+repr(downloaded_version)+'\n'+\ - 'Current version: '+repr(current_version) - raise tuf.RepositoryError(message) - - # Reject the metadata if any specified targets are not allowed. - if metadata_signable['signed']['_type'] == 'Targets': - self._ensure_all_targets_allowed(metadata_role, metadata_signable['signed']) + metadata_file_object = \ + self.safely_get_metadata_file(metadata_role, metadata_filename, + file_length, file_hashes, + compression=compression) # The metadata has been verified. Move the metadata file into place. # First, move the 'current' metadata file to the 'previous' directory @@ -882,10 +872,11 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): metadata_file_object.move(current_uncompressed_filepath) else: metadata_file_object.move(current_filepath) - + # Extract the metadata object so we can store it to the metadata store. # 'current_metadata_object' set to 'None' if there is not an object # stored for 'metadata_role'. + metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) updated_metadata_object = metadata_signable['signed'] current_metadata_object = self.metadata['current'].get(metadata_role) @@ -898,6 +889,7 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): + def _update_metadata_if_changed(self, metadata_role, referenced_metadata='release'): """ @@ -2283,9 +2275,14 @@ def download_target(self, target, destination_directory): try: os.makedirs(target_dirpath) except OSError, e: - if e.errno == errno.EEXIST: - pass - else: - raise - + if e.errno == errno.EEXIST: pass + else: raise + else: + logger.warn(str(target_dirpath)+' does not exist.') + target_file_object.move(destination) + + + + + From e64e938d210ad7123033fc861123b7059f19c490 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 14:38:30 -0400 Subject: [PATCH 062/119] Fix a couple of bugs. Read file before it is closed. Remove incorrect slow retrieval defense. --- tuf/client/updater.py | 2 +- tuf/conf.py | 4 ++-- tuf/download.py | 19 +++++++------------ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 8ba75dbd..e4cf0cb8 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -865,6 +865,7 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): # Next, move the verified updated metadata file to the 'current' directory. # Note that the 'move' method comes from tuf.util's TempFile class. # 'metadata_file_object' is an instance of tuf.util.TempFile. + metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) if compression == 'gzip': current_uncompressed_filepath = os.path.join(self.metadata_directory['current'], uncompressed_metadata_filename) @@ -876,7 +877,6 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): # Extract the metadata object so we can store it to the metadata store. # 'current_metadata_object' set to 'None' if there is not an object # stored for 'metadata_role'. - metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) updated_metadata_object = metadata_signable['signed'] current_metadata_object = self.metadata['current'].get(metadata_role) diff --git a/tuf/conf.py b/tuf/conf.py index 281409b3..f92a8f5f 100755 --- a/tuf/conf.py +++ b/tuf/conf.py @@ -47,7 +47,7 @@ # The maximum chunk of data, in bytes, we would download in every round. CHUNK_SIZE = 8192 -# The maximum number of slowly-retrieved chunks that we would tolerate. -MAX_NUM_OF_SLOW_CHUNKS = 5 +# The maximum number of socket operation time-outs that we would tolerate. +MAX_NUM_OF_SOCKET_TIMEOUTS = 5 diff --git a/tuf/download.py b/tuf/download.py index 0aec7e2f..45a16337 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -72,8 +72,8 @@ def __init__(self, sock, mode='rb', bufsize=-1, close=False): super(SaferSocketFileObject, self).__init__(sock, mode=mode, bufsize=bufsize, close=close) - # Count the number of slowly-retrieved chunks. - self.__number_of_slow_chunks = 0 + # Count the number of socket operation time-outs. + self.__number_of_socket_timeouts = 0 @@ -135,7 +135,7 @@ def read(self, size): return rv self._rbuf = StringIO() # reset _rbuf. we consume it via buf. - while self.__number_of_slow_chunks < tuf.conf.MAX_NUM_OF_SLOW_CHUNKS: + while self.__number_of_socket_timeouts < tuf.conf.MAX_NUM_OF_SOCKET_TIMEOUTS: left = size - buf_len # recv() will malloc the amount of memory given as its # parameter even though it often returns much less data @@ -147,8 +147,8 @@ def read(self, size): except socket.timeout: # Since the socket recv operation timed out, we increment the running # counter of slow chunks and try again. - self.__number_of_slow_chunks += 1 - logger.warn('slow chunk {0}'.format(self.__number_of_slow_chunks)) + self.__number_of_socket_timeouts += 1 + logger.warn('socket timeouts {0}'.format(self.__number_of_socket_timeouts)) continue except socket.error, e: if e.args[0] == EINTR: @@ -173,18 +173,13 @@ def read(self, size): buf_len += n del data # explicit free #assert buf_len == buf.tell() - # Since n < left with timeout on self._sock.recv, this is a slow chunk. - # We assume that 'size' is accurate w.r.t. to the overall file length; - # otherwise, we will miscount the number of truly slow chunks. - self.__number_of_slow_chunks += 1 - logger.warn('slow chunk {0}: {1} <= {2}'.format(self.__number_of_slow_chunks, n, left)) else: # Since we saw more than a tolerable number of slow chunks, we flag this # as a possible slow-retrieval attack. This threshold will determine our # bias: if it is too slow, we will have more false negatives; if it is # too high, we will have more false positives. - logger.warn('slow chunks: {0}'.format(self.__number_of_slow_chunks)) - raise tuf.SlowRetrievalError(self.__number_of_slow_chunks) + logger.warn('socket timeouts: {0}'.format(self.__number_of_socket_timeouts)) + raise tuf.SlowRetrievalError(self.__number_of_socket_timeouts) return buf.getvalue() From 954f0558f666a2e47d6ffdfa31fefbf24fa90b65 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 14:43:15 -0400 Subject: [PATCH 063/119] Shorter code. --- tuf/client/updater.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index e4cf0cb8..79074531 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -729,19 +729,15 @@ def __get_file(self, filepath, verify_file, reference_metadata, if compression: file_object.decompress_temp_file_object(compression) + verify_file(file_object) + except Exception, exception: # Remember the error from this mirror, and "reset" the target file. logger.exception('Download failed from '+file_mirror+'.') file_mirror_errors[file_mirror] = exception file_object = None else: - try: - verify_file(file_object) - except Exception, exception: - file_mirror_errors[file_mirror] = exception - file_object = None - else: - break + break if file_object: return file_object From 558b1f0d8275337e82b1c791639f4ddc6486e22d Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 14:46:48 -0400 Subject: [PATCH 064/119] Try fix for updater unit test. --- tuf/tests/test_updater.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index acd536c7..f683517e 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -223,7 +223,8 @@ def _mock_download(url, length): temp_fileobj.write(file_obj.read()) return temp_fileobj - # Patch tuf.download.safe_download(). + # Patch tuf.download functions. + tuf.download.unsafe_download = _mock_download tuf.download.safe_download = _mock_download From 3b6bb586e2a48fa8323fbdc8630e6524f0d50db5 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 15:22:32 -0400 Subject: [PATCH 065/119] Remove unnecessary code from test_updater. --- tuf/tests/test_updater.py | 47 --------------------------------------- 1 file changed, 47 deletions(-) diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index f683517e..cdb853af 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -489,9 +489,6 @@ def test_3__update_metadata(self): """ This unit test verifies the method's proper behaviour on the expected input. """ - - # Setup - original_download = tuf.download.safe_download # Since client's '.../metadata/current' will need to have separate # gzipped metadata file in order to test compressed file handling, @@ -554,9 +551,6 @@ def test_3__update_metadata(self): os.remove(os.path.join(self.client_current_dir,'targets.txt.gz')) self._remove_target_from_targets_dir(added_target_1) - # RESTORE - tuf.download.safe_download = original_download - def test_1__update_fileinfo(self): @@ -615,9 +609,6 @@ def test_3__update_metadata_if_changed(self): """ This unit test verifies the method's proper behaviour on expected input. """ - - # Setup - original_download = tuf.download.safe_download # To test updater._update_metadata_if_changed, 'targets' metadata file is # going to be modified at the server's repository. @@ -707,9 +698,6 @@ def test_3__update_metadata_if_changed(self): os.remove(os.path.join(self.client_current_dir, 'release.txt.gz')) self._remove_target_from_targets_dir(added_target_1) - # RESTORE - tuf.download.safe_download = original_download - @@ -766,8 +754,6 @@ def test_2__ensure_not_expired(self): def test_4_refresh(self): - # Setup. - original_download = tuf.download.safe_download # This unit test is based on adding an extra target file to the # server and rebuilding all server-side metadata. When 'refresh' @@ -799,16 +785,10 @@ def test_4_refresh(self): self._mock_download_url_to_tempfileobj(self.all_role_paths) setup.build_server_repository(self.server_repo_dir, self.targets_dir) - # RESTORE - tuf.download.safe_download = original_download - def test_4__refresh_targets_metadata(self): - - # Setup - original_download = tuf.download.safe_download # To test this method a target file would be added to a delegated role, # and metadata on the server side would be rebuilt. @@ -864,9 +844,6 @@ def test_4__refresh_targets_metadata(self): shutil.rmtree(os.path.join(self.server_repo_dir, 'keystore')) setup.build_server_repository(self.server_repo_dir, self.targets_dir) - # RESTORE - tuf.download.safe_download = original_download - @@ -893,9 +870,6 @@ def test_3__targets_of_role(self): def test_5_all_targets(self): - - # Setup - original_download = tuf.download.safe_download # As with '_refresh_targets_metadata()', tuf.roledb._roledb_dict # has to be populated. The 'tuf.download.safe_download' method @@ -925,9 +899,6 @@ def test_5_all_targets(self): # targets in 'all_targets' should then be 6. self.assertTrue(len(all_targets) is 6) - # RESTORE - tuf.download.safe_download = original_download - @@ -984,9 +955,6 @@ def test_6_target(self): def test_6_download_target(self): - - # Setup: - original_download = tuf.download.safe_download # 'tuf.download.safe_download' method should be patched. target_rel_paths_src = self._get_list_of_target_paths(self.targets_dir) @@ -1032,17 +1000,11 @@ def test_6_download_target(self): for mirror_name, mirror_info in mirrors.items(): mirrors[mirror_name]['confined_target_dirs'] = [''] - # RESTORE - tuf.download.safe_download = original_download - def test_7_updated_targets(self): - # Setup: - original_download = tuf.download.safe_download - # In this test, client will have two target files. Server will modify # one of them. As with 'all_targets' function, tuf.roledb._roledb_dict # has to be populated. 'tuf.download.safe_download' method @@ -1103,17 +1065,11 @@ def test_7_updated_targets(self): msg = 'A file that need not to be updated is indicated as updated.' self.fail(msg) - # RESTORE - tuf.download.safe_download = original_download - def test_8_remove_obsolete_targets(self): - # Setup: - original_download = tuf.download.safe_download - # This unit test should be last, because it removes target files from the # server's targets directory. It is done to avoid adding files, rebuilding # and updating metadata. @@ -1162,9 +1118,6 @@ def test_8_remove_obsolete_targets(self): self.Repository.remove_obsolete_targets(dest_dir) self.assertTrue(os.listdir(dest_dir), 2) - # RESTORE - tuf.download.safe_download = original_download - def tearDownModule(): # tearDownModule() is called after all the tests have run. From 54cab17f0a2eda951e880e1c2fcafbf63969a706 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 16:34:27 -0400 Subject: [PATCH 066/119] Document new functions in updater. --- tuf/__init__.py | 11 ++ tuf/client/updater.py | 236 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 219 insertions(+), 28 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index a4e47c3f..2869ba2d 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -217,6 +217,14 @@ class UnknownRoleError(Error): +class UnknownTargetError(Error): + """Indicate an error trying to locate or identify a specified target.""" + pass + + + + + class InvalidNameError(Error): """Indicate an error while trying to validate any type of named object""" pass @@ -235,6 +243,9 @@ def __init__(self, mirror_errors): # Dictionary of URL strings to Exception instances self.mirror_errors = mirror_errors + def __str__(self): + return str(self.mirror_errors) + diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 79074531..b741d148 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -637,18 +637,84 @@ def __check_hashes(self, input_file, trusted_hashes): def get_target_file(self, target_filepath, file_length, file_hashes): + """ + + Safely download a target file up to a certain length, and check its + hashes thereafter. + + + target_filepath: + The relative target filepath obtained from TUF targets metadata. + + file_length: + The expected length of the target file. + + file_hashes: + The expected hashes of the target file. + + + tuf.UpdateError: + The target could not be fetched. This is raised only when all known + mirrors failed to provide a valid copy of the desired target file. + + + The target file is downloaded from all known repository mirrors in the + worst case. If a valid copy of the target file is found, it is stored in + a temporary file and returned. + + + A tuf.util.TempFile file-like object containing the target. + + """ def verify_target_file(target_file_object): + # Every target file must have its hashes inspected. self.__check_hashes(target_file_object, file_hashes) - + return self.__get_file(target_filepath, verify_target_file, 'target', - file_length, download_safely=True, compression=None) + file_length, download_safely=True, compression=None) def __verify_metadata_file(self, metadata_file_object, metadata_role): + """ + + A private helpe function to verify a downloaded metadata file. + + + metadata_file_object: + A tuf.util.TempFile instance containing the metadata file. + + metadata_role: + The role name of the metadata. + + + tuf.ForbiddenTargetError: + In case a targets role has signed for a target it was not delegated to. + + tuf.FormatError: + In case the metadata file is somehow not valid. + + tuf.ReplayedMetadataError: + In case the downloaded metadata file is older than the current one. + + tuf.RepositoryError: + In case the repository is somehow inconsistent; e.g. a parent has not + delegated to a child (contrary to expectations). + + tuf.SignatureError: + In case the metadata file does not have a valid signature. + + + None. + + + None. + + """ + # Ensure the loaded 'metadata_signable' is properly formatted. metadata_signable = \ tuf.util.load_json_string(metadata_file_object.read()) @@ -683,6 +749,36 @@ def __verify_metadata_file(self, metadata_file_object, metadata_role): def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, file_length): + """ + + Unsafely download a metadata file up to a certain length. The actual file + length may not be strictly equal to its expected length. File hashes will + not be checked because it is expected to be unknown. + + + metadata_role: + The role name of the metadata. + + metadata_filepath: + The relative metadata filepath. + + file_length: + The expected length of the metadata file. + + + tuf.UpdateError: + The metadata could not be fetched. This is raised only when all known + mirrors failed to provide a valid copy of the desired metadata file. + + + The metadata file is downloaded from all known repository mirrors in the + worst case. If a valid copy of the metadata file is found, it is stored + in a temporary file and returned. + + + A tuf.util.TempFile file-like object containing the metadata. + + """ def unsafely_verify_metadata_file(metadata_file_object): self.__verify_metadata_file(metadata_file_object, metadata_role) @@ -696,7 +792,42 @@ def unsafely_verify_metadata_file(metadata_file_object): def safely_get_metadata_file(self, metadata_role, metadata_filepath, - file_length, file_hashes, compression): + file_length, file_hashes, compression): + """ + + Safely download a metadata file up to a certain length, and check its + hashes thereafter. + + + metadata_role: + The role name of the metadata. + + metadata_filepath: + The relative metadata filepath. + + file_length: + The expected length of the metadata file. + + file_hashes: + The expected hashes of the metadata file. + + compression: + The name of the compression algorithm used to compress the metadata. + + + tuf.UpdateError: + The metadata could not be fetched. This is raised only when all known + mirrors failed to provide a valid copy of the desired metadata file. + + + The metadata file is downloaded from all known repository mirrors in the + worst case. If a valid copy of the metadata file is found, it is stored + in a temporary file and returned. + + + A tuf.util.TempFile file-like object containing the metadata. + + """ def safely_verify_metadata_file(metadata_file_object): self.__check_hashes(metadata_file_object, file_hashes) @@ -710,10 +841,57 @@ def safely_verify_metadata_file(metadata_file_object): - def __get_file(self, filepath, verify_file, reference_metadata, - trusted_length, download_safely, compression): - file_mirrors = tuf.mirrors.get_list_of_mirrors(reference_metadata, - filepath, self.mirrors) + # TODO: Instead of the more fragile 'download_safely' switch, unroll the + # function into two separate ones: one for "safe" download, and the other one + # for "unsafe" download? This should induce safer and more readable code. + def __get_file(self, filepath, verify_file, file_type, + file_length, download_safely, compression): + """ + + Try downloading, up to a certain length, a metadata or target file from a + list of known mirrors. As soon as the first valid copy of the file is + found, the rest of the mirrors will be skipped. + + + filepath: + The relative metadata or target filepath. + + verify_file: + A function which expects a file-like object and which will raise an + exception in case the file is not valid for any reason. + + file_type: + Type of data needed for download, must correspond to one of the strings + in the list ['meta', 'target']. 'meta' for metadata file type or + 'target' for target file type. It should correspond to NAME_SCHEMA + format. + + file_length: + The expected length of the metadata or target file. + + download_safely: + A boolean switch to toggle safe or unsafe download of the file. + + compression: + The name of the compression algorithm used to compress the file. + + + tuf.UpdateError: + The metadata could not be fetched. This is raised only when all known + mirrors failed to provide a valid copy of the desired metadata file. + + + The file is downloaded from all known repository mirrors in the worst + case. If a valid copy of the file is found, it is stored in a temporary + file and returned. + + + A tuf.util.TempFile file-like object containing the metadata or target. + + """ + + file_mirrors = tuf.mirrors.get_list_of_mirrors(file_type, filepath, + self.mirrors) # file_mirror (URL): error (Exception) file_mirror_errors = {} file_object = None @@ -721,10 +899,9 @@ def __get_file(self, filepath, verify_file, reference_metadata, for file_mirror in file_mirrors: try: if download_safely: - file_object = tuf.download.safe_download(file_mirror, trusted_length) + file_object = tuf.download.safe_download(file_mirror, file_length) else: - file_object = tuf.download.unsafe_download(file_mirror, - trusted_length) + file_object = tuf.download.unsafe_download(file_mirror, file_length) if compression: file_object.decompress_temp_file_object(compression) @@ -1064,7 +1241,7 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): 'signable' object). - tuf.RepositoryError: + tuf.ForbiddenTargetError: If the targets of 'metadata_role' are not allowed according to the parent's metadata file. The 'paths' and 'path_hash_prefixes' attributes are verified. @@ -1106,10 +1283,11 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): consistent = self._paths_are_consistent_with_hash_prefixes if not consistent(actual_child_targets, allowed_child_path_hash_prefixes): - raise tuf.RepositoryError('Role '+repr(metadata_role)+' specifies '+\ - 'target which does not have a path hash '+\ - 'prefix matching the prefix listed by '+\ - 'the parent role '+repr(parent_role)+'.') + raise tuf.ForbiddenTargetError('Role '+repr(metadata_role)+\ + ' specifies target which does not'+\ + ' have a path hash prefix matching'+\ + ' the prefix listed by the parent'+\ + ' role '+repr(parent_role)+'.') elif allowed_child_paths is not None: @@ -1126,25 +1304,27 @@ def _ensure_all_targets_allowed(self, metadata_role, metadata_object): if prefix == allowed_child_path: break else: - message = 'Role '+repr(metadata_role)+' specifies target '+\ - repr(child_target)+' which is not an allowed path according '+\ - 'to the delegations set by '+repr(parent_role)+'.' - raise tuf.RepositoryError(message) + raise tuf.ForbiddenTargetError('Role '+repr(metadata_role)+\ + ' specifies target '+\ + repr(child_target)+' which is not'+\ + ' an allowed path according to'+\ + ' the delegations set by '+\ + repr(parent_role)+'.') else: # 'role' should have been validated when it was downloaded. # The 'paths' or 'path_hash_prefixes' attributes should not be missing, - # so log a warning if this clause is reached. - logger.warn(repr(role)+' unexpectedly did not contain one of '+\ - 'the required fields ("paths" or "path_hash_prefixes").') + # so raise an error in case this clause is reached. + raise tuf.FormatError(repr(role)+' did not contain one of '+\ + 'the required fields ("paths" or '+\ + '"path_hash_prefixes").') # Raise an exception if the parent has not delegated to the specified # 'metadata_role' child role. else: - message = repr(parent_role)+' has not delegated to '+\ - repr(metadata_role)+'.' - raise tuf.RepositoryError(message) + raise tuf.RepositoryError(repr(parent_role)+' has not delegated to '+\ + repr(metadata_role)+'.') @@ -1789,7 +1969,7 @@ def target(self, target_filepath): tuf.FormatError: If 'target_filepath' is improperly formatted. - tuf.RepositoryError: + tuf.UnknownTargetError: If 'target_filepath' was not found. Any other unforeseen runtime exception. @@ -1814,7 +1994,7 @@ def target(self, target_filepath): if target is None: message = target_filepath+' not found.' logger.error(message) - raise tuf.RepositoryError(message) + raise tuf.UnknownTargetError(message) # Otherwise, return the found target. else: return target @@ -1962,7 +2142,7 @@ def _visit_child_role(self, child_role, target_filepath): Ensure that we explore only delegated roles trusted with the target. We assume conservation of delegated paths in the complete tree of delegations. Note that the call to _ensure_all_targets_allowed in - _update_metadata should already ensure that all targets metadata is + __verify_metadata_file should already ensure that all targets metadata is valid; i.e. that the targets signed by a delegatee is a proper subset of the targets delegated to it by the delegator. Nevertheless, we check it again here for performance and safety reasons. From e7704355a9da026b16a04eef35f6a971c6f1c105 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 16:58:18 -0400 Subject: [PATCH 067/119] Rename UpdateError exception to NoWorkingMirrorError. --- tuf/__init__.py | 3 ++- tuf/client/updater.py | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index 2869ba2d..05036833 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -232,7 +232,8 @@ class InvalidNameError(Error): -class UpdateError(Error): + +class NoWorkingMirrorError(Error): """An updater will throw this exception in case it could not download a metadata or target file. diff --git a/tuf/client/updater.py b/tuf/client/updater.py index b741d148..32e1866b 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -544,7 +544,7 @@ def refresh(self): None. - tuf.UpdateError: + tuf.NoWorkingMirrorError: If the metadata for any of the top-level roles cannot be updated. tuf.ExpiredMetadataError: @@ -570,7 +570,7 @@ def refresh(self): # Update the top-level metadata. The _update_metadata_if_changed() and # _update_metadata() calls below do NOT perform an update if there # is insufficient trusted signatures for the specified metadata. - # Raise 'tuf.UpdateError' if an update fails. + # Raise 'tuf.NoWorkingMirrorError' if an update fails. # Use default but sane information for timestamp metadata, and do not # require strict checks on its required length. @@ -653,7 +653,7 @@ def get_target_file(self, target_filepath, file_length, file_hashes): The expected hashes of the target file. - tuf.UpdateError: + tuf.NoWorkingMirrorError: The target could not be fetched. This is raised only when all known mirrors failed to provide a valid copy of the desired target file. @@ -766,7 +766,7 @@ def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, The expected length of the metadata file. - tuf.UpdateError: + tuf.NoWorkingMirrorError: The metadata could not be fetched. This is raised only when all known mirrors failed to provide a valid copy of the desired metadata file. @@ -815,7 +815,7 @@ def safely_get_metadata_file(self, metadata_role, metadata_filepath, The name of the compression algorithm used to compress the metadata. - tuf.UpdateError: + tuf.NoWorkingMirrorError: The metadata could not be fetched. This is raised only when all known mirrors failed to provide a valid copy of the desired metadata file. @@ -876,7 +876,7 @@ def __get_file(self, filepath, verify_file, file_type, The name of the compression algorithm used to compress the file. - tuf.UpdateError: + tuf.NoWorkingMirrorError: The metadata could not be fetched. This is raised only when all known mirrors failed to provide a valid copy of the desired metadata file. @@ -921,7 +921,7 @@ def __get_file(self, filepath, verify_file, file_type, else: logger.exception('Failed to download {0}: {1}'.format(filepath, file_mirror_errors)) - raise tuf.UpdateError(file_mirror_errors) + raise tuf.NoWorkingMirrorError(file_mirror_errors) @@ -960,7 +960,7 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): are considered. Any other string is ignored. - tuf.UpdateError: + tuf.NoWorkingMirrorError: The metadata could not be updated. This is not specific to a single failure but rather indicates that all possible ways to update the metadata have been tried and failed. @@ -1101,7 +1101,7 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas is 'timestamp'. See refresh(). - tuf.UpdateError: + tuf.NoWorkingMirrorError: If 'metadata_role' could not be downloaded after determining that it had changed. @@ -1980,7 +1980,7 @@ def target(self, target_filepath): The target information for 'target_filepath', conformant to 'tuf.formats.TARGETFILE_SCHEMA'. - + """ # Does 'target_filepath' have the correct format? @@ -2036,7 +2036,7 @@ def _preorder_depth_first_walk(self, target_filepath): role_names = ['targets'] # Ensure the client has the most up-to-date version of 'targets.txt'. - # Raise 'tuf.UpdateError' if the changed metadata cannot be successfully + # Raise 'tuf.NoWorkingMirrorError' if the changed metadata cannot be successfully # downloaded and 'tuf.RepositoryError' if the referenced metadata is # missing. Target methods such as this one are called after the top-level # metadata have been refreshed (i.e., updater.refresh()). @@ -2413,7 +2413,7 @@ def download_target(self, target, destination_directory): tuf.FormatError: If 'target' is not properly formatted. - tuf.UpdateError: + tuf.NoWorkingMirrorError: If a target could not be downloaded from any of the mirrors. From 5b1daa5293844d495d66b4a5f8943895f7351d85 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 19:08:57 -0400 Subject: [PATCH 068/119] Correctly check metadata of both compressed and uncompressed files. --- tuf/client/updater.py | 54 +++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 32e1866b..60a2f807 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1121,7 +1121,7 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas """ - metadata_filename = metadata_role + '.txt' + uncompressed_metadata_filename = metadata_role + '.txt' # Ensure the referenced metadata has been loaded. The 'root' role may be # updated without having 'release' available. @@ -1144,44 +1144,48 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas # metadata available on the repository, including any that may be in # compressed form. compression = None - - # Extract the new fileinfo of the uncompressed version of 'metadata_role'. - new_fileinfo = self.metadata['current'][referenced_metadata] \ - ['meta'][metadata_filename] - + + # Extract the fileinfo of the uncompressed version of 'metadata_role'. + uncompressed_fileinfo = self.metadata['current'][referenced_metadata] \ + ['meta'] \ + [uncompressed_metadata_filename] + # Check for availability of compressed versions of 'release.txt', # 'targets.txt', and delegated Targets, which also start with 'targets'. # For 'targets.txt' and delegated metadata, 'referenced_metata' # should always be 'release'. 'release.txt' specifies all roles # provided by a repository, including their file sizes and hashes. if metadata_role == 'release' or metadata_role.startswith('targets'): - gzip_metadata_filename = metadata_filename + '.gz' + gzip_metadata_filename = uncompressed_metadata_filename + '.gz' if gzip_metadata_filename in self.metadata['current'] \ [referenced_metadata]['meta']: compression = 'gzip' - # FIXME: Get the hash of the uncompressed file, because we will be - # checking the hash of the uncompressed file, not the compressed file. - previous_hashes = new_fileinfo['hashes'] - new_fileinfo = self.metadata['current'][referenced_metadata] \ + compressed_fileinfo = self.metadata['current'][referenced_metadata] \ ['meta'][gzip_metadata_filename] - # FIXME: Replace the hashes to point to the uncompressed file ones, not - # the compressed file ones. - new_fileinfo['hashes'] = previous_hashes - metadata_filename = gzip_metadata_filename + # NOTE: When we download the compressed file, we care about its + # compressed length. However, we check the hash of the decompressed + # file, therefore we use the hashes of the uncompressed file. + fileinfo = {'length': compressed_fileinfo['length'], + 'hashes': uncompressed_fileinfo['hashes']} + logger.debug('Compressed version of '+\ + repr(uncompressed_metadata_filename)+' is available at '+\ + repr(gzip_metadata_filename)+'.') else: - message = 'Compressed version of '+repr(metadata_filename)+\ - ' not available.' - logger.debug(message) + logger.debug('Compressed version of '+\ + repr(uncompressed_metadata_filename)+' not available.') + fileinfo = uncompressed_fileinfo - # Simply return if the fileinfo has not changed, according to the - # fileinfo provided by the referenced metadata. - if not self._fileinfo_has_changed(metadata_filename, new_fileinfo): + # Simply return if the file has not changed, according to the metadata + # about the uncompressed file provided by the referenced metadata. + if not self._fileinfo_has_changed(uncompressed_metadata_filename, + uncompressed_fileinfo): return - logger.debug('Metadata '+repr(metadata_filename)+' has changed.') + logger.debug('Metadata '+repr(uncompressed_metadata_filename)+\ + ' has changed.') try: - self._update_metadata(metadata_role, fileinfo=new_fileinfo, + self._update_metadata(metadata_role, fileinfo=fileinfo, compression=compression) except: # The current metadata we have is not current but we couldn't @@ -1527,11 +1531,11 @@ def _move_current_to_previous(self, metadata_role): metadata_filepath) current_filepath = os.path.join(self.metadata_directory['current'], metadata_filepath) - + # Remove the previous path if it exists. if os.path.exists(previous_filepath): os.remove(previous_filepath) - + # Move the current path to the previous path. if os.path.exists(current_filepath): tuf.util.ensure_parent_dir(previous_filepath) From df9d9a9ad11fbad3b08464f9c720a5aae44cc784 Mon Sep 17 00:00:00 2001 From: dachshund Date: Fri, 6 Sep 2013 19:25:43 -0400 Subject: [PATCH 069/119] Fix a variable shadow bug in tuf.log. --- tuf/client/updater.py | 2 +- tuf/log.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 60a2f807..d36929f0 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1164,7 +1164,7 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas ['meta'][gzip_metadata_filename] # NOTE: When we download the compressed file, we care about its # compressed length. However, we check the hash of the decompressed - # file, therefore we use the hashes of the uncompressed file. + # file; therefore we use the hashes of the uncompressed file. fileinfo = {'length': compressed_fileinfo['length'], 'hashes': uncompressed_fileinfo['hashes']} logger.debug('Compressed version of '+\ diff --git a/tuf/log.py b/tuf/log.py index 51397d1f..0667e4ad 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -229,6 +229,7 @@ def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): # Set the console handler for the logger. The built-in console handler will # log messages to 'sys.stderr' and capture 'log_level' messages. + global console_handler console_handler = logging.StreamHandler() console_handler.setLevel(log_level) console_handler.setFormatter(formatter) From d92860c8009bd8901a317b243c247aba4f96f3a4 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 7 Sep 2013 18:04:58 -0400 Subject: [PATCH 070/119] Disable console logging in system tests. --- tuf/tests/system_tests/util_test_tools.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/system_tests/util_test_tools.py index 25f710c0..d79cfeed 100755 --- a/tuf/tests/system_tests/util_test_tools.py +++ b/tuf/tests/system_tests/util_test_tools.py @@ -137,13 +137,14 @@ import subprocess import tuf +import tuf.client.updater import tuf.formats import tuf.interposition -import tuf.util -import tuf.client.updater +import tuf.log import tuf.repo.signercli as signercli import tuf.repo.signerlib as signerlib import tuf.repo.keystore as keystore +import tuf.util logger = logging.getLogger('tuf.tests.system_tests.util_test_tools') @@ -153,6 +154,10 @@ tuf_configurations = None +def disable_console_logging(): + tuf.log.logger.removeHandler(tuf.log.console_handler) + + def init_repo(tuf=False, port=None): # Temp root directory for regular and tuf repositories. # WARNING: tuf client stores files in '{root_repo}/downloads/' directory! @@ -184,6 +189,7 @@ def init_repo(tuf=False, port=None): keyids = None if tuf: + disable_console_logging() keyids = init_tuf(root_repo) create_interposition_config(root_repo, url) From c259b3edcb212ec3c9f8af4438a4f5cffd59dcd8 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 7 Sep 2013 18:33:04 -0400 Subject: [PATCH 071/119] Processes must wait longer for the slow retrieval test. --- tuf/tests/system_tests/test_slow_retrieval_attack.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tuf/tests/system_tests/test_slow_retrieval_attack.py b/tuf/tests/system_tests/test_slow_retrieval_attack.py index 025535f7..9cbed665 100755 --- a/tuf/tests/system_tests/test_slow_retrieval_attack.py +++ b/tuf/tests/system_tests/test_slow_retrieval_attack.py @@ -71,14 +71,14 @@ def _download(url, filename, TUF=False): def test_slow_retrieval_attack(TUF=False, mode=None): - WAIT_TIME = 10 # Number of seconds to wait until download completes. + WAIT_TIME = 20 # Number of seconds to wait until download completes. ERROR_MSG = mode + '\tSlow Retrieval Attack was Successful!\n\n' # Launch the server. port = random.randint(30000, 45000) command = ['python', 'slow_retrieval_server.py', str(port), mode] server_process = subprocess.Popen(command, stderr=subprocess.PIPE) - time.sleep(.1) + time.sleep(1) try: # Setup. @@ -119,11 +119,9 @@ def test_slow_retrieval_attack(TUF=False, mode=None): proc.terminate() raise SlowRetrievalAttackAlert(ERROR_MSG) - finally: if server_process.returncode is None: server_process.kill() - print 'Communication with slow server aborted. Terminate the slow server.\n' util_test_tools.cleanup(root_repo, server_proc) From 0c80f27cad5f3af2734bd1978695b7d9c2ec7d2a Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 7 Sep 2013 19:57:09 -0400 Subject: [PATCH 072/119] Fix a bug. --- tuf/client/updater.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index d36929f0..15dc1000 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1174,6 +1174,8 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas logger.debug('Compressed version of '+\ repr(uncompressed_metadata_filename)+' not available.') fileinfo = uncompressed_fileinfo + else: + fileinfo = uncompressed_fileinfo # Simply return if the file has not changed, according to the metadata # about the uncompressed file provided by the referenced metadata. From 14364d01c733f83806fff6defb53e4c9ae4a52c6 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sat, 7 Sep 2013 20:21:45 -0400 Subject: [PATCH 073/119] A few fixes for the updater unit test. --- tuf/tests/test_updater.py | 40 ++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index cdb853af..5dd7ed58 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -512,15 +512,16 @@ def test_3__update_metadata(self): # Test: Invalid file downloaded. # Patch 'download.download_url_to_tempfileobj' function. self._mock_download_url_to_tempfileobj(self.release_filepath) - # TODO: Set fileinfo to a valid object. - self.assertRaises(tuf.RepositoryError, _update_metadata, 'targets', None) + + # TODO: Is this the original intent of this test? + self.assertRaises(TypeError, _update_metadata, 'targets', None) # Test: normal case. # Patch 'download.download_url_to_tempfileobj' function. self._mock_download_url_to_tempfileobj(self.targets_filepath) - # TODO: Set fileinfo to a valid object. - _update_metadata('targets', None) + _update_metadata('targets', + signerlib.get_metadata_file_info(self.targets_filepath)) list_of_targets = self.Repository.metadata['current']['targets']['targets'] # Verify that the added target's path is listed in target's metadata. @@ -537,8 +538,12 @@ def test_3__update_metadata(self): # Re-patch 'download.download_url_to_tempfileobj' function. self._mock_download_url_to_tempfileobj(targets_filepath_compressed) - # TODO: Set fileinfo to a valid object. - _update_metadata('targets', None, compression='gzip') + # TODO: Not convinced this is actually being tested correctly. + # See how we get fileinfo in tuf.client.updater._update_metadata_if_changed + _update_metadata('targets', + #signerlib.get_metadata_file_info(self.targets_filepath), + None, + compression='gzip') list_of_targets = self.Repository.metadata['current']['targets']['targets'] # Verify that the added target's path is listed in target's metadata. @@ -548,7 +553,7 @@ def test_3__update_metadata(self): # Restoring server's repository to the initial state. os.remove(targets_filepath_compressed) - os.remove(os.path.join(self.client_current_dir,'targets.txt.gz')) + os.remove(os.path.join(self.client_current_dir,'targets.txt')) self._remove_target_from_targets_dir(added_target_1) @@ -690,8 +695,13 @@ def test_3__update_metadata_if_changed(self): # Test: Invalid targets metadata file downloaded. # Patch 'download.download_url_to_tempfileobj' and update targets. self._mock_download_url_to_tempfileobj(self.root_filepath) - self.assertRaises(tuf.MetadataNotAvailableError, update_if_changed, - 'targets') + + # TODO: Is this the original intent of this test? + try: + update_if_changed('targets') + except tuf.NoWorkingMirrorError, exception: + for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + assert isinstance(mirror_error, tuf.BadHashError) # Restoring repositories to the initial state. os.remove(release_filepath_compressed) @@ -947,7 +957,7 @@ def test_6_target(self): # Test: invalid target path. - self.assertRaises(tuf.RepositoryError, target, self.random_path()) + self.assertRaises(tuf.UnknownTargetError, target, self.random_path()) @@ -993,9 +1003,13 @@ def test_6_download_target(self): # Patch 'download.download_url_to_tempfileobj' and verify that an # exception is raised. self._mock_download_url_to_tempfileobj(os.path.join(self.targets_dir, file_path)) - self.assertRaises(tuf.DownloadError, self.Repository.download_target, - target_info, - dest_dir) + + try: + self.Repository.download_target(target_info, dest_dir) + except tuf.NoWorkingMirrorError, exception: + # Ensure that no mirrors were found due to mismatch in confined target + # directories. + assert len(exception.mirror_errors) == 0 for mirror_name, mirror_info in mirrors.items(): mirrors[mirror_name]['confined_target_dirs'] = [''] From 01db53dac620db87801642c688c90b43f3d8832e Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 8 Sep 2013 00:32:09 -0400 Subject: [PATCH 074/119] A way to mitigate #42. --- tuf/__init__.py | 9 +- tuf/conf.py | 11 +- tuf/download.py | 115 +++++++++++++++--- .../test_slow_retrieval_attack.py | 20 +-- 4 files changed, 121 insertions(+), 34 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index 05036833..9e66d15b 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -186,8 +186,13 @@ class DownloadLengthMismatchError(DownloadError): class SlowRetrievalError(DownloadError): """"Indicate that downloading a file took an unreasonably long time.""" - def __init__(self, number_of_slow_chunks): - self.number_of_slow_chunks = number_of_slow_chunks + def __init__(self, cumulative_moving_average_of_speed): + self.__cumulative_moving_average_of_speed = \ + cumulative_moving_average_of_speed#bytes/second + + def __str__(self): + return "Cumulative moving average of download speed: "+\ + str(self.__cumulative_moving_average_of_speed)+" bytes/second" diff --git a/tuf/conf.py b/tuf/conf.py index f92a8f5f..042892be 100755 --- a/tuf/conf.py +++ b/tuf/conf.py @@ -39,15 +39,16 @@ # Since the timestamp role does not have signed metadata about itself, we set a # default but sane upper bound for the number of bytes required to download it. -DEFAULT_TIMESTAMP_REQUIRED_LENGTH = 2048 +DEFAULT_TIMESTAMP_REQUIRED_LENGTH = 2048#bytes # Set a timeout value in seconds (float) for non-blocking socket operations. -SOCKET_TIMEOUT = 1 +SOCKET_TIMEOUT = 1#seconds # The maximum chunk of data, in bytes, we would download in every round. -CHUNK_SIZE = 8192 +CHUNK_SIZE = 8192#bytes -# The maximum number of socket operation time-outs that we would tolerate. -MAX_NUM_OF_SOCKET_TIMEOUTS = 5 +# The minimum cumulative moving average of download speed (bytes/second) that +# must be met to avoid being considered as a slow retrieval attack. +MIN_CUMULATIVE_MOVING_AVERAGE_OF_DOWNLOAD_SPEED = CHUNK_SIZE#bytes/second diff --git a/tuf/download.py b/tuf/download.py index 45a16337..a89a4973 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -22,10 +22,14 @@ """ +# Induce "true division" (http://www.python.org/dev/peps/pep-0238/). +from __future__ import division + import httplib import logging import os.path import socket +import time import tuf import tuf.conf @@ -55,7 +59,6 @@ errno = None EINTR = getattr(errno, 'EINTR', 4) - # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.download') @@ -72,17 +75,96 @@ def __init__(self, sock, mode='rb', bufsize=-1, close=False): super(SaferSocketFileObject, self).__init__(sock, mode=mode, bufsize=bufsize, close=close) - # Count the number of socket operation time-outs. - self.__number_of_socket_timeouts = 0 + # Measure the cumulative moving average of download speed. + self.__cumulative_moving_average_of_speed = 0 + # Count the number of socket receive operations. + self.__number_of_receive_operations = 0 + # Remember the time a clock was started. + self.__start_time = None + + + + + + def __start_clock(self): + """ + + Start the clock to measure time difference later. + + + None. + + + AssertionError: When any internal condition is not true. + + + Start time is kept inside this object. + + + None. + + """ + + # We must have reset the clock before this. + assert self.__start_time is None + self.__start_time = time.time() + + + + + + def __stop_clock(self, data_length): + """ + + Stop the clock and try to detect slow retrieval. + + + data_length: A nonnegative integer indicating the size of data retrieved. + + + tuf.SlowRetrievalError: When slow retrieval is detected. + + AssertionError: When any internal condition is not true. + + + Start time is cleared inside this object. + + + None. + + """ + + stop_time = time.time() + # We must have already started the clock. + assert self.__start_time > 0, self.__start_time + time_delta = stop_time-self.__start_time + # Reset the clock. + self.__start_time = None + speed = data_length/time_delta + logger.debug('Speed: '+str(speed)+' ('+str(data_length)+'/'+\ + str(time_delta)+') bytes/second') + + # Measure the cumulative moving average of the download speed. + #https://en.wikipedia.org/wiki/Moving_average#Cumulative_moving_average + self.__number_of_receive_operations += 1 + numerator = speed+((self.__number_of_receive_operations-1)*self.__cumulative_moving_average_of_speed) + denominator = self.__number_of_receive_operations + self.__cumulative_moving_average_of_speed = numerator/denominator + + # If the cumulative moving average of the download speed is below a certain + # threshold, we flag this as a possible slow-retrieval attack. This + # threshold will determine our bias: if it is too slow, we will have more + # false negatives; if it is too high, we will have more false positives. + if self.__cumulative_moving_average_of_speed < tuf.conf.MIN_CUMULATIVE_MOVING_AVERAGE_OF_DOWNLOAD_SPEED: + raise tuf.SlowRetrievalError(self.__cumulative_moving_average_of_speed) + logger.debug('Cumulative moving average of download speed: '+\ + str(self.__cumulative_moving_average_of_speed)+\ + ' bytes/second') - # TODO: Better protection against slow-retrieval attacks. For example, we do - # not take into consideration that a sufficiently large file might take an - # intolerably long time with our present methods. We should be able to better - # protect ourselves with more careful state-keeping (such as measuring time). def read(self, size): """ @@ -135,7 +217,8 @@ def read(self, size): return rv self._rbuf = StringIO() # reset _rbuf. we consume it via buf. - while self.__number_of_socket_timeouts < tuf.conf.MAX_NUM_OF_SOCKET_TIMEOUTS: + # Since we try to detect slow retrieval, this should not be an infinite loop. + while True: left = size - buf_len # recv() will malloc the amount of memory given as its # parameter even though it often returns much less data @@ -143,17 +226,18 @@ def read(self, size): # as we copy it into a StringIO and free it. This avoids # fragmentation issues on many platforms. try: + self.__start_clock() data = self._sock.recv(left) except socket.timeout: - # Since the socket recv operation timed out, we increment the running - # counter of slow chunks and try again. - self.__number_of_socket_timeouts += 1 - logger.warn('socket timeouts {0}'.format(self.__number_of_socket_timeouts)) + self.__stop_clock(0) continue except socket.error, e: if e.args[0] == EINTR: + self.__stop_clock(0) continue raise + else: + self.__stop_clock(len(data)) if not data: break n = len(data) @@ -173,13 +257,6 @@ def read(self, size): buf_len += n del data # explicit free #assert buf_len == buf.tell() - else: - # Since we saw more than a tolerable number of slow chunks, we flag this - # as a possible slow-retrieval attack. This threshold will determine our - # bias: if it is too slow, we will have more false negatives; if it is - # too high, we will have more false positives. - logger.warn('socket timeouts: {0}'.format(self.__number_of_socket_timeouts)) - raise tuf.SlowRetrievalError(self.__number_of_socket_timeouts) return buf.getvalue() diff --git a/tuf/tests/system_tests/test_slow_retrieval_attack.py b/tuf/tests/system_tests/test_slow_retrieval_attack.py index 9cbed665..09d684ae 100755 --- a/tuf/tests/system_tests/test_slow_retrieval_attack.py +++ b/tuf/tests/system_tests/test_slow_retrieval_attack.py @@ -62,8 +62,8 @@ def _download(url, filename, TUF=False): # If timeout or RepositoryError is raised, this means # that TUF has prevented the slow retrieval attack. Enable # the logging to see, what actually happened. - except (socket.timeout, tuf.RepositoryError), e: - print "Download exits with " + str(e) + "! Successfully avoid slow retrieval attack!\n\n" + except tuf.NoWorkingMirrorError, exception: + print "Download exits with " + str(exception) + "! Successfully avoid slow retrieval attack!\n\n" else: urllib.urlretrieve(url, filename) @@ -71,7 +71,7 @@ def _download(url, filename, TUF=False): def test_slow_retrieval_attack(TUF=False, mode=None): - WAIT_TIME = 20 # Number of seconds to wait until download completes. + WAIT_TIME = 60 # Number of seconds to wait until download completes. ERROR_MSG = mode + '\tSlow Retrieval Attack was Successful!\n\n' # Launch the server. @@ -88,7 +88,7 @@ def test_slow_retrieval_attack(TUF=False, mode=None): downloads = os.path.join(root_repo, 'downloads') # Add file to 'repo' directory: {root_repo} - filepath = util_test_tools.add_file_to_repository(reg_repo, 'A'*10) + filepath = util_test_tools.add_file_to_repository(reg_repo, 'A'*30) file_basename = os.path.basename(filepath) url_to_file = url+'reg_repo/'+file_basename downloaded_file = os.path.join(downloads, file_basename) @@ -128,20 +128,19 @@ def test_slow_retrieval_attack(TUF=False, mode=None): + # Stimulates two kinds of slow retrieval attacks. # mode_1: When download begins,the server blocks the download # for a long time by doing nothing before it sends first byte of data. # mode_2: During the download process, the server blocks the download # by sending just several characters every few seconds. try: - #test_slow_retrieval_attack(TUF=False, mode = "mode_1") - pass + test_slow_retrieval_attack(TUF=False, mode = "mode_1") except SlowRetrievalAttackAlert, error: print error try: - #test_slow_retrieval_attack(TUF=False, mode = "mode_2") - pass + test_slow_retrieval_attack(TUF=False, mode = "mode_2") except SlowRetrievalAttackAlert, error: print error @@ -154,3 +153,8 @@ def test_slow_retrieval_attack(TUF=False, mode=None): test_slow_retrieval_attack(TUF=True, mode = "mode_2") except SlowRetrievalAttackAlert, error: print error + + + + + From dc504f0395dc37fa60dcbbbeb90440b2118c857e Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 8 Sep 2013 02:00:45 -0400 Subject: [PATCH 075/119] Better output in slow retrieval test. --- .../test_slow_retrieval_attack.py | 50 ++++++++++++------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/tuf/tests/system_tests/test_slow_retrieval_attack.py b/tuf/tests/system_tests/test_slow_retrieval_attack.py index 09d684ae..b0970106 100755 --- a/tuf/tests/system_tests/test_slow_retrieval_attack.py +++ b/tuf/tests/system_tests/test_slow_retrieval_attack.py @@ -37,14 +37,15 @@ """ +from __future__ import print_function + +from multiprocessing import Process import os -import time -import urllib import random import subprocess -from multiprocessing import Process +import time import tuf -import socket +import urllib import tuf.tests.system_tests.util_test_tools as util_test_tools @@ -59,11 +60,21 @@ def _download(url, filename, TUF=False): if TUF: try: urllib_tuf.urlretrieve(url, filename) - # If timeout or RepositoryError is raised, this means - # that TUF has prevented the slow retrieval attack. Enable - # the logging to see, what actually happened. except tuf.NoWorkingMirrorError, exception: - print "Download exits with " + str(exception) + "! Successfully avoid slow retrieval attack!\n\n" + slow_retrieval = False + for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + if isinstance(mirror_error, tuf.SlowRetrievalError): + slow_retrieval = True + break + + # We must fail due to a slow retrieval error; otherwise we will exit with + # a "successful termination" exit status to indicate that slow retrieval + # detection failed. + if slow_retrieval: + print('TUF stopped the update because it detected slow retrieval.') + else: + print('TUF stopped the update due to something other than slow retrieval.') + sys.exit(0) else: urllib.urlretrieve(url, filename) @@ -72,7 +83,8 @@ def _download(url, filename, TUF=False): def test_slow_retrieval_attack(TUF=False, mode=None): WAIT_TIME = 60 # Number of seconds to wait until download completes. - ERROR_MSG = mode + '\tSlow Retrieval Attack was Successful!\n\n' + ERROR_MSG = 'Slow retrieval attack succeeded (TUF: '+str(TUF)+', mode: '+\ + str(mode)+').' # Launch the server. port = random.randint(30000, 45000) @@ -95,7 +107,6 @@ def test_slow_retrieval_attack(TUF=False, mode=None): if TUF: - print 'TUF ...' tuf_repo = os.path.join(root_repo, 'tuf_repo') # Update TUF metadata before attacker modifies anything. @@ -115,14 +126,13 @@ def test_slow_retrieval_attack(TUF=False, mode=None): proc.start() proc.join(WAIT_TIME) - if proc.exitcode is None: + # In case the process did not exit or successfully exited, we failed. + if not proc.exitcode: proc.terminate() raise SlowRetrievalAttackAlert(ERROR_MSG) finally: - if server_process.returncode is None: - server_process.kill() - + server_process.kill() util_test_tools.cleanup(root_repo, server_proc) @@ -137,22 +147,26 @@ def test_slow_retrieval_attack(TUF=False, mode=None): try: test_slow_retrieval_attack(TUF=False, mode = "mode_1") except SlowRetrievalAttackAlert, error: - print error + print(error) + print() try: test_slow_retrieval_attack(TUF=False, mode = "mode_2") except SlowRetrievalAttackAlert, error: - print error + print(error) + print() try: test_slow_retrieval_attack(TUF=True, mode = "mode_1") except SlowRetrievalAttackAlert, error: - print error + print(error) + print() try: test_slow_retrieval_attack(TUF=True, mode = "mode_2") except SlowRetrievalAttackAlert, error: - print error + print(error) + print() From 9f1e4f60e955c831e63765ed46afdf0eaedf2c09 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 8 Sep 2013 02:14:22 -0400 Subject: [PATCH 076/119] Address comments from 01db53dac620db87801642c688c90b43f3d8832e. --- tuf/__init__.py | 2 +- tuf/conf.py | 8 ++++---- tuf/download.py | 17 ++++++++++------- tuf/tests/system_tests/slow_retrieval_server.py | 1 - 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index 9e66d15b..f1ec70ec 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -188,7 +188,7 @@ class SlowRetrievalError(DownloadError): def __init__(self, cumulative_moving_average_of_speed): self.__cumulative_moving_average_of_speed = \ - cumulative_moving_average_of_speed#bytes/second + cumulative_moving_average_of_speed #bytes/second def __str__(self): return "Cumulative moving average of download speed: "+\ diff --git a/tuf/conf.py b/tuf/conf.py index 042892be..8769dbed 100755 --- a/tuf/conf.py +++ b/tuf/conf.py @@ -39,16 +39,16 @@ # Since the timestamp role does not have signed metadata about itself, we set a # default but sane upper bound for the number of bytes required to download it. -DEFAULT_TIMESTAMP_REQUIRED_LENGTH = 2048#bytes +DEFAULT_TIMESTAMP_REQUIRED_LENGTH = 2048 #bytes # Set a timeout value in seconds (float) for non-blocking socket operations. -SOCKET_TIMEOUT = 1#seconds +SOCKET_TIMEOUT = 1 #seconds # The maximum chunk of data, in bytes, we would download in every round. -CHUNK_SIZE = 8192#bytes +CHUNK_SIZE = 8192 #bytes # The minimum cumulative moving average of download speed (bytes/second) that # must be met to avoid being considered as a slow retrieval attack. -MIN_CUMULATIVE_MOVING_AVERAGE_OF_DOWNLOAD_SPEED = CHUNK_SIZE#bytes/second +MIN_CUMULATIVE_MOVING_AVERAGE_OF_DOWNLOAD_SPEED = CHUNK_SIZE #bytes/second diff --git a/tuf/download.py b/tuf/download.py index a89a4973..370d36f4 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -107,13 +107,14 @@ def __start_clock(self): # We must have reset the clock before this. assert self.__start_time is None + # We are using wall time, so it will be imprecise sometimes. self.__start_time = time.time() - def __stop_clock(self, data_length): + def __stop_clock_and_check_speed(self, data_length): """ Stop the clock and try to detect slow retrieval. @@ -134,9 +135,10 @@ def __stop_clock(self, data_length): """ + # We are using wall time, so it will be imprecise sometimes. stop_time = time.time() # We must have already started the clock. - assert self.__start_time > 0, self.__start_time + assert self.__start_time > 0 time_delta = stop_time-self.__start_time # Reset the clock. self.__start_time = None @@ -153,8 +155,9 @@ def __stop_clock(self, data_length): # If the cumulative moving average of the download speed is below a certain # threshold, we flag this as a possible slow-retrieval attack. This - # threshold will determine our bias: if it is too slow, we will have more - # false negatives; if it is too high, we will have more false positives. + # threshold will determine our bias: if it is too low, we will have more + # false positives; if it is too high, we will have more false negatives. + # Presently, we know that this will punish a server with a slow start. if self.__cumulative_moving_average_of_speed < tuf.conf.MIN_CUMULATIVE_MOVING_AVERAGE_OF_DOWNLOAD_SPEED: raise tuf.SlowRetrievalError(self.__cumulative_moving_average_of_speed) logger.debug('Cumulative moving average of download speed: '+\ @@ -229,15 +232,15 @@ def read(self, size): self.__start_clock() data = self._sock.recv(left) except socket.timeout: - self.__stop_clock(0) + self.__stop_clock_and_check_speed(0) continue except socket.error, e: if e.args[0] == EINTR: - self.__stop_clock(0) + self.__stop_clock_and_check_speed(0) continue raise else: - self.__stop_clock(len(data)) + self.__stop_clock_and_check_speed(len(data)) if not data: break n = len(data) diff --git a/tuf/tests/system_tests/slow_retrieval_server.py b/tuf/tests/system_tests/slow_retrieval_server.py index 6ccb3cd2..dfddab79 100755 --- a/tuf/tests/system_tests/slow_retrieval_server.py +++ b/tuf/tests/system_tests/slow_retrieval_server.py @@ -89,7 +89,6 @@ def get_random_port(): def run(port, test_mode): server_address = ('localhost', port) httpd = HTTPServer_Test(server_address, Handler, test_mode) - print('Slow server is active on port: '+str(port)+' ...') httpd.handle_request() From 3d340de4e8265199a2e51d2d4cf02dbb2853ee4c Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 8 Sep 2013 02:31:50 -0400 Subject: [PATCH 077/119] Recognize a possible endless data attack in the form of invalid JSON metadata. --- tuf/__init__.py | 15 +++++++ tuf/client/updater.py | 17 ++++--- .../system_tests/test_endless_data_attack.py | 45 +++++++++++++------ 3 files changed, 58 insertions(+), 19 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index f1ec70ec..55c07137 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -53,6 +53,21 @@ class FormatError(Error): +class InvalidMetadataJSONError(FormatError): + """Indicate that a metadata file is not valid JSON.""" + + def __init__(self, exception): + # Store the original exception. + self.exception = exception + + def __str__(self): + # Show the original exception. + return str(self.exception) + + + + + class UnsupportedAlgorithmError(Error): """Indicate an error while trying to identify a user-specified algorithm.""" pass diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 15dc1000..ddbcf277 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -695,7 +695,10 @@ def __verify_metadata_file(self, metadata_file_object, metadata_role): In case a targets role has signed for a target it was not delegated to. tuf.FormatError: - In case the metadata file is somehow not valid. + In case the metadata file is valid JSON, but not valid TUF metadata. + + tuf.InvalidMetadataJSONError: + In case the metadata file is not valid JSON. tuf.ReplayedMetadataError: In case the downloaded metadata file is older than the current one. @@ -715,10 +718,14 @@ def __verify_metadata_file(self, metadata_file_object, metadata_role): """ - # Ensure the loaded 'metadata_signable' is properly formatted. - metadata_signable = \ - tuf.util.load_json_string(metadata_file_object.read()) - tuf.formats.check_signable_object_format(metadata_signable) + metadata = metadata_file_object.read() + try: + metadata_signable = tuf.util.load_json_string(metadata) + except Exception, exception: + raise tuf.InvalidMetadataJSONError(exception) + else: + # Ensure the loaded 'metadata_signable' is properly formatted. + tuf.formats.check_signable_object_format(metadata_signable) # Is 'metadata_signable' newer than the currently installed # version? diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index e2472425..0fa49e1b 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -31,6 +31,8 @@ """ +from __future__ import print_function + import os import shutil import urllib @@ -46,10 +48,9 @@ class EndlessDataAttack(Exception): -def _download(url, filename, tuf=False): - if tuf: +def _download(url, filename, TUF=False): + if TUF: urllib_tuf.urlretrieve(url, filename) - else: urllib.urlretrieve(url, filename) @@ -115,13 +116,28 @@ def test_arbitrary_package_attack(TUF=False, TIMESTAMP=False): try: # Client downloads (tries to download) the file. - _download(url=url_to_repo, filename=downloaded_file, tuf=TUF) + _download(url=url_to_repo, filename=downloaded_file, TUF=TUF) - except (tuf.DownloadError, tuf.RepositoryError), e: - # If tuf.DownloadError or tuf.RepositoryError is raised, this means - # that TUF has prevented the download of an unrecognized file. Enable - # logging to see what actually happened. - logger.warn('Download failed: '+repr(e)) + except tuf.NoWorkingMirrorError, exception: + endless_data_attack = False + + for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + # We would get a bad hash error if the file was actually larger than + # the metadata said it was. + if isinstance(mirror_error, tuf.BadHashError): + endless_data_attack = True + break + # We would get invalid metadata JSON if the server deliberately sent + # malformed JSON as part of an endless data attack. + elif isinstance(mirror_error, tuf.InvalidMetadataJSONError): + endless_data_attack = True + break + + # In case we did not detect what was likely an endless data attack, we + # reraise the exception to indicate that endless data attack detection + # failed. + if not endless_data_attack: + raise else: # Check whether the attack succeeded by inspecting the content of the @@ -156,11 +172,12 @@ def test_arbitrary_package_attack(TUF=False, TIMESTAMP=False): try: - # FIXME: This test passes, but not yet because we avoided an endless data - # attack with timestamp metadata, but rather because the timestamp metadata - # is invalid. + # This test fails because the timestamp metadata has been extended with + # random data from its true length, thereby resulting in invalid JSON. test_arbitrary_package_attack(TUF=True, TIMESTAMP=True) - raise EndlessDataAttack('Timestamp metadata is not yet immune from the endless data attack!') except EndlessDataAttack, error: - print('With TUF: '+str(error)) + print('With TUF: '+str(error)) + + + From f245eee70776a3cb6621a0a2d404a57c373701a5 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 8 Sep 2013 12:11:40 -0400 Subject: [PATCH 078/119] Update README to look better on GitHub. Read on for more on 9ff22ddd4e. About 9ff22ddd4e: I was teaching a class of students about using GitHub, and wanted to show that cloning a Git repository off an HTTPS url would not give the cloner write permissions. Unfortunately, I had cloned off the SSH url and was able to push my example modifications without failure. --- README.md | 37 +++++++++++++++++++++++++++++++++++++ README.txt | 40 ---------------------------------------- 2 files changed, 37 insertions(+), 40 deletions(-) create mode 100644 README.md delete mode 100644 README.txt diff --git a/README.md b/README.md new file mode 100644 index 00000000..87f96f42 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# A Framework for Securing Software Update Systems + +TUF (The Update Framework) helps developers secure their new or existing +software update systems. Software update systems are vulnerable to many known +attacks, including those that can result in clients being compromised or +crashed. TUF helps solve this problem by providing a flexible security +framework that can be added to software updaters. + +# What Is a Software Update System? + +Generally, a software update system is an application (or part of an +application) running on a client system that obtains and installs software. +This can include updates to software that is already installed or even +completely new software. + +Three major classes of software update systems are: + +* Application Updaters - which are used by applications use to update +themselves. For example, Firefox updates itself through its own application +updater. + +* Library Package Managers - such as those offered by many programming +languages for installing additional libraries. These are systems such as +Python's pip/easy_install + PyPI, Perl's CPAN, Ruby's Gems, and PHP's PEAR. + +* System Package Managers - used by operating systems to update and install all +of the software on a client system. Debian's APT, Red Hat's YUM, and openSUSE's +YaST are examples of these. + +# Our Approach + +There are literally thousands of different software update systems in common +use today. (In fact the average Windows user has about two dozen different +software updaters on their machine!) + +We are building a library that can be universally (and in most cases +transparently) used to secure software update systems. diff --git a/README.txt b/README.txt deleted file mode 100644 index 953720f9..00000000 --- a/README.txt +++ /dev/null @@ -1,40 +0,0 @@ -A Framework for Securing Software Update Systems ------------------------------------------------- - -TUF (The Update Framework) helps developers secure their new or existing -software update systems. Software update systems are vulnerable to many known -attacks, including those that can result in clients being compromised or crashed. -TUF helps solve this problem by providing a flexible security framework that can -be added to software updaters. - - -What Is a Software Update System? ---------------------------------- - -Generally, a software update system is an application (or part of an application) -running on a client system that obtains and installs software. This can include -updates to software that is already installed or even completely new software. - -Three major classes of software update systems are: - -Application Updaters - which are used by applications use to update themselves. -For example, Firefox updates itself through its own application updater. - -Library Package Managers - such as those offered by many programming languages -for installing additional libraries. These are systems such as Python's -pip/easy_install + PyPI, Perl's CPAN, Ruby's Gems, and PHP's PEAR. - -System Package Managers - used by operating systems to update and install all of -the software on a client system. Debian's APT, Red Hat's YUM, and openSUSE's -YaST are examples of these. - - -Our Approach ------------- - -There are literally thousands of different software update systems in common use -today. (In fact the average Windows user has about two dozen different software -updaters on their machine!) - -We are building a library that can be universally (and in most cases transparently) -used to secure software update systems. From a1ff2b197db433d74e74dfcb8f7f181993e07019 Mon Sep 17 00:00:00 2001 From: dachshund Date: Sun, 8 Sep 2013 17:49:07 -0400 Subject: [PATCH 079/119] Upgrade TUF version number to reflect current status. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bbbc3c85..7523827d 100755 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ setup( name='tuf', - version='0.1', + version='0.7.5', description='A secure updater framework for Python', author='https://www.updateframework.com', author_email='info@updateframework.com', From 21051f497153e38b48af0c38630aee79efee108a Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 9 Sep 2013 10:19:36 -0400 Subject: [PATCH 080/119] Fix bug in test_signercli.py and update test case following PyCrypto changes A bug in test_signercli.py prevents required keys from loading. A password for a test keyid is modified but not properly restored. This particular bug made it difficulty in adding a new feature (i.e., derived keys) and updating the unit tests. We need to simplify the unit tests, specifically the removal of side effects, monkey patches, pseudo repositories/data structures, and dependency on scripts that expect user input. 'signerlib.py' should be called instead of the signer tools (e.g., signercli.py). --- tuf/repo/signercli.py | 2 +- tuf/tests/test_signercli.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tuf/repo/signercli.py b/tuf/repo/signercli.py index 48182e33..5787b420 100755 --- a/tuf/repo/signercli.py +++ b/tuf/repo/signercli.py @@ -366,7 +366,7 @@ def _get_role_config_keyids(config_filepath, keystore_directory, role): # Ensure we loaded all the keyids. for keyid in value['keyids']: if keyid not in role_keyids: - raise tuf.Error('Could not load a required role key.') + raise tuf.Error('Could not load a required role key: '+keyid+'.') if not role_keyids: raise tuf.Error('Could not load the required keys for '+role) diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index 765035e0..297eee18 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -365,7 +365,7 @@ def _mock_get_password(msg): # Test: invalid password. keyids = ['quit', self.rsa_keyids[0]] - saved_pw = self.rsa_passwords[keyid] + saved_pw = self.rsa_passwords[self.rsa_keyids[0]] # Invalid password self.rsa_passwords[self.rsa_keyids[0]] = self.random_string() @@ -475,11 +475,10 @@ def test_2__get_role_config_keyids(self): # Patch '_get_password' method. self.get_passwords() - # TESTS for role in self.role_list: # Test: normal cases. - #keystore.clear_keystore() + keystore.clear_keystore() signercli._get_role_config_keyids(config_filepath, keystore_dir, role) # Test: incorrect passwords. @@ -634,10 +633,15 @@ def _mock_get_password(msg, confirm=False, old_pw=old_password, # TESTS # Test: normal case. + # Verify that the derived key is modified. A new salt is generated, so + # we cannot predict or verify a specific derived key corresponding for + # the new password. Save the derived key for 'test_keyid' and check that + # is updated. + old_derived_key = keystore._derived_keys[test_keyid] signercli.change_password(keystore_dir) # Verify password change. - self.assertEqual(keystore._key_passwords[test_keyid], new_password) + self.assertNotEqual(keystore._derived_keys[test_keyid], old_derived_key) # Test: non-existing keyid. keystore.clear_keystore() From 9a3820bfcfac07bb903ed2d5730ce59af76b8146 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 11:39:39 -0400 Subject: [PATCH 081/119] Fix #42. --- tuf/__init__.py | 9 ++- tuf/conf.py | 9 ++- tuf/download.py | 57 ++++++++++--------- .../test_slow_retrieval_attack.py | 2 + 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index 55c07137..d2031fb4 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -201,13 +201,12 @@ class DownloadLengthMismatchError(DownloadError): class SlowRetrievalError(DownloadError): """"Indicate that downloading a file took an unreasonably long time.""" - def __init__(self, cumulative_moving_average_of_speed): - self.__cumulative_moving_average_of_speed = \ - cumulative_moving_average_of_speed #bytes/second + def __init__(self, average_download_speed): + self.__average_download_speed = average_download_speed #bytes/second def __str__(self): - return "Cumulative moving average of download speed: "+\ - str(self.__cumulative_moving_average_of_speed)+" bytes/second" + return "Average download speed: "+str(self.__average_download_speed)+\ + " bytes/second" diff --git a/tuf/conf.py b/tuf/conf.py index 8769dbed..de9ad7f7 100755 --- a/tuf/conf.py +++ b/tuf/conf.py @@ -47,8 +47,11 @@ # The maximum chunk of data, in bytes, we would download in every round. CHUNK_SIZE = 8192 #bytes -# The minimum cumulative moving average of download speed (bytes/second) that -# must be met to avoid being considered as a slow retrieval attack. -MIN_CUMULATIVE_MOVING_AVERAGE_OF_DOWNLOAD_SPEED = CHUNK_SIZE #bytes/second +# The minimum average of download speed (bytes/second) that must be met to +# avoid being considered as a slow retrieval attack. +MIN_AVERAGE_DOWNLOAD_SPEED = CHUNK_SIZE #bytes/second + +# The time (in seconds) we ignore a server with a slow initial retrieval speed. +SLOW_START_GRACE_PERIOD = 30 #seconds diff --git a/tuf/download.py b/tuf/download.py index 370d36f4..5f1f2f30 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -75,10 +75,10 @@ def __init__(self, sock, mode='rb', bufsize=-1, close=False): super(SaferSocketFileObject, self).__init__(sock, mode=mode, bufsize=bufsize, close=close) - # Measure the cumulative moving average of download speed. - self.__cumulative_moving_average_of_speed = 0 - # Count the number of socket receive operations. - self.__number_of_receive_operations = 0 + # Count the number of bytes received with this socket. + self.__number_of_bytes_received = 0 + # Count the seconds spent receiving with this socket. + self.__seconds_spent_receiving = 0 # Remember the time a clock was started. self.__start_time = None @@ -95,7 +95,8 @@ def __start_clock(self): None. - AssertionError: When any internal condition is not true. + AssertionError: + When any internal condition is not true. Start time is kept inside this object. @@ -120,12 +121,15 @@ def __stop_clock_and_check_speed(self, data_length): Stop the clock and try to detect slow retrieval. - data_length: A nonnegative integer indicating the size of data retrieved. + data_length: + A nonnegative integer indicating the size of data retrieved in bytes. - tuf.SlowRetrievalError: When slow retrieval is detected. + tuf.SlowRetrievalError: + When slow retrieval is detected. - AssertionError: When any internal condition is not true. + AssertionError: + When any internal condition is not true. Start time is cleared inside this object. @@ -142,27 +146,26 @@ def __stop_clock_and_check_speed(self, data_length): time_delta = stop_time-self.__start_time # Reset the clock. self.__start_time = None - speed = data_length/time_delta - logger.debug('Speed: '+str(speed)+' ('+str(data_length)+'/'+\ - str(time_delta)+') bytes/second') - # Measure the cumulative moving average of the download speed. - #https://en.wikipedia.org/wiki/Moving_average#Cumulative_moving_average - self.__number_of_receive_operations += 1 - numerator = speed+((self.__number_of_receive_operations-1)*self.__cumulative_moving_average_of_speed) - denominator = self.__number_of_receive_operations - self.__cumulative_moving_average_of_speed = numerator/denominator + # Measure the average download speed. + self.__number_of_bytes_received += data_length + self.__seconds_spent_receiving += time_delta + average_download_speed = \ + self.__number_of_bytes_received/self.__seconds_spent_receiving - # If the cumulative moving average of the download speed is below a certain - # threshold, we flag this as a possible slow-retrieval attack. This - # threshold will determine our bias: if it is too low, we will have more - # false positives; if it is too high, we will have more false negatives. - # Presently, we know that this will punish a server with a slow start. - if self.__cumulative_moving_average_of_speed < tuf.conf.MIN_CUMULATIVE_MOVING_AVERAGE_OF_DOWNLOAD_SPEED: - raise tuf.SlowRetrievalError(self.__cumulative_moving_average_of_speed) - logger.debug('Cumulative moving average of download speed: '+\ - str(self.__cumulative_moving_average_of_speed)+\ - ' bytes/second') + # If the average download speed is below a certain threshold, we flag this + # as a possible slow-retrieval attack. This threshold will determine our + # bias: if it is too low, we will have more false positives; if it is too + # high, we will have more false negatives. + if average_download_speed < tuf.conf.MIN_AVERAGE_DOWNLOAD_SPEED: + if self.__seconds_spent_receiving <= tuf.conf.SLOW_START_GRACE_PERIOD: + logger.debug('Slow average download speed: '+\ + str(average_download_speed)+' bytes/second') + else: + raise tuf.SlowRetrievalError(average_download_speed) + else: + logger.debug('Good average download speed: '+\ + str(average_download_speed)+' bytes/second') diff --git a/tuf/tests/system_tests/test_slow_retrieval_attack.py b/tuf/tests/system_tests/test_slow_retrieval_attack.py index b0970106..92b46da7 100755 --- a/tuf/tests/system_tests/test_slow_retrieval_attack.py +++ b/tuf/tests/system_tests/test_slow_retrieval_attack.py @@ -43,6 +43,7 @@ import os import random import subprocess +import sys import time import tuf import urllib @@ -72,6 +73,7 @@ def _download(url, filename, TUF=False): # detection failed. if slow_retrieval: print('TUF stopped the update because it detected slow retrieval.') + sys.exit(-1) else: print('TUF stopped the update due to something other than slow retrieval.') sys.exit(0) From f2a6d848a96df28359044799a19809b41136429a Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 9 Sep 2013 12:12:06 -0400 Subject: [PATCH 082/119] Set a lower PBKDF count for some of the unit tests The default tuf.keystore._PBKDF_ITERATIONS (90,510) slow down the unit tests considerably. Reduce it temporarily for the unit tests depending on it and that do not test the strength of derived keys. All 164 tests now run in approximately 2 minutes, down from approximately 17 minutes. The lowered PBKDF count set for the unit tests is equal to the previous key derivation count used by evpy. --- tuf/tests/system_tests/test_delegations.py | 3 +++ tuf/tests/unittest_toolbox.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/tuf/tests/system_tests/test_delegations.py b/tuf/tests/system_tests/test_delegations.py index 35a56b8e..4312d439 100755 --- a/tuf/tests/system_tests/test_delegations.py +++ b/tuf/tests/system_tests/test_delegations.py @@ -33,6 +33,9 @@ import util_test_tools version = 1 +# Modify the number of iterations (from the higher default count) so the unit +# tests run faster. +keystore._PBKDF2_ITERATIONS = 1000 class TestDelegationFunctions(unittest.TestCase): diff --git a/tuf/tests/unittest_toolbox.py b/tuf/tests/unittest_toolbox.py index e7ca2894..0ff48392 100755 --- a/tuf/tests/unittest_toolbox.py +++ b/tuf/tests/unittest_toolbox.py @@ -30,6 +30,10 @@ import tuf.rsa_key as rsa_key import tuf.repo.keystore as keystore +# Modify the number of iterations (from the higher default count) so the unit +# tests run faster. +keystore._PBKDF2_ITERATIONS = 1000 + class Modified_TestCase(unittest.TestCase): """ From 772905a67e42b8953fcf9fb03918ca3dde43e24d Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 13:50:11 -0400 Subject: [PATCH 083/119] Accurately account for slow server start grace period. --- tuf/download.py | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/tuf/download.py b/tuf/download.py index 5f1f2f30..dfdb33b5 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -29,7 +29,7 @@ import logging import os.path import socket -import time +import timeit import tuf import tuf.conf @@ -77,8 +77,11 @@ def __init__(self, sock, mode='rb', bufsize=-1, close=False): # Count the number of bytes received with this socket. self.__number_of_bytes_received = 0 - # Count the seconds spent receiving with this socket. - self.__seconds_spent_receiving = 0 + # Count the seconds spent receiving with this socket. Tolerate servers with + # a slow start by ignoring their delivery speed for + # tuf.conf.SLOW_START_GRACE_PERIOD seconds. + assert tuf.conf.SLOW_START_GRACE_PERIOD > 0 + self.__seconds_spent_receiving = -tuf.conf.SLOW_START_GRACE_PERIOD # Remember the time a clock was started. self.__start_time = None @@ -108,8 +111,8 @@ def __start_clock(self): # We must have reset the clock before this. assert self.__start_time is None - # We are using wall time, so it will be imprecise sometimes. - self.__start_time = time.time() + # We use (platform-specific) wall time, so it will be imprecise sometimes. + self.__start_time = timeit.default_timer() @@ -139,8 +142,8 @@ def __stop_clock_and_check_speed(self, data_length): """ - # We are using wall time, so it will be imprecise sometimes. - stop_time = time.time() + # We use (platform-specific) wall time, so it will be imprecise sometimes. + stop_time = timeit.default_timer() # We must have already started the clock. assert self.__start_time > 0 time_delta = stop_time-self.__start_time @@ -150,22 +153,24 @@ def __stop_clock_and_check_speed(self, data_length): # Measure the average download speed. self.__number_of_bytes_received += data_length self.__seconds_spent_receiving += time_delta - average_download_speed = \ - self.__number_of_bytes_received/self.__seconds_spent_receiving - # If the average download speed is below a certain threshold, we flag this - # as a possible slow-retrieval attack. This threshold will determine our - # bias: if it is too low, we will have more false positives; if it is too - # high, we will have more false negatives. - if average_download_speed < tuf.conf.MIN_AVERAGE_DOWNLOAD_SPEED: - if self.__seconds_spent_receiving <= tuf.conf.SLOW_START_GRACE_PERIOD: - logger.debug('Slow average download speed: '+\ - str(average_download_speed)+' bytes/second') + if self.__seconds_spent_receiving > 0: + average_download_speed = \ + self.__number_of_bytes_received/self.__seconds_spent_receiving + + # If the average download speed is below a certain threshold, we flag this + # as a possible slow-retrieval attack. This threshold will determine our + # bias: if it is too low, we will have more false positives; if it is too + # high, we will have more false negatives. + if average_download_speed < tuf.conf.MIN_AVERAGE_DOWNLOAD_SPEED: + raise tuf.SlowRetrievalError(average_download_speed) else: - raise tuf.SlowRetrievalError(average_download_speed) + logger.debug('Good average download speed: '+\ + str(average_download_speed)+' bytes/second') else: - logger.debug('Good average download speed: '+\ - str(average_download_speed)+' bytes/second') + logger.debug('Ignoring average download speed for another: '+\ + str(-self.__seconds_spent_receiving)+' seconds') + From 5d78f269075fec64b123d2c6aa2df0aa6ac0799f Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 9 Sep 2013 14:16:27 -0400 Subject: [PATCH 084/119] Remove unit and system tests from setup.py --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 7523827d..5fe1d03b 100755 --- a/setup.py +++ b/setup.py @@ -75,9 +75,7 @@ 'tuf.interposition', 'tuf.pushtools', 'tuf.pushtools.transfer', - 'tuf.repo', - 'tuf.tests', - 'tuf.tests.system_tests' + 'tuf.repo' ], scripts=[ 'tuf/repo/quickstart.py', From c12cb52aa36fd19e5b022ad4b4b143e22424c83e Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 9 Sep 2013 14:40:08 -0400 Subject: [PATCH 085/119] Add pycrypto requirement to setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5fe1d03b..ff1158cf 100755 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ """ -from distutils.core import setup +from setuptools import setup setup( name='tuf', @@ -67,6 +67,7 @@ author='https://www.updateframework.com', author_email='info@updateframework.com', url='https://www.updateframework.com', + install_requires=['pycrypto>2.0'], packages=[ 'evpy', 'tuf', From 1e245237ee76e8d6fc3dcfd2442a65c9a62a31e6 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 15:48:43 -0400 Subject: [PATCH 086/119] Fix #74. --- .../system_tests/test_endless_data_attack.py | 89 ++++++++++--------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/system_tests/test_endless_data_attack.py index 0fa49e1b..fecb8ab5 100755 --- a/tuf/tests/system_tests/test_endless_data_attack.py +++ b/tuf/tests/system_tests/test_endless_data_attack.py @@ -34,14 +34,13 @@ from __future__ import print_function import os -import shutil import urllib -import tempfile import util_test_tools import tuf from tuf.interposition import urllib_tuf -from tuf.log import logger + + class EndlessDataAttack(Exception): pass @@ -79,12 +78,17 @@ def test_arbitrary_package_attack(TUF=False, TIMESTAMP=False): downloads = os.path.join(root_repo, 'downloads') tuf_targets = os.path.join(tuf_repo, 'targets') + # Original data. + INTENDED_DATA = 'Test A' + # Add a file to 'repo' directory: {root_repo} - filepath = util_test_tools.add_file_to_repository(reg_repo, 'Test A') + filepath = util_test_tools.add_file_to_repository(reg_repo, INTENDED_DATA) file_basename = os.path.basename(filepath) url_to_repo = url+'reg_repo/'+file_basename downloaded_file = os.path.join(downloads, file_basename) - endless_data = 'A'*100000 + # We do not deliver truly endless data, but we will extend the original + # file by many bytes. + noisy_data = 'X'*100000 if TUF: @@ -99,55 +103,57 @@ def test_arbitrary_package_attack(TUF=False, TIMESTAMP=False): # Attacker modifies the file at the targets repository. target = os.path.join(tuf_targets, file_basename) - util_test_tools.modify_file_at_repository(target, endless_data) + original_data = util_test_tools.read_file_content(target) + larger_original_data = original_data + noisy_data + util_test_tools.modify_file_at_repository(target, larger_original_data) # Attacker modifies the timestamp.txt metadata. if TIMESTAMP: metadata = os.path.join(tuf_repo, 'metadata') timestamp = os.path.join(metadata, 'timestamp.txt') - # FIXME: This does not correctly "patch" the timestamp metadata. - util_test_tools.modify_file_at_repository(timestamp, endless_data) + original_data = util_test_tools.read_file_content(timestamp) + larger_original_data = original_data + noisy_data + util_test_tools.modify_file_at_repository(timestamp, + larger_original_data) # Attacker modifies the file at the regular repository. - util_test_tools.modify_file_at_repository(filepath, endless_data) + original_data = util_test_tools.read_file_content(filepath) + larger_original_data = original_data + noisy_data + util_test_tools.modify_file_at_repository(filepath, larger_original_data) # End Setup. + # Client downloads (tries to download) the file. try: - # Client downloads (tries to download) the file. _download(url=url_to_repo, filename=downloaded_file, TUF=TUF) + except Exception, exception: + # Because we are extending the true timestamp TUF metadata with invalid + # JSON, we except to catch an error about invalid metadata JSON. + if TUF and TIMESTAMP: + endless_data_attack = False - except tuf.NoWorkingMirrorError, exception: - endless_data_attack = False + for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + if isinstance(mirror_error, tuf.InvalidMetadataJSONError): + endless_data_attack = True + break - for mirror_url, mirror_error in exception.mirror_errors.iteritems(): - # We would get a bad hash error if the file was actually larger than - # the metadata said it was. - if isinstance(mirror_error, tuf.BadHashError): - endless_data_attack = True - break - # We would get invalid metadata JSON if the server deliberately sent - # malformed JSON as part of an endless data attack. - elif isinstance(mirror_error, tuf.InvalidMetadataJSONError): - endless_data_attack = True - break + # In case we did not detect what was likely an endless data attack, we + # reraise the exception to indicate that endless data attack detection + # failed. + if not endless_data_attack: raise + else: raise - # In case we did not detect what was likely an endless data attack, we - # reraise the exception to indicate that endless data attack detection - # failed. - if not endless_data_attack: - raise - - else: + # When we test downloading "endless" timestamp with TUF, we want to skip + # the following test because downloading the timestamp should have failed. + if not (TUF and TIMESTAMP): # Check whether the attack succeeded by inspecting the content of the # update. The update should contain 'Test A'. Technically it suffices # to check whether the file was downloaded or not. downloaded_content = util_test_tools.read_file_content(downloaded_file) - if 'Test A' != downloaded_content: + if downloaded_content != INTENDED_DATA: raise EndlessDataAttack(ERROR_MSG) - finally: util_test_tools.cleanup(root_repo, server_proc) @@ -157,27 +163,26 @@ def test_arbitrary_package_attack(TUF=False, TIMESTAMP=False): try: test_arbitrary_package_attack(TUF=False, TIMESTAMP=False) - except EndlessDataAttack, error: - print('Without TUF: '+str(error)) - - + print('Endless data attack worked on download without TUF!') try: test_arbitrary_package_attack(TUF=True, TIMESTAMP=False) - except EndlessDataAttack, error: - print('With TUF: '+str(error)) - - + print('Endless data attack worked on download without TUF!') + print(str(error)) +else: + print('Endless data attack did not work on download with TUF!') try: # This test fails because the timestamp metadata has been extended with # random data from its true length, thereby resulting in invalid JSON. test_arbitrary_package_attack(TUF=True, TIMESTAMP=True) - except EndlessDataAttack, error: - print('With TUF: '+str(error)) + print('Endless data attack worked on download without TUF!') + print(str(error)) +else: + print('Endless data attack did not work on download with TUF!') From a3c63a3678e238001fdc9b175fb39be5146daaa8 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 16:05:51 -0400 Subject: [PATCH 087/119] Remove console logging from the wrong place. --- tuf/interposition/utility.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tuf/interposition/utility.py b/tuf/interposition/utility.py index b70fd702..8e1c4cd9 100644 --- a/tuf/interposition/utility.py +++ b/tuf/interposition/utility.py @@ -20,7 +20,6 @@ class Logger(object): """A static logging object for tuf.interposition.""" - tuf.log.add_console_handler() __logger = logging.getLogger("tuf.interposition") From baf4d5b26929a0453c68c69f716c196990cbb5d4 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 16:51:01 -0400 Subject: [PATCH 088/119] Relative, instead of absolute, imports in unit tests. --- tuf/tests/repository_setup.py | 2 +- tuf/tests/test_download.py | 2 +- tuf/tests/test_mirrors.py | 6 +++--- tuf/tests/test_push.py | 2 +- tuf/tests/test_quickstart.py | 4 ++-- tuf/tests/test_signercli.py | 2 +- tuf/tests/test_signerlib.py | 4 ++-- tuf/tests/test_updater.py | 4 ++-- tuf/tests/test_util.py | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tuf/tests/repository_setup.py b/tuf/tests/repository_setup.py index 066ca61e..b79e5b57 100755 --- a/tuf/tests/repository_setup.py +++ b/tuf/tests/repository_setup.py @@ -28,7 +28,7 @@ import tuf.repo.keystore as keystore import tuf.repo.signerlib as signerlib import tuf.repo.signercli as signercli -import tuf.tests.unittest_toolbox as unittest_toolbox +import unittest_toolbox as unittest_toolbox diff --git a/tuf/tests/test_download.py b/tuf/tests/test_download.py index 77d0d5fd..c93038dc 100755 --- a/tuf/tests/test_download.py +++ b/tuf/tests/test_download.py @@ -37,7 +37,7 @@ import tuf.conf as conf import tuf.download as download import tuf.log -import tuf.tests.unittest_toolbox as unittest_toolbox +import unittest_toolbox as unittest_toolbox logger = logging.getLogger('tuf.test_download') diff --git a/tuf/tests/test_mirrors.py b/tuf/tests/test_mirrors.py index 4afca814..f762a809 100755 --- a/tuf/tests/test_mirrors.py +++ b/tuf/tests/test_mirrors.py @@ -21,15 +21,15 @@ import tuf import tuf.formats as formats import tuf.mirrors as mirrors -import tuf.tests.unittest_toolbox +import unittest_toolbox -class TestMirrors(tuf.tests.unittest_toolbox.Modified_TestCase): +class TestMirrors(unittest_toolbox.Modified_TestCase): def setUp(self): - tuf.tests.unittest_toolbox.Modified_TestCase.setUp(self) + unittest_toolbox.Modified_TestCase.setUp(self) self.mirrors = \ {'mirror1': {'url_prefix' : 'http://mirror1.com', diff --git a/tuf/tests/test_push.py b/tuf/tests/test_push.py index c491b22f..a48bb5aa 100755 --- a/tuf/tests/test_push.py +++ b/tuf/tests/test_push.py @@ -28,7 +28,7 @@ import tuf.pushtools.push as push import tuf.pushtools.transfer.scp as scp import tuf.pushtools.pushtoolslib as pushtoolslib -import tuf.tests.system_tests.util_test_tools as util_test_tools +import system_tests.util_test_tools as util_test_tools logger = logging.getLogger('tuf.test_push') diff --git a/tuf/tests/test_quickstart.py b/tuf/tests/test_quickstart.py index a211f905..42ef8771 100755 --- a/tuf/tests/test_quickstart.py +++ b/tuf/tests/test_quickstart.py @@ -30,10 +30,10 @@ import tuf.log import tuf.repo.quickstart as quickstart import tuf.util -import tuf.tests.unittest_toolbox +import unittest_toolbox logger = logging.getLogger('tuf.test_quickstart') -unit_tbox = tuf.tests.unittest_toolbox.Modified_TestCase +unit_tbox = unittest_toolbox.Modified_TestCase logger.info('from test_quickstart') diff --git a/tuf/tests/test_signercli.py b/tuf/tests/test_signercli.py index 297eee18..378cc0f7 100755 --- a/tuf/tests/test_signercli.py +++ b/tuf/tests/test_signercli.py @@ -52,7 +52,7 @@ class guarantees the order of unit tests. So that, 'test_something_A' import tuf.repo.signercli as signercli # Helper module unittest_toolbox.py -import tuf.tests.unittest_toolbox as unittest_toolbox +import unittest_toolbox as unittest_toolbox logger = logging.getLogger('tuf.test_signercli') diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/test_signerlib.py index 38e2f6bf..5efdfa3c 100755 --- a/tuf/tests/test_signerlib.py +++ b/tuf/tests/test_signerlib.py @@ -61,12 +61,12 @@ import tuf.repo.signerlib as signerlib import tuf.repo.keystore -import tuf.tests.unittest_toolbox +import unittest_toolbox logger = logging.getLogger('tuf.test_signerlib') # 'unittest_toolbox.Modified_TestCase' is too long, I'll set it to 'unit_tbox'. -unit_tbox = tuf.tests.unittest_toolbox.Modified_TestCase +unit_tbox = unittest_toolbox.Modified_TestCase diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index 5dd7ed58..afa6606c 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -53,8 +53,8 @@ class guarantees the order of unit tests. So that, 'test_something_A' import tuf.repo.keystore as keystore import tuf.repo.signerlib as signerlib import tuf.roledb -import tuf.tests.repository_setup as setup -import tuf.tests.unittest_toolbox as unittest_toolbox +import repository_setup as setup +import unittest_toolbox as unittest_toolbox import tuf.util logger = logging.getLogger('tuf.test_updater') diff --git a/tuf/tests/test_util.py b/tuf/tests/test_util.py index 561e1cc7..dd6fc5d7 100755 --- a/tuf/tests/test_util.py +++ b/tuf/tests/test_util.py @@ -28,7 +28,7 @@ import tuf.log import tuf.hash import tuf.util as util -import tuf.tests.unittest_toolbox as unittest_toolbox +import unittest_toolbox as unittest_toolbox logger = logging.getLogger('tuf.test_util') From 7dbf4d459a7ce152a6a80633c258c87d179f4de6 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 16:54:27 -0400 Subject: [PATCH 089/119] Use relative, instead of absolute, imports in system tests. --- tuf/tests/system_tests/test_replay_attack.py | 2 +- tuf/tests/system_tests/test_slow_retrieval_attack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tuf/tests/system_tests/test_replay_attack.py b/tuf/tests/system_tests/test_replay_attack.py index 1c190dfa..de05b42e 100755 --- a/tuf/tests/system_tests/test_replay_attack.py +++ b/tuf/tests/system_tests/test_replay_attack.py @@ -38,7 +38,7 @@ import urllib import tempfile -import tuf.tests.system_tests.util_test_tools as util_test_tools +import util_test_tools as util_test_tools from tuf.interposition import urllib_tuf diff --git a/tuf/tests/system_tests/test_slow_retrieval_attack.py b/tuf/tests/system_tests/test_slow_retrieval_attack.py index 92b46da7..da81343c 100755 --- a/tuf/tests/system_tests/test_slow_retrieval_attack.py +++ b/tuf/tests/system_tests/test_slow_retrieval_attack.py @@ -49,7 +49,7 @@ import urllib -import tuf.tests.system_tests.util_test_tools as util_test_tools +import util_test_tools as util_test_tools from tuf.interposition import urllib_tuf From 6f19f94867e442cabb2345809b8ba7e5ee165979 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 17:00:11 -0400 Subject: [PATCH 090/119] Rename system->integration tests. --- .gitignore | 1 + tuf/tests/{system_tests => integration_tests}/__init__.py | 0 .../{system_tests => integration_tests}/slow_retrieval_server.py | 0 .../test_arbitrary_package_attack.py | 0 .../{system_tests => integration_tests}/test_delegations.py | 0 .../test_endless_data_attack.py | 0 .../test_extraneous_dependencies_attack.py | 0 .../test_indefinite_freeze_attack.py | 0 .../test_mix_and_match_attack.py | 0 .../{system_tests => integration_tests}/test_replay_attack.py | 0 .../test_slow_retrieval_attack.py | 0 .../{system_tests => integration_tests}/test_util_test_tools.py | 0 tuf/tests/{system_tests => integration_tests}/util_test_tools.py | 0 13 files changed, 1 insertion(+) rename tuf/tests/{system_tests => integration_tests}/__init__.py (100%) rename tuf/tests/{system_tests => integration_tests}/slow_retrieval_server.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_arbitrary_package_attack.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_delegations.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_endless_data_attack.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_extraneous_dependencies_attack.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_indefinite_freeze_attack.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_mix_and_match_attack.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_replay_attack.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_slow_retrieval_attack.py (100%) rename tuf/tests/{system_tests => integration_tests}/test_util_test_tools.py (100%) rename tuf/tests/{system_tests => integration_tests}/util_test_tools.py (100%) diff --git a/.gitignore b/.gitignore index e7ef56c9..0f43b246 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ build/* *.session *.swo *.swp +tuf.egg-info diff --git a/tuf/tests/system_tests/__init__.py b/tuf/tests/integration_tests/__init__.py similarity index 100% rename from tuf/tests/system_tests/__init__.py rename to tuf/tests/integration_tests/__init__.py diff --git a/tuf/tests/system_tests/slow_retrieval_server.py b/tuf/tests/integration_tests/slow_retrieval_server.py similarity index 100% rename from tuf/tests/system_tests/slow_retrieval_server.py rename to tuf/tests/integration_tests/slow_retrieval_server.py diff --git a/tuf/tests/system_tests/test_arbitrary_package_attack.py b/tuf/tests/integration_tests/test_arbitrary_package_attack.py similarity index 100% rename from tuf/tests/system_tests/test_arbitrary_package_attack.py rename to tuf/tests/integration_tests/test_arbitrary_package_attack.py diff --git a/tuf/tests/system_tests/test_delegations.py b/tuf/tests/integration_tests/test_delegations.py similarity index 100% rename from tuf/tests/system_tests/test_delegations.py rename to tuf/tests/integration_tests/test_delegations.py diff --git a/tuf/tests/system_tests/test_endless_data_attack.py b/tuf/tests/integration_tests/test_endless_data_attack.py similarity index 100% rename from tuf/tests/system_tests/test_endless_data_attack.py rename to tuf/tests/integration_tests/test_endless_data_attack.py diff --git a/tuf/tests/system_tests/test_extraneous_dependencies_attack.py b/tuf/tests/integration_tests/test_extraneous_dependencies_attack.py similarity index 100% rename from tuf/tests/system_tests/test_extraneous_dependencies_attack.py rename to tuf/tests/integration_tests/test_extraneous_dependencies_attack.py diff --git a/tuf/tests/system_tests/test_indefinite_freeze_attack.py b/tuf/tests/integration_tests/test_indefinite_freeze_attack.py similarity index 100% rename from tuf/tests/system_tests/test_indefinite_freeze_attack.py rename to tuf/tests/integration_tests/test_indefinite_freeze_attack.py diff --git a/tuf/tests/system_tests/test_mix_and_match_attack.py b/tuf/tests/integration_tests/test_mix_and_match_attack.py similarity index 100% rename from tuf/tests/system_tests/test_mix_and_match_attack.py rename to tuf/tests/integration_tests/test_mix_and_match_attack.py diff --git a/tuf/tests/system_tests/test_replay_attack.py b/tuf/tests/integration_tests/test_replay_attack.py similarity index 100% rename from tuf/tests/system_tests/test_replay_attack.py rename to tuf/tests/integration_tests/test_replay_attack.py diff --git a/tuf/tests/system_tests/test_slow_retrieval_attack.py b/tuf/tests/integration_tests/test_slow_retrieval_attack.py similarity index 100% rename from tuf/tests/system_tests/test_slow_retrieval_attack.py rename to tuf/tests/integration_tests/test_slow_retrieval_attack.py diff --git a/tuf/tests/system_tests/test_util_test_tools.py b/tuf/tests/integration_tests/test_util_test_tools.py similarity index 100% rename from tuf/tests/system_tests/test_util_test_tools.py rename to tuf/tests/integration_tests/test_util_test_tools.py diff --git a/tuf/tests/system_tests/util_test_tools.py b/tuf/tests/integration_tests/util_test_tools.py similarity index 100% rename from tuf/tests/system_tests/util_test_tools.py rename to tuf/tests/integration_tests/util_test_tools.py From f72f5751d1893c402933a2e320f7f75f85003b59 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 21:21:32 -0400 Subject: [PATCH 091/119] Redundantly verify file length in updater. You may argue that the redundancy is unnecessary (pun intended), but it is there because redundancy means one safety check will work where another fails. I introduced this redundant file length check because the updater unit test is mocking the download functions, which means that file length checks in the download functions are being bypassed. Redundancy is a good thing for safety. --- tuf/__init__.py | 9 ++- tuf/client/updater.py | 135 ++++++++++++++++++++++++++++++-------- tuf/download.py | 2 +- tuf/tests/test_updater.py | 14 ++-- tuf/tests/test_util.py | 9 +-- tuf/util.py | 27 +++++++- 6 files changed, 153 insertions(+), 43 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index d2031fb4..f44657e6 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -192,7 +192,14 @@ class DownloadError(Error): class DownloadLengthMismatchError(DownloadError): """Indicate that a mismatch of lengths was seen while downloading a file.""" - pass + + def __init__(self, expected_length, observed_length): + self.expected_length = expected_length #bytes + self.observed_length = observed_length #bytes + + def __str__(self): + return 'Observed length ('+str(self.observed_length)+\ + ') <= expected length ('+str(self.expected_length)+')' diff --git a/tuf/client/updater.py b/tuf/client/updater.py index ddbcf277..e2e90207 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -592,17 +592,17 @@ def refresh(self): - def __check_hashes(self, input_file, trusted_hashes): + def __check_hashes(self, file_object, trusted_hashes): """ A helper function that verifies multiple secure hashes of the downloaded file. If any of these fail it raises an exception. This is to conform with the TUF specs, which support clients with different hashing algorithms. The 'hash.py' module is used to compute the hashes of the - 'input_file'. + 'file_object'. - input_file: + file_object: A file-like object. trusted_hashes: @@ -624,7 +624,7 @@ def __check_hashes(self, input_file, trusted_hashes): # any of the hashes are incorrect and return if all are correct. for algorithm, trusted_hash in trusted_hashes.items(): digest_object = tuf.hash.digest(algorithm) - digest_object.update(input_file.read()) + digest_object.update(file_object.read()) computed_hash = digest_object.hexdigest() if trusted_hash != computed_hash: raise tuf.BadHashError('Hashes do not match! Expected '+ @@ -636,6 +636,74 @@ def __check_hashes(self, input_file, trusted_hashes): + def __hard_check_length(self, file_object, trusted_length): + """ + + A helper function that checks the expected length of a file-like object. + The length of the file must be strictly equal to the expected length. + This is a deliberately redundant implementation designed to complement + tuf.download._check_downloaded_length(). + + + file_object: + A file-like object. + + trusted_length: + A nonnegative integer that is the expected length of the file. + + + tuf.DownloadLengthMismatchError, if the lengths don't match. + + + None. + + + None. + + """ + + observed_length = len(file_object) + if observed_length != trusted_length: + raise tuf.DownloadLengthMismatchError(trusted_length, observed_length) + + + + + + def __soft_check_length(self, file_object, trusted_length): + """ + + A helper function that checks the expected length of a file-like object. + The length of the file must be less than or equal to the expected length. + This is a deliberately redundant implementation designed to complement + tuf.download._check_downloaded_length(). + + + file_object: + A file-like object. + + trusted_length: + A nonnegative integer that is the expected length of the file. + + + tuf.DownloadLengthMismatchError, if the lengths don't match. + + + None. + + + None. + + """ + + observed_length = len(file_object) + if observed_length > trusted_length: + raise tuf.DownloadLengthMismatchError(trusted_length, observed_length) + + + + + def get_target_file(self, target_filepath, file_length, file_hashes): """ @@ -667,21 +735,25 @@ def get_target_file(self, target_filepath, file_length, file_hashes): """ - def verify_target_file(target_file_object): - # Every target file must have its hashes inspected. + def verify_decompressed_target_file(target_file_object): + # Every target file must have its length and hashes inspected. + self.__hard_check_length(target_file_object, file_length) self.__check_hashes(target_file_object, file_hashes) - return self.__get_file(target_filepath, verify_target_file, 'target', - file_length, download_safely=True, compression=None) + return self.__get_file(target_filepath, verify_decompressed_target_file, + 'target', file_length, download_safely=True, + compression=None) - def __verify_metadata_file(self, metadata_file_object, metadata_role): + def __verify_decompressed_metadata_file(self, metadata_file_object, + metadata_role): """ - A private helpe function to verify a downloaded metadata file. + A private helper function to verify a decompressed downloaded metadata + file. metadata_file_object: @@ -787,11 +859,14 @@ def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, """ - def unsafely_verify_metadata_file(metadata_file_object): - self.__verify_metadata_file(metadata_file_object, metadata_role) + def unsafely_verify_decompressed_metadata_file(metadata_file_object): + self.__soft_check_length(metadata_file_object, file_length) + self.__verify_decompressed_metadata_file(metadata_file_object, + metadata_role) - return self.__get_file(metadata_filepath, unsafely_verify_metadata_file, - 'meta', file_length, download_safely=False, + return self.__get_file(metadata_filepath, + unsafely_verify_decompressed_metadata_file, 'meta', + file_length, download_safely=False, compression=None) @@ -836,12 +911,15 @@ def safely_get_metadata_file(self, metadata_role, metadata_filepath, """ - def safely_verify_metadata_file(metadata_file_object): + def safely_verify_decompressed_metadata_file(metadata_file_object): + self.__hard_check_length(metadata_file_object, file_length) self.__check_hashes(metadata_file_object, file_hashes) - self.__verify_metadata_file(metadata_file_object, metadata_role) + self.__verify_decompressed_metadata_file(metadata_file_object, + metadata_role) - return self.__get_file(metadata_filepath, safely_verify_metadata_file, - 'meta', file_length, download_safely=True, + return self.__get_file(metadata_filepath, + safely_verify_decompressed_metadata_file, 'meta', + file_length, download_safely=True, compression=compression) @@ -851,7 +929,7 @@ def safely_verify_metadata_file(metadata_file_object): # TODO: Instead of the more fragile 'download_safely' switch, unroll the # function into two separate ones: one for "safe" download, and the other one # for "unsafe" download? This should induce safer and more readable code. - def __get_file(self, filepath, verify_file, file_type, + def __get_file(self, filepath, verify_decompressed_file, file_type, file_length, download_safely, compression): """ @@ -863,9 +941,9 @@ def __get_file(self, filepath, verify_file, file_type, filepath: The relative metadata or target filepath. - verify_file: - A function which expects a file-like object and which will raise an - exception in case the file is not valid for any reason. + verify_decompressed_file: + A function which expects a decompressed file-like object and which will + raise an exception in case the file is not valid for any reason. file_type: Type of data needed for download, must correspond to one of the strings @@ -911,9 +989,12 @@ def __get_file(self, filepath, verify_file, file_type, file_object = tuf.download.unsafe_download(file_mirror, file_length) if compression: + logger.debug('Decompressing '+str(file_mirror)) file_object.decompress_temp_file_object(compression) + else: + logger.debug('Not decompressing '+str(file_mirror)) - verify_file(file_object) + verify_decompressed_file(file_object) except Exception, exception: # Remember the error from this mirror, and "reset" the target file. @@ -2155,10 +2236,10 @@ def _visit_child_role(self, child_role, target_filepath): Ensure that we explore only delegated roles trusted with the target. We assume conservation of delegated paths in the complete tree of delegations. Note that the call to _ensure_all_targets_allowed in - __verify_metadata_file should already ensure that all targets metadata is - valid; i.e. that the targets signed by a delegatee is a proper subset of - the targets delegated to it by the delegator. Nevertheless, we check it - again here for performance and safety reasons. + __verify_decompressed_metadata_file should already ensure that all + targets metadata is valid; i.e. that the targets signed by a delegatee is + a proper subset of the targets delegated to it by the delegator. + Nevertheless, we check it again here for performance and safety reasons. TODO: Should the TUF spec restrict the repository to one particular algorithm? Should we allow the repository to specify in the role diff --git a/tuf/download.py b/tuf/download.py index dfdb33b5..4c3d0006 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -647,7 +647,7 @@ def _check_downloaded_length(total_downloaded, required_length, if STRICT_REQUIRED_LENGTH: # This must be due to a programming error, and must never happen! logger.error(message) - raise tuf.DownloadLengthMismatchError(message) + raise tuf.DownloadLengthMismatchError(required_length, total_downloaded) else: # We specifically disabled strict checking of required length, but we # will log a warning anyway. This is useful when we wish to download the diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index afa6606c..ccaf8659 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -534,15 +534,15 @@ def test_3__update_metadata(self): added_target_2 = self._add_target_to_targets_dir(targets_keyids) # To test compressed file handling, compress targets metadata file. - targets_filepath_compressed = self._compress_file(self.targets_filepath) + targets_filepath_compressed = self._compress_file(self.targets_filepath) # Re-patch 'download.download_url_to_tempfileobj' function. self._mock_download_url_to_tempfileobj(targets_filepath_compressed) - # TODO: Not convinced this is actually being tested correctly. - # See how we get fileinfo in tuf.client.updater._update_metadata_if_changed + # FIXME: The length (but not the hash) passed to this function is + # incorrect. The length must be that of the compressed file, whereas the + # hash must be that of the uncompressed file. _update_metadata('targets', - #signerlib.get_metadata_file_info(self.targets_filepath), - None, + signerlib.get_metadata_file_info(self.targets_filepath), compression='gzip') list_of_targets = self.Repository.metadata['current']['targets']['targets'] @@ -696,12 +696,12 @@ def test_3__update_metadata_if_changed(self): # Patch 'download.download_url_to_tempfileobj' and update targets. self._mock_download_url_to_tempfileobj(self.root_filepath) - # TODO: Is this the original intent of this test? + # FIXME: What is the original intent of this test? try: update_if_changed('targets') except tuf.NoWorkingMirrorError, exception: for mirror_url, mirror_error in exception.mirror_errors.iteritems(): - assert isinstance(mirror_error, tuf.BadHashError) + assert isinstance(mirror_error, tuf.DownloadLengthMismatchError) # Restoring repositories to the initial state. os.remove(release_filepath_compressed) diff --git a/tuf/tests/test_util.py b/tuf/tests/test_util.py index dd6fc5d7..c568eea7 100755 --- a/tuf/tests/test_util.py +++ b/tuf/tests/test_util.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_util.py @@ -37,7 +39,6 @@ class TestUtil(unittest_toolbox.Modified_TestCase): def setUp(self): unittest_toolbox.Modified_TestCase.setUp(self) - util.tempfile.TemporaryFile = tempfile.TemporaryFile self.temp_fileobj = util.TempFile() @@ -71,10 +72,10 @@ def _extract_tempfile_directory(self, config_temp_dir=None): # Patching 'tempfile.TemporaryFile()' (by substituting # temfile.TemporaryFile() with tempfile.mkstemp()) in order to get the # directory of the stored tempfile object. - saved_tempfile_TemporaryFile = util.tempfile.TemporaryFile - util.tempfile.TemporaryFile = tempfile.mkstemp + saved_tempfile_TemporaryFile = util.tempfile.NamedTemporaryFile + util.tempfile.NamedTemporaryFile = tempfile.mkstemp _temp_fileobj = util.TempFile() - util.tempfile.TemporaryFile = saved_tempfile_TemporaryFile + util.tempfile.NamedTemporaryFile = saved_tempfile_TemporaryFile junk, _tempfilepath = _temp_fileobj.temporary_file _tempfile_dir = os.path.dirname(_tempfilepath) diff --git a/tuf/util.py b/tuf/util.py index cd422db8..2c2f01d4 100755 --- a/tuf/util.py +++ b/tuf/util.py @@ -52,7 +52,7 @@ class TempFile(object): def _default_temporary_directory(self, prefix): """__init__ helper.""" try: - self.temporary_file = tempfile.TemporaryFile(prefix=prefix) + self.temporary_file = tempfile.NamedTemporaryFile(prefix=prefix) except OSError, err: logger.critical('Temp file in '+temp_dir+'failed: '+repr(err)) raise tuf.Error(err) @@ -66,7 +66,7 @@ def __init__(self, prefix='tuf_temp_'): prefix: - A string argument to be used with tempfile.TemporaryFile function. + A string argument to be used with tempfile.NamedTemporaryFile function. tuf.Error on failure to load temp dir. @@ -82,7 +82,8 @@ def __init__(self, prefix='tuf_temp_'): temp_dir = tuf.conf.temporary_directory if temp_dir is not None and isinstance(temp_dir, str): try: - self.temporary_file = tempfile.TemporaryFile(prefix=prefix, dir=temp_dir) + self.temporary_file = tempfile.NamedTemporaryFile(prefix=prefix, + dir=temp_dir) except OSError, err: logger.error('Temp file in '+temp_dir+' failed: '+repr(err)) logger.error('Will attempt to use system default temp dir.') @@ -92,6 +93,26 @@ def __init__(self, prefix='tuf_temp_'): + def __len__(self): + """ + + Initializes TempFile. + + + None. + + + OSError. + + + Nonnegative integer representing file size. + + """ + + return os.stat(self.temporary_file.name).st_size + + + def flush(self): """ From 7042418281c328e2c27426b1a5539457f1e3c6b3 Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 21:32:09 -0400 Subject: [PATCH 092/119] Fix a bug in updater unit test. --- tuf/tests/test_updater.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index ccaf8659..636b6f51 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -520,8 +520,9 @@ def test_3__update_metadata(self): # Test: normal case. # Patch 'download.download_url_to_tempfileobj' function. self._mock_download_url_to_tempfileobj(self.targets_filepath) - _update_metadata('targets', - signerlib.get_metadata_file_info(self.targets_filepath)) + uncompressed_fileinfo = \ + signerlib.get_metadata_file_info(self.targets_filepath) + _update_metadata('targets', uncompressed_fileinfo) list_of_targets = self.Repository.metadata['current']['targets']['targets'] # Verify that the added target's path is listed in target's metadata. @@ -532,18 +533,24 @@ def test_3__update_metadata(self): # Test: normal case, compressed metadata file. # Add a file to targets directory and rebuild targets metadata. added_target_2 = self._add_target_to_targets_dir(targets_keyids) + uncompressed_fileinfo = \ + signerlib.get_metadata_file_info(self.targets_filepath) # To test compressed file handling, compress targets metadata file. targets_filepath_compressed = self._compress_file(self.targets_filepath) + compressed_fileinfo = \ + signerlib.get_metadata_file_info(targets_filepath_compressed) # Re-patch 'download.download_url_to_tempfileobj' function. self._mock_download_url_to_tempfileobj(targets_filepath_compressed) - # FIXME: The length (but not the hash) passed to this function is - # incorrect. The length must be that of the compressed file, whereas the - # hash must be that of the uncompressed file. - _update_metadata('targets', - signerlib.get_metadata_file_info(self.targets_filepath), - compression='gzip') + # The length (but not the hash) passed to this function is incorrect. The + # length must be that of the compressed file, whereas the hash must be that + # of the uncompressed file. + mixed_fileinfo = { + 'length': compressed_fileinfo['length'], + 'hashes': uncompressed_fileinfo['hashes'] + } + _update_metadata('targets', mixed_fileinfo, compression='gzip') list_of_targets = self.Repository.metadata['current']['targets']['targets'] # Verify that the added target's path is listed in target's metadata. From 0fd533a9b46753037385cccde7516a48a93df92e Mon Sep 17 00:00:00 2001 From: dachshund Date: Mon, 9 Sep 2013 22:14:01 -0400 Subject: [PATCH 093/119] Much better organization of tests. --- setup.py | 3 ++- tuf/tests/__init__.py | 1 + .../slow_retrieval_server.py | 0 .../test_arbitrary_package_attack.py | 2 +- .../{integration_tests => integration}/test_delegations.py | 2 +- .../test_endless_data_attack.py | 2 +- .../test_extraneous_dependencies_attack.py | 4 +++- .../test_indefinite_freeze_attack.py | 2 +- .../test_mix_and_match_attack.py | 2 +- .../{integration_tests => integration}/test_replay_attack.py | 2 +- .../test_slow_retrieval_attack.py | 2 +- tuf/tests/integration_tests/__init__.py | 1 - tuf/tests/{integration_tests => }/test_util_test_tools.py | 2 ++ tuf/tests/{ => unit}/aggregate_tests.py | 0 tuf/tests/{ => unit}/repository_setup.py | 0 tuf/tests/{ => unit}/simple_server.py | 0 tuf/tests/{ => unit}/statement_coverage.py | 0 tuf/tests/{ => unit}/test_download.py | 0 tuf/tests/{ => unit}/test_formats.py | 0 tuf/tests/{ => unit}/test_hash.py | 0 tuf/tests/{ => unit}/test_keydb.py | 0 tuf/tests/{ => unit}/test_keystore.py | 0 tuf/tests/{ => unit}/test_mirrors.py | 0 tuf/tests/{ => unit}/test_push.py | 2 +- tuf/tests/{ => unit}/test_pushtoolslib.py | 0 tuf/tests/{ => unit}/test_quickstart.py | 0 tuf/tests/{ => unit}/test_roledb.py | 0 tuf/tests/{ => unit}/test_rsa_key.py | 0 tuf/tests/{ => unit}/test_schema.py | 0 tuf/tests/{ => unit}/test_sig.py | 0 tuf/tests/{ => unit}/test_signercli.py | 0 tuf/tests/{ => unit}/test_signerlib.py | 0 tuf/tests/{ => unit}/test_updater.py | 0 tuf/tests/{ => unit}/test_util.py | 0 tuf/tests/{ => unit}/unittest_toolbox.py | 0 tuf/tests/{integration_tests => }/util_test_tools.py | 0 36 files changed, 16 insertions(+), 11 deletions(-) mode change 100755 => 100644 tuf/tests/__init__.py rename tuf/tests/{integration_tests => integration}/slow_retrieval_server.py (100%) rename tuf/tests/{integration_tests => integration}/test_arbitrary_package_attack.py (99%) rename tuf/tests/{integration_tests => integration}/test_delegations.py (99%) rename tuf/tests/{integration_tests => integration}/test_endless_data_attack.py (99%) rename tuf/tests/{integration_tests => integration}/test_extraneous_dependencies_attack.py (98%) mode change 100644 => 100755 rename tuf/tests/{integration_tests => integration}/test_indefinite_freeze_attack.py (99%) rename tuf/tests/{integration_tests => integration}/test_mix_and_match_attack.py (99%) rename tuf/tests/{integration_tests => integration}/test_replay_attack.py (99%) rename tuf/tests/{integration_tests => integration}/test_slow_retrieval_attack.py (99%) delete mode 100755 tuf/tests/integration_tests/__init__.py rename tuf/tests/{integration_tests => }/test_util_test_tools.py (99%) rename tuf/tests/{ => unit}/aggregate_tests.py (100%) rename tuf/tests/{ => unit}/repository_setup.py (100%) rename tuf/tests/{ => unit}/simple_server.py (100%) rename tuf/tests/{ => unit}/statement_coverage.py (100%) rename tuf/tests/{ => unit}/test_download.py (100%) rename tuf/tests/{ => unit}/test_formats.py (100%) rename tuf/tests/{ => unit}/test_hash.py (100%) rename tuf/tests/{ => unit}/test_keydb.py (100%) rename tuf/tests/{ => unit}/test_keystore.py (100%) rename tuf/tests/{ => unit}/test_mirrors.py (100%) rename tuf/tests/{ => unit}/test_push.py (98%) rename tuf/tests/{ => unit}/test_pushtoolslib.py (100%) rename tuf/tests/{ => unit}/test_quickstart.py (100%) rename tuf/tests/{ => unit}/test_roledb.py (100%) rename tuf/tests/{ => unit}/test_rsa_key.py (100%) rename tuf/tests/{ => unit}/test_schema.py (100%) rename tuf/tests/{ => unit}/test_sig.py (100%) rename tuf/tests/{ => unit}/test_signercli.py (100%) rename tuf/tests/{ => unit}/test_signerlib.py (100%) rename tuf/tests/{ => unit}/test_updater.py (100%) rename tuf/tests/{ => unit}/test_util.py (100%) rename tuf/tests/{ => unit}/unittest_toolbox.py (100%) rename tuf/tests/{integration_tests => }/util_test_tools.py (100%) mode change 100755 => 100644 diff --git a/setup.py b/setup.py index ff1158cf..f53495b9 100755 --- a/setup.py +++ b/setup.py @@ -76,7 +76,8 @@ 'tuf.interposition', 'tuf.pushtools', 'tuf.pushtools.transfer', - 'tuf.repo' + 'tuf.repo', + 'tuf.tests' ], scripts=[ 'tuf/repo/quickstart.py', diff --git a/tuf/tests/__init__.py b/tuf/tests/__init__.py old mode 100755 new mode 100644 index e69de29b..582fc3b3 --- a/tuf/tests/__init__.py +++ b/tuf/tests/__init__.py @@ -0,0 +1 @@ +__all__ = ['util_test_tools'] diff --git a/tuf/tests/integration_tests/slow_retrieval_server.py b/tuf/tests/integration/slow_retrieval_server.py similarity index 100% rename from tuf/tests/integration_tests/slow_retrieval_server.py rename to tuf/tests/integration/slow_retrieval_server.py diff --git a/tuf/tests/integration_tests/test_arbitrary_package_attack.py b/tuf/tests/integration/test_arbitrary_package_attack.py similarity index 99% rename from tuf/tests/integration_tests/test_arbitrary_package_attack.py rename to tuf/tests/integration/test_arbitrary_package_attack.py index b5f207ce..67ac627e 100755 --- a/tuf/tests/integration_tests/test_arbitrary_package_attack.py +++ b/tuf/tests/integration/test_arbitrary_package_attack.py @@ -33,10 +33,10 @@ import shutil import urllib import tempfile -import util_test_tools import tuf from tuf.interposition import urllib_tuf +from tuf.tests import util_test_tools diff --git a/tuf/tests/integration_tests/test_delegations.py b/tuf/tests/integration/test_delegations.py similarity index 99% rename from tuf/tests/integration_tests/test_delegations.py rename to tuf/tests/integration/test_delegations.py index 4312d439..d84127d0 100755 --- a/tuf/tests/integration_tests/test_delegations.py +++ b/tuf/tests/integration/test_delegations.py @@ -30,7 +30,7 @@ import tuf.repo.keystore as keystore import tuf.repo.signercli as signercli import tuf.repo.signerlib as signerlib -import util_test_tools +from tuf.tests import util_test_tools version = 1 # Modify the number of iterations (from the higher default count) so the unit diff --git a/tuf/tests/integration_tests/test_endless_data_attack.py b/tuf/tests/integration/test_endless_data_attack.py similarity index 99% rename from tuf/tests/integration_tests/test_endless_data_attack.py rename to tuf/tests/integration/test_endless_data_attack.py index fecb8ab5..4d445daf 100755 --- a/tuf/tests/integration_tests/test_endless_data_attack.py +++ b/tuf/tests/integration/test_endless_data_attack.py @@ -35,10 +35,10 @@ import os import urllib -import util_test_tools import tuf from tuf.interposition import urllib_tuf +from tuf.tests import util_test_tools diff --git a/tuf/tests/integration_tests/test_extraneous_dependencies_attack.py b/tuf/tests/integration/test_extraneous_dependencies_attack.py old mode 100644 new mode 100755 similarity index 98% rename from tuf/tests/integration_tests/test_extraneous_dependencies_attack.py rename to tuf/tests/integration/test_extraneous_dependencies_attack.py index 996d9881..f07b4cde --- a/tuf/tests/integration_tests/test_extraneous_dependencies_attack.py +++ b/tuf/tests/integration/test_extraneous_dependencies_attack.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_extraneous_dependencies_attack.py @@ -42,7 +44,7 @@ import tuf import tuf.interposition -import util_test_tools +from tuf.tests import util_test_tools class ExtraneousDependencyAlert(Exception): diff --git a/tuf/tests/integration_tests/test_indefinite_freeze_attack.py b/tuf/tests/integration/test_indefinite_freeze_attack.py similarity index 99% rename from tuf/tests/integration_tests/test_indefinite_freeze_attack.py rename to tuf/tests/integration/test_indefinite_freeze_attack.py index 85f457d2..10887433 100755 --- a/tuf/tests/integration_tests/test_indefinite_freeze_attack.py +++ b/tuf/tests/integration/test_indefinite_freeze_attack.py @@ -25,12 +25,12 @@ import shutil import urllib import tempfile -import util_test_tools import tuf import tuf.formats import tuf.repo.signerlib as signerlib from tuf.interposition import urllib_tuf +from tuf.tests import util_test_tools class IndefiniteFreezeAttackAlert(Exception): diff --git a/tuf/tests/integration_tests/test_mix_and_match_attack.py b/tuf/tests/integration/test_mix_and_match_attack.py similarity index 99% rename from tuf/tests/integration_tests/test_mix_and_match_attack.py rename to tuf/tests/integration/test_mix_and_match_attack.py index 29ee9fcf..ae6efd59 100755 --- a/tuf/tests/integration_tests/test_mix_and_match_attack.py +++ b/tuf/tests/integration/test_mix_and_match_attack.py @@ -40,8 +40,8 @@ import tempfile import tuf -import util_test_tools from tuf.interposition import urllib_tuf +from tuf.tests import util_test_tools class MixAndMatchAttackAlert(Exception): diff --git a/tuf/tests/integration_tests/test_replay_attack.py b/tuf/tests/integration/test_replay_attack.py similarity index 99% rename from tuf/tests/integration_tests/test_replay_attack.py rename to tuf/tests/integration/test_replay_attack.py index de05b42e..f2b7fec6 100755 --- a/tuf/tests/integration_tests/test_replay_attack.py +++ b/tuf/tests/integration/test_replay_attack.py @@ -38,8 +38,8 @@ import urllib import tempfile -import util_test_tools as util_test_tools from tuf.interposition import urllib_tuf +from tuf.tests import util_test_tools class TestSetupError(Exception): diff --git a/tuf/tests/integration_tests/test_slow_retrieval_attack.py b/tuf/tests/integration/test_slow_retrieval_attack.py similarity index 99% rename from tuf/tests/integration_tests/test_slow_retrieval_attack.py rename to tuf/tests/integration/test_slow_retrieval_attack.py index da81343c..b80328ef 100755 --- a/tuf/tests/integration_tests/test_slow_retrieval_attack.py +++ b/tuf/tests/integration/test_slow_retrieval_attack.py @@ -49,8 +49,8 @@ import urllib -import util_test_tools as util_test_tools from tuf.interposition import urllib_tuf +from tuf.tests import util_test_tools class SlowRetrievalAttackAlert(Exception): diff --git a/tuf/tests/integration_tests/__init__.py b/tuf/tests/integration_tests/__init__.py deleted file mode 100755 index 8b137891..00000000 --- a/tuf/tests/integration_tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tuf/tests/integration_tests/test_util_test_tools.py b/tuf/tests/test_util_test_tools.py similarity index 99% rename from tuf/tests/integration_tests/test_util_test_tools.py rename to tuf/tests/test_util_test_tools.py index a36ad1fd..8ca8ffa8 100755 --- a/tuf/tests/integration_tests/test_util_test_tools.py +++ b/tuf/tests/test_util_test_tools.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_util_test_tools.py diff --git a/tuf/tests/aggregate_tests.py b/tuf/tests/unit/aggregate_tests.py similarity index 100% rename from tuf/tests/aggregate_tests.py rename to tuf/tests/unit/aggregate_tests.py diff --git a/tuf/tests/repository_setup.py b/tuf/tests/unit/repository_setup.py similarity index 100% rename from tuf/tests/repository_setup.py rename to tuf/tests/unit/repository_setup.py diff --git a/tuf/tests/simple_server.py b/tuf/tests/unit/simple_server.py similarity index 100% rename from tuf/tests/simple_server.py rename to tuf/tests/unit/simple_server.py diff --git a/tuf/tests/statement_coverage.py b/tuf/tests/unit/statement_coverage.py similarity index 100% rename from tuf/tests/statement_coverage.py rename to tuf/tests/unit/statement_coverage.py diff --git a/tuf/tests/test_download.py b/tuf/tests/unit/test_download.py similarity index 100% rename from tuf/tests/test_download.py rename to tuf/tests/unit/test_download.py diff --git a/tuf/tests/test_formats.py b/tuf/tests/unit/test_formats.py similarity index 100% rename from tuf/tests/test_formats.py rename to tuf/tests/unit/test_formats.py diff --git a/tuf/tests/test_hash.py b/tuf/tests/unit/test_hash.py similarity index 100% rename from tuf/tests/test_hash.py rename to tuf/tests/unit/test_hash.py diff --git a/tuf/tests/test_keydb.py b/tuf/tests/unit/test_keydb.py similarity index 100% rename from tuf/tests/test_keydb.py rename to tuf/tests/unit/test_keydb.py diff --git a/tuf/tests/test_keystore.py b/tuf/tests/unit/test_keystore.py similarity index 100% rename from tuf/tests/test_keystore.py rename to tuf/tests/unit/test_keystore.py diff --git a/tuf/tests/test_mirrors.py b/tuf/tests/unit/test_mirrors.py similarity index 100% rename from tuf/tests/test_mirrors.py rename to tuf/tests/unit/test_mirrors.py diff --git a/tuf/tests/test_push.py b/tuf/tests/unit/test_push.py similarity index 98% rename from tuf/tests/test_push.py rename to tuf/tests/unit/test_push.py index a48bb5aa..4acc8d75 100755 --- a/tuf/tests/test_push.py +++ b/tuf/tests/unit/test_push.py @@ -28,7 +28,7 @@ import tuf.pushtools.push as push import tuf.pushtools.transfer.scp as scp import tuf.pushtools.pushtoolslib as pushtoolslib -import system_tests.util_test_tools as util_test_tools +import tuf.tests.util_test_tools as util_test_tools logger = logging.getLogger('tuf.test_push') diff --git a/tuf/tests/test_pushtoolslib.py b/tuf/tests/unit/test_pushtoolslib.py similarity index 100% rename from tuf/tests/test_pushtoolslib.py rename to tuf/tests/unit/test_pushtoolslib.py diff --git a/tuf/tests/test_quickstart.py b/tuf/tests/unit/test_quickstart.py similarity index 100% rename from tuf/tests/test_quickstart.py rename to tuf/tests/unit/test_quickstart.py diff --git a/tuf/tests/test_roledb.py b/tuf/tests/unit/test_roledb.py similarity index 100% rename from tuf/tests/test_roledb.py rename to tuf/tests/unit/test_roledb.py diff --git a/tuf/tests/test_rsa_key.py b/tuf/tests/unit/test_rsa_key.py similarity index 100% rename from tuf/tests/test_rsa_key.py rename to tuf/tests/unit/test_rsa_key.py diff --git a/tuf/tests/test_schema.py b/tuf/tests/unit/test_schema.py similarity index 100% rename from tuf/tests/test_schema.py rename to tuf/tests/unit/test_schema.py diff --git a/tuf/tests/test_sig.py b/tuf/tests/unit/test_sig.py similarity index 100% rename from tuf/tests/test_sig.py rename to tuf/tests/unit/test_sig.py diff --git a/tuf/tests/test_signercli.py b/tuf/tests/unit/test_signercli.py similarity index 100% rename from tuf/tests/test_signercli.py rename to tuf/tests/unit/test_signercli.py diff --git a/tuf/tests/test_signerlib.py b/tuf/tests/unit/test_signerlib.py similarity index 100% rename from tuf/tests/test_signerlib.py rename to tuf/tests/unit/test_signerlib.py diff --git a/tuf/tests/test_updater.py b/tuf/tests/unit/test_updater.py similarity index 100% rename from tuf/tests/test_updater.py rename to tuf/tests/unit/test_updater.py diff --git a/tuf/tests/test_util.py b/tuf/tests/unit/test_util.py similarity index 100% rename from tuf/tests/test_util.py rename to tuf/tests/unit/test_util.py diff --git a/tuf/tests/unittest_toolbox.py b/tuf/tests/unit/unittest_toolbox.py similarity index 100% rename from tuf/tests/unittest_toolbox.py rename to tuf/tests/unit/unittest_toolbox.py diff --git a/tuf/tests/integration_tests/util_test_tools.py b/tuf/tests/util_test_tools.py old mode 100755 new mode 100644 similarity index 100% rename from tuf/tests/integration_tests/util_test_tools.py rename to tuf/tests/util_test_tools.py From 85567f4ec2ddbe9fd5450b4fcbcee65f11b3337d Mon Sep 17 00:00:00 2001 From: zanefisher Date: Mon, 9 Sep 2013 22:32:26 -0400 Subject: [PATCH 094/119] Clean up download function patching in test_updater's module teardown. --- tuf/tests/test_updater.py | 240 +++++++++++++++++++------------------- 1 file changed, 121 insertions(+), 119 deletions(-) diff --git a/tuf/tests/test_updater.py b/tuf/tests/test_updater.py index 5dd7ed58..3fae7c08 100755 --- a/tuf/tests/test_updater.py +++ b/tuf/tests/test_updater.py @@ -67,6 +67,8 @@ class guarantees the order of unit tests. So that, 'test_something_A' 'length': tuf.conf.DEFAULT_TIMESTAMP_REQUIRED_LENGTH } +original_safe_download = tuf.download.safe_download +original_unsafe_download = tuf.download.unsafe_download class TestUpdater_init_(unittest_toolbox.Modified_TestCase): @@ -404,6 +406,27 @@ def test_1__rebuild_key_and_role_db(self): + def test_1__update_fileinfo(self): + # Tests + # Verify that fileinfo dictionary is empty. + self.assertFalse(self.Repository.fileinfo) + + # Load file info for top level roles. This populates the fileinfo + # dictionary. + for role in self.role_list: + self.Repository._update_fileinfo(role+'.txt') + + # Verify that fileinfo has been populated and contains appropriate data. + self.assertTrue(self.Repository.fileinfo) + for role in self.role_list: + role_filepath = os.path.join(self.client_current_dir, role+'.txt') + role_info = tuf.util.get_file_details(role_filepath) + role_info_dict = {'length':role_info[0], 'hashes':role_info[1]} + self.assertTrue(role+'.txt' in self.Repository.fileinfo.keys()) + self.assertEqual(self.Repository.fileinfo[role+'.txt'], role_info_dict) + + + def test_2__import_delegations(self): @@ -485,6 +508,86 @@ def test_2__ensure_all_targets_allowed(self): + def test_2__fileinfo_has_changed(self): + # Verify that the method returns 'False' if file info was not changed. + for role in self.role_list: + role_filepath = os.path.join(self.client_current_dir, role+'.txt') + role_info = tuf.util.get_file_details(role_filepath) + role_info_dict = {'length':role_info[0], 'hashes':role_info[1]} + self.assertFalse(self.Repository._fileinfo_has_changed(role+'.txt', + role_info_dict)) + + # Verify that the method returns 'True' if length or hashes were changed. + for role in self.role_list: + role_filepath = os.path.join(self.client_current_dir, role+'.txt') + role_info = tuf.util.get_file_details(role_filepath) + role_info_dict = {'length':8, 'hashes':role_info[1]} + self.assertTrue(self.Repository._fileinfo_has_changed(role+'.txt', + role_info_dict)) + + for role in self.role_list: + role_filepath = os.path.join(self.client_current_dir, role+'.txt') + role_info = tuf.util.get_file_details(role_filepath) + role_info_dict = {'length':role_info[0], + 'hashes':{'sha256':self.random_string()}} + self.assertTrue(self.Repository._fileinfo_has_changed(role+'.txt', + role_info_dict)) + + + + + def test_2__move_current_to_previous(self): + # The test will consist of removing a metadata file from client's + # {client_repository}/metadata/previous directory, executing the method + # and then verifying that the 'previous' directory contains + # the release file. + release_meta_path = os.path.join(self.client_previous_dir, 'release.txt') + os.remove(release_meta_path) + self.assertFalse(os.path.exists(release_meta_path)) + self.Repository._move_current_to_previous('release') + self.assertTrue(os.path.exists(release_meta_path)) + shutil.copy(release_meta_path, self.client_current_dir) + + + + + + def test_2__delete_metadata(self): + # This test will verify that 'root' metadata is never deleted, when + # role is deleted verify that the file is not present in the + # self.Repository.metadata dictionary. + self.Repository._delete_metadata('root') + self.assertTrue('root' in self.Repository.metadata['current']) + self.Repository._delete_metadata('timestamp') + self.assertFalse('timestamp' in self.Repository.metadata['current']) + timestamp_meta_path = os.path.join(self.client_previous_dir, + 'timestamp.txt') + shutil.copy(timestamp_meta_path, self.client_current_dir) + + + + + + def test_2__ensure_not_expired(self): + # This test condition will verify that nothing is raised when a metadata + # file has a future expiration date. + self.Repository._ensure_not_expired('root') + + # 'tuf.ExpiredMetadataError' should be raised in this next test condition, + # because the expiration_date has expired by 10 seconds. + expires = tuf.formats.format_time(time.time() - 10) + self.Repository.metadata['current']['root']['expires'] = expires + + # Ensure the 'expires' field of the root file is properly formatted. + self.assertTrue(tuf.formats.ROOT_SCHEMA.matches(self.Repository.metadata\ + ['current']['root'])) + self.assertRaises(tuf.ExpiredMetadataError, + self.Repository._ensure_not_expired, 'root') + + + + + def test_3__update_metadata(self): """ This unit test verifies the method's proper behaviour on the expected input. @@ -540,10 +643,8 @@ def test_3__update_metadata(self): self._mock_download_url_to_tempfileobj(targets_filepath_compressed) # TODO: Not convinced this is actually being tested correctly. # See how we get fileinfo in tuf.client.updater._update_metadata_if_changed - _update_metadata('targets', - #signerlib.get_metadata_file_info(self.targets_filepath), - None, - compression='gzip') + self.Repository._update_metadata_if_changed('targets') + list_of_targets = self.Repository.metadata['current']['targets']['targets'] # Verify that the added target's path is listed in target's metadata. @@ -558,56 +659,6 @@ def test_3__update_metadata(self): - def test_1__update_fileinfo(self): - # Tests - # Verify that fileinfo dictionary is empty. - self.assertFalse(self.Repository.fileinfo) - - # Load file info for top level roles. This populates the fileinfo - # dictionary. - for role in self.role_list: - self.Repository._update_fileinfo(role+'.txt') - - # Verify that fileinfo has been populated and contains appropriate data. - self.assertTrue(self.Repository.fileinfo) - for role in self.role_list: - role_filepath = os.path.join(self.client_current_dir, role+'.txt') - role_info = tuf.util.get_file_details(role_filepath) - role_info_dict = {'length':role_info[0], 'hashes':role_info[1]} - self.assertTrue(role+'.txt' in self.Repository.fileinfo.keys()) - self.assertEqual(self.Repository.fileinfo[role+'.txt'], role_info_dict) - - - - - - def test_2__fileinfo_has_changed(self): - # Verify that the method returns 'False' if file info was not changed. - for role in self.role_list: - role_filepath = os.path.join(self.client_current_dir, role+'.txt') - role_info = tuf.util.get_file_details(role_filepath) - role_info_dict = {'length':role_info[0], 'hashes':role_info[1]} - self.assertFalse(self.Repository._fileinfo_has_changed(role+'.txt', - role_info_dict)) - - # Verify that the method returns 'True' if length or hashes were changed. - for role in self.role_list: - role_filepath = os.path.join(self.client_current_dir, role+'.txt') - role_info = tuf.util.get_file_details(role_filepath) - role_info_dict = {'length':8, 'hashes':role_info[1]} - self.assertTrue(self.Repository._fileinfo_has_changed(role+'.txt', - role_info_dict)) - - for role in self.role_list: - role_filepath = os.path.join(self.client_current_dir, role+'.txt') - role_info = tuf.util.get_file_details(role_filepath) - role_info_dict = {'length':role_info[0], - 'hashes':{'sha256':self.random_string()}} - self.assertTrue(self.Repository._fileinfo_has_changed(role+'.txt', - role_info_dict)) - - - def test_3__update_metadata_if_changed(self): @@ -711,53 +762,23 @@ def test_3__update_metadata_if_changed(self): - def test_2__move_current_to_previous(self): - # The test will consist of removing a metadata file from client's - # {client_repository}/metadata/previous directory, executing the method - # and then verifying that the 'previous' directory contains - # the release file. - release_meta_path = os.path.join(self.client_previous_dir, 'release.txt') - os.remove(release_meta_path) - self.assertFalse(os.path.exists(release_meta_path)) - self.Repository._move_current_to_previous('release') - self.assertTrue(os.path.exists(release_meta_path)) - shutil.copy(release_meta_path, self.client_current_dir) + def test_3__targets_of_role(self): + # Setup + targets_dir_content = os.listdir(self.targets_dir) - - - - def test_2__delete_metadata(self): - # This test will verify that 'root' metadata is never deleted, when - # role is deleted verify that the file is not present in the - # self.Repository.metadata dictionary. - self.Repository._delete_metadata('root') - self.assertTrue('root' in self.Repository.metadata['current']) - self.Repository._delete_metadata('timestamp') - self.assertFalse('timestamp' in self.Repository.metadata['current']) - timestamp_meta_path = os.path.join(self.client_previous_dir, - 'timestamp.txt') - shutil.copy(timestamp_meta_path, self.client_current_dir) - - - - - - def test_2__ensure_not_expired(self): - # This test condition will verify that nothing is raised when a metadata - # file has a future expiration date. - self.Repository._ensure_not_expired('root') + # Test: normal case. + targets_list = self.Repository._targets_of_role('targets') - # 'tuf.ExpiredMetadataError' should be raised in this next test condition, - # because the expiration_date has expired by 10 seconds. - expires = tuf.formats.format_time(time.time() - 10) - self.Repository.metadata['current']['root']['expires'] = expires - - # Ensure the 'expires' field of the root file is properly formatted. - self.assertTrue(tuf.formats.ROOT_SCHEMA.matches(self.Repository.metadata\ - ['current']['root'])) - self.assertRaises(tuf.ExpiredMetadataError, - self.Repository._ensure_not_expired, 'root') + # Verify that list of targets was returned, + # and that it contains valid target file. + self.assertTrue(tuf.formats.TARGETFILES_SCHEMA.matches(targets_list)) + targets_filepaths = [] + for target in range(len(targets_list)): + targets_filepaths.append(targets_list[target]['filepath']) + for dir_target in targets_dir_content: + if dir_target.endswith('.txt'): + self.assertTrue(dir_target in targets_filepaths) @@ -857,27 +878,6 @@ def test_4__refresh_targets_metadata(self): - def test_3__targets_of_role(self): - # Setup - targets_dir_content = os.listdir(self.targets_dir) - - - # Test: normal case. - targets_list = self.Repository._targets_of_role('targets') - - # Verify that list of targets was returned, - # and that it contains valid target file. - self.assertTrue(tuf.formats.TARGETFILES_SCHEMA.matches(targets_list)) - targets_filepaths = [] - for target in range(len(targets_list)): - targets_filepaths.append(targets_list[target]['filepath']) - for dir_target in targets_dir_content: - if dir_target.endswith('.txt'): - self.assertTrue(dir_target in targets_filepaths) - - - - def test_5_all_targets(self): @@ -1138,6 +1138,8 @@ def tearDownModule(): # http://docs.python.org/2/library/unittest.html#class-and-module-fixtures setup.remove_all_repositories(TestUpdater.repositories['main_repository']) unittest_toolbox.Modified_TestCase.clear_toolbox() + tuf.download.safe_download = original_safe_download + tuf.download.unsafe_download = original_unsafe_download if __name__ == '__main__': unittest.main() From 78344636deb0bd956b4962fd57aefacd4cb63e52 Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 10 Sep 2013 00:24:34 -0400 Subject: [PATCH 095/119] Even better organization of tests; remove evpy. Require higher versions of PyCrypto. --- README.md | 6 +- evpy/__init__.py | 0 evpy/cipher.py | 200 ------ evpy/envelope.py | 329 --------- evpy/evp.py | 318 --------- evpy/signature.py | 215 ------ evpy/test.py | 668 ------------------ setup.py | 3 +- .../integration/slow_retrieval_server.py | 0 .../test_arbitrary_package_attack.py | 0 .../integration/test_delegations.py | 0 .../integration/test_endless_data_attack.py | 0 .../test_extraneous_dependencies_attack.py | 0 .../test_indefinite_freeze_attack.py | 0 .../integration/test_mix_and_match_attack.py | 0 .../integration/test_replay_attack.py | 0 .../integration/test_slow_retrieval_attack.py | 0 {tuf/tests => tests}/unit/aggregate_tests.py | 0 {tuf/tests => tests}/unit/repository_setup.py | 0 {tuf/tests => tests}/unit/simple_server.py | 0 .../unit/statement_coverage.py | 0 {tuf/tests => tests}/unit/test_download.py | 0 {tuf/tests => tests}/unit/test_formats.py | 0 {tuf/tests => tests}/unit/test_hash.py | 0 {tuf/tests => tests}/unit/test_keydb.py | 0 {tuf/tests => tests}/unit/test_keystore.py | 0 {tuf/tests => tests}/unit/test_mirrors.py | 0 {tuf/tests => tests}/unit/test_push.py | 0 .../tests => tests}/unit/test_pushtoolslib.py | 0 {tuf/tests => tests}/unit/test_quickstart.py | 0 {tuf/tests => tests}/unit/test_roledb.py | 0 {tuf/tests => tests}/unit/test_rsa_key.py | 0 {tuf/tests => tests}/unit/test_schema.py | 0 {tuf/tests => tests}/unit/test_sig.py | 0 {tuf/tests => tests}/unit/test_signercli.py | 0 {tuf/tests => tests}/unit/test_signerlib.py | 0 {tuf/tests => tests}/unit/test_updater.py | 0 {tuf/tests => tests}/unit/test_util.py | 0 .../unit}/test_util_test_tools.py | 3 +- {tuf/tests => tests}/unit/unittest_toolbox.py | 0 tuf/tests/__init__.py | 3 + 41 files changed, 9 insertions(+), 1736 deletions(-) delete mode 100755 evpy/__init__.py delete mode 100755 evpy/cipher.py delete mode 100755 evpy/envelope.py delete mode 100755 evpy/evp.py delete mode 100755 evpy/signature.py delete mode 100755 evpy/test.py rename {tuf/tests => tests}/integration/slow_retrieval_server.py (100%) rename {tuf/tests => tests}/integration/test_arbitrary_package_attack.py (100%) rename {tuf/tests => tests}/integration/test_delegations.py (100%) rename {tuf/tests => tests}/integration/test_endless_data_attack.py (100%) rename {tuf/tests => tests}/integration/test_extraneous_dependencies_attack.py (100%) rename {tuf/tests => tests}/integration/test_indefinite_freeze_attack.py (100%) rename {tuf/tests => tests}/integration/test_mix_and_match_attack.py (100%) rename {tuf/tests => tests}/integration/test_replay_attack.py (100%) rename {tuf/tests => tests}/integration/test_slow_retrieval_attack.py (100%) rename {tuf/tests => tests}/unit/aggregate_tests.py (100%) rename {tuf/tests => tests}/unit/repository_setup.py (100%) rename {tuf/tests => tests}/unit/simple_server.py (100%) rename {tuf/tests => tests}/unit/statement_coverage.py (100%) rename {tuf/tests => tests}/unit/test_download.py (100%) rename {tuf/tests => tests}/unit/test_formats.py (100%) rename {tuf/tests => tests}/unit/test_hash.py (100%) rename {tuf/tests => tests}/unit/test_keydb.py (100%) rename {tuf/tests => tests}/unit/test_keystore.py (100%) rename {tuf/tests => tests}/unit/test_mirrors.py (100%) rename {tuf/tests => tests}/unit/test_push.py (100%) rename {tuf/tests => tests}/unit/test_pushtoolslib.py (100%) rename {tuf/tests => tests}/unit/test_quickstart.py (100%) rename {tuf/tests => tests}/unit/test_roledb.py (100%) rename {tuf/tests => tests}/unit/test_rsa_key.py (100%) rename {tuf/tests => tests}/unit/test_schema.py (100%) rename {tuf/tests => tests}/unit/test_sig.py (100%) rename {tuf/tests => tests}/unit/test_signercli.py (100%) rename {tuf/tests => tests}/unit/test_signerlib.py (100%) rename {tuf/tests => tests}/unit/test_updater.py (100%) rename {tuf/tests => tests}/unit/test_util.py (100%) rename {tuf/tests => tests/unit}/test_util_test_tools.py (99%) rename {tuf/tests => tests}/unit/unittest_toolbox.py (100%) diff --git a/README.md b/README.md index 87f96f42..3a92ed4b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# A Framework for Securing Software Update Systems +## A Framework for Securing Software Update Systems TUF (The Update Framework) helps developers secure their new or existing software update systems. Software update systems are vulnerable to many known @@ -6,7 +6,7 @@ attacks, including those that can result in clients being compromised or crashed. TUF helps solve this problem by providing a flexible security framework that can be added to software updaters. -# What Is a Software Update System? +## What Is a Software Update System? Generally, a software update system is an application (or part of an application) running on a client system that obtains and installs software. @@ -27,7 +27,7 @@ Python's pip/easy_install + PyPI, Perl's CPAN, Ruby's Gems, and PHP's PEAR. of the software on a client system. Debian's APT, Red Hat's YUM, and openSUSE's YaST are examples of these. -# Our Approach +## Our Approach There are literally thousands of different software update systems in common use today. (In fact the average Windows user has about two dozen different diff --git a/evpy/__init__.py b/evpy/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/evpy/cipher.py b/evpy/cipher.py deleted file mode 100755 index de904067..00000000 --- a/evpy/cipher.py +++ /dev/null @@ -1,200 +0,0 @@ -#! /usr/bin/env python - -""" -cipher.py - -Written by Geremy Condra -Released on 18 March 2010 -Licensed under MIT License - -This module provides a basic interface to OpenSSL's EVP -cipher functions. - -All the functions in this module raise CipherError on -malfunction. - -From an end-user perspective, this module should be used -in situations where you want to have a single generally -human-readable or human-generated key used for both -encryption and decryption. - -This means that as a general rule, if your application -involves transmitting this key over an insecure channel -you should not be using this module, but rather -evpy.envelope. - -Usage: - - >>> from evpy import cipher - >>> message = b"this is data" - >>> pw = b"mypassword" - >>> salt, iv, enc = cipher.encrypt(message, pw) - >>> cipher.decrypt(salt, iv, enc, pw) - 'this is data' -""" - -import ctypes -import time - -import evp - - -class CipherError(evp.SSLError): - pass - -def _strengthen_password(pw, iv, salt=None): - # add the hash - evp.OpenSSL_add_all_digests() - # build the key buffer - key = ctypes.create_string_buffer(24) - # either take the existing salt or build a new one - if not salt: - salt = ctypes.create_string_buffer(8) - # get the needed entropy, bailing if it doesn't work in - # the first thousand tries - for i in range(1000): - if evp.RAND_bytes(salt, 8): break - else: - raise CipherError("Could not generate enough entropy") - # extract the salt - salt = salt.raw - # get the hash - evp_hash = evp.EVP_get_digestbyname("sha512") - if not evp_hash: - raise CipherError("Could not create hash object") - # fill the key - if not evp.EVP_BytesToKey(evp.EVP_aes_192_cbc(), evp_hash, salt, pw, len(pw), 1000, key, iv): - raise CipherError("Could not strengthen key") - # go home - return salt, key.raw - - -def encrypt(data, password): - """Encrypts the given data, raising CipherError on failure. - - This uses AES192 to encrypt and strengthens the given - passphrase using SHA512. - - Usage: - >>> from evpy import cipher - >>> f = open("test/short.txt", "rb") - >>> data = f.read() - >>> pw = b"mypassword" - >>> salt, iv, enc = cipher.encrypt(data, pw) - >>> cipher.decrypt(salt, iv, enc, pw) == data - True - """ - # ensure data exists - if not len(data): - raise CipherError("Data must actually exist") - if not len(password): - raise CipherError("Password must actually exist") - - # build and initialize the context - ctx = evp.EVP_CIPHER_CTX_new() - if not ctx: - raise CipherError("Could not create context") - evp.EVP_CIPHER_CTX_init(ctx) - - # get the cipher object - cipher_object = evp.EVP_aes_192_cbc() - if not cipher_object: - raise CipherError("Could not create cipher object") - - # finish the context and cipher object - if not evp.EVP_EncryptInit_ex(ctx, cipher_object, None, None, None): - raise CipherError("Could not finish context") - - # build the randomized iv - iv_length = evp.EVP_CIPHER_CTX_iv_length(ctx) - iv = ctypes.create_string_buffer(iv_length) - # get the needed entropy, bailing if it doesn't work in - # the first thousand tries - for i in range(1000): - if evp.RAND_bytes(iv, iv_length): break - else: - raise CipherError("Not enough entropy for IV") - output_iv = iv.raw - - # strengthen the password into an honest-to-goodness key - salt, aes_key = _strengthen_password(password, iv) - - # initialize the encryption operation - if not evp.EVP_EncryptInit_ex(ctx, None, None, aes_key, iv): - raise CipherError("Could not start encryption operation") - - # build the output buffer - buf = ctypes.create_string_buffer(len(data) + 16) - written = ctypes.c_int(0) - final = ctypes.c_int(0) - - # update - if not evp.EVP_EncryptUpdate(ctx, buf, ctypes.byref(written), data, len(data)): - raise CipherError("Could not update ciphertext") - output = buf.raw[:written.value] - - # finalize - if not evp.EVP_EncryptFinal_ex(ctx, buf, ctypes.byref(final)): - raise CipherError("Could not finalize ciphertext") - output += buf.raw[:final.value] - - # ...and go home - return salt, output_iv, output - - -def decrypt(salt, iv, data, password): - """Decrypts the given data, raising CipherError on failure. - - Usage: - >>> from evpy import cipher - >>> f = open("test/short.txt", "rb") - >>> data = f.read() - >>> pw = b"mypassword" - >>> salt, iv, enc = cipher.encrypt(data, pw) - >>> cipher.decrypt(salt, iv, enc, pw) == data - True - """ - # ensure inputs are the correct size - if not len(data): - raise CipherError("Data must actually exist") - if not len(password): - raise CipherError("Password must actually exist") - if len(salt) != 8: - raise CipherError("Incorrect salt size") - if len(iv) != 16: - raise CipherError("Incorrect iv size") - - # build and initialize the context - ctx = evp.EVP_CIPHER_CTX_new() - if not ctx: - raise CipherError("Could not create context") - evp.EVP_CIPHER_CTX_init(ctx) - - # get the cipher object - cipher_object = evp.EVP_aes_192_cbc() - if not cipher_object: - raise CipherError("Could not create cipher object") - - # build the key - salt, key = _strengthen_password(password, iv, salt) - - # start decrypting the ciphertext - if not evp.EVP_DecryptInit_ex(ctx, cipher_object, None, key, iv): - raise CipherError("Could not open envelope") - - # build the output buffers - buf = ctypes.create_string_buffer(len(data) + 16) - written = ctypes.c_int(0) - final = ctypes.c_int(0) - - # update - if not evp.EVP_DecryptUpdate(ctx, buf, ctypes.byref(written), data, len(data)): - raise CipherError("Could not update plaintext") - output = buf.raw[:written.value] - - # finalize - if not evp.EVP_DecryptFinal_ex(ctx, buf, ctypes.byref(final)): - raise CipherError("Could not finalize decryption") - output += buf.raw[:final.value] - - return output diff --git a/evpy/envelope.py b/evpy/envelope.py deleted file mode 100755 index 28b2d811..00000000 --- a/evpy/envelope.py +++ /dev/null @@ -1,329 +0,0 @@ -#! /usr/bin/env python - -""" -envelope.py - -Written by Geremy Condra -Released on 18 March 2010 -Licensed under MIT License - -This module provides a basic interface to OpenSSL's EVP -envelope functions. - -In a nutshell, these functions are designed to provide -the primary benefit of public key cryptography (the -ability to provide secrecy without first sharing a -secret) without its primary downside (small message -length). It does this by generating a random AES key -with which to encrypt the data, then encrypting that -key against the provided RSA key. - -This means that if you have an application in which -you wish to share sensitive data but do not wish to -share a common secret, this is your module. Be aware -that compromising your private key is effectively -game over with this scheme. - -If you require a shared secret and want the key to -be human readable, then you will probably want to -use the cipher module instead. - -All the functions in this module raise EnvelopeError on -malfunction. - -Usage: - - >>> from evpy import envelope - >>> f = open("test/short.txt", "rb") - >>> data = f.read() - >>> public_key = "test/keys/public1.pem" - >>> private_key = "test/keys/private1.pem" - >>> iv, key, ciphertext = envelope.encrypt(data, public_key) - >>> envelope.decrypt(iv, key, ciphertext, private_key) == data - True -""" - -import ctypes - -import evp -from signature import _string_to_bio - -class EnvelopeError(evp.SSLError): - pass - -class KeygenError(evp.SSLError): - pass - -def _build_dkey_from_file(keyfile): - fp = evp.fopen(keyfile, "r") - if not fp: - raise EnvelopeError("Could not open keyfile") - # get the decryption key - skey = evp.PEM_read_PrivateKey(fp, None, None, None) - if not skey: - evp.fclose(fp) - raise EnvelopeError("Could not read decryption key") - # close the file - evp.fclose(fp) - return skey - -def _build_dkey_from_string(key): - bio = _string_to_bio(key) - dkey = evp.PEM_read_bio_PrivateKey(bio, None, None, None) - if not dkey: - raise EnvelopeError("Could not build decryption key from string") - evp.BIO_free(bio) - return dkey - -def _build_ekey_from_file(keyfile): - fp = evp.fopen(keyfile, "r") - if not fp: - raise EnvelopeError("Could not open keyfile") - # get the encryption key - ekey = evp.PEM_read_PUBKEY(fp, None, None, None) - if not ekey: - evp.fclose(fp) - raise EnvelopeError("Could not read encryption key") - # close the file - evp.fclose(fp) - return ekey - -def _build_ekey_from_string(key): - bio = _string_to_bio(key) - ekey = evp.PEM_read_bio_PUBKEY(bio, None, None, None) - if not ekey: - raise EnvelopeError("Could not create encryption key from string") - evp.BIO_free(bio) - return ekey - -def _build_bio(): - method = evp.BIO_s_mem() - return evp.BIO_new(method); - -def _asn1_hex_to_int(value): - print(value) - return int(''.join(value.split(':')), 16) - -def _parse_printed_key(k): - attrs = {} - current = "" - current_attr = "" - for line in k.splitlines()[1:]: - # its a continuation of the current block - if line.startswith(' '): - current += line.strip() - else: - # special case the public exponent - if "publicExponent" in current_attr: - attrs['publicExponent'] = int(current_attr.split()[1]) - elif current_attr: - attrs[current_attr] = _asn1_hex_to_int(current) - current_attr = line.strip(':') - current = "" - translator = {'publicExponent': 'e', 'privateExponent': 'd', 'modulus': 'n', 'prime1': 'p', 'prime2': 'q'} - translated_attrs = {} - for key, value in attrs.items(): - try: - translated_attrs[translator[key]] = value - except: pass - return translated_attrs - - -def keygen(bitlength=1024, e=65537, pem=True): - key = evp.RSA_generate_key(bitlength, e, None, None) - if not key: - raise EnvelopeError("Could not generate key") - if pem: - private_bio = evp.BIO_new(evp.BIO_s_mem()) - if not private_bio: - raise KeygenError("Could not create temporary storage") - public_bio = evp.BIO_new(evp.BIO_s_mem()) - if not public_bio: - raise KeygenError("Could not create temporary storage") - private_buf = ctypes.create_string_buffer('', 65537) - if not private_buf: - raise MemoryError("Could not allocate key storage") - public_buf = ctypes.create_string_buffer('', 65537) - if not public_buf: - raise MemoryError("Could not allocate key storage") - if not evp.PEM_write_bio_RSAPrivateKey(private_bio, key, None, None, 0, 0, None): - raise KeygenError("Could not write private key") - if not evp.PEM_write_bio_RSA_PUBKEY(public_bio, key): - raise KeygenError("Could not write public key") - public_len = evp.BIO_read(public_bio, public_buf, 65537) - private_len = evp.BIO_read(private_bio, private_buf, 65537) - evp.BIO_free(public_bio) - evp.BIO_free(private_bio) - return public_buf.value, private_buf.value - else: - # we go through this rigamarole because if there's an engine - # in place it won't populate the RSA key's values properly. - key_bio = evp.BIO_new(evp.BIO_s_mem()) - if not key_bio: - raise KeygenError("Could not create temporary storage") - if not evp.RSA_print(key_bio, key, 0): - raise KeygenError("Could not stringify key") - key_buf = ctypes.create_string_buffer('', 65537) - if not key_buf: - raise MemoryError("Could not allocate key storage") - evp.BIO_read(key_bio, key_buf, 65537) - evp.BIO_free(key_bio) - key_string = key_buf.value - return key, _parse_printed_key(key_string) - - - - -def encrypt(data, keyfile=None, key=None): - """Encrypts the given data, raising EnvelopeError on failure. - - This uses AES192 to do bulk encryption and RSA to encrypt - the given public key. - - Usage: - >>> from evpy import envelope - >>> f = open("test/short.txt", "rb") - >>> data = f.read() - >>> public_key = "test/keys/public1.pem" - >>> private_key = "test/keys/private1.pem" - >>> iv, key, ciphertext = envelope.encrypt(data, public_key) - >>> envelope.decrypt(iv, key, ciphertext, private_key) == data - True - """ - # validate the incoming data - if not data: - raise EnvelopeError("Incoming data must be bytes") - if not len(data): - raise EnvelopeError("Data must actually exist") - - # build and initialize the context - ctx = evp.EVP_CIPHER_CTX_new() - if not ctx: - raise EnvelopeError("Could not create context") - evp.EVP_CIPHER_CTX_init(ctx) - - # get the key from the keyfile - if key and not keyfile: - ekey = _build_ekey_from_string(key) - elif keyfile and not key: - ekey = _build_ekey_from_file(keyfile) - else: - raise EnvelopeError("Must specify exactly one key or keyfile") - - # get the cipher object - cipher_object = evp.EVP_aes_192_cbc() - if not cipher_object: - raise EnvelopeError("Could not create cipher object") - - # finish the context and cipher object - if not evp.EVP_EncryptInit_ex(ctx, cipher_object, None, None, None): - raise EnvelopeError("Could not finish context") - - # build the randomized iv - iv_length = evp.EVP_CIPHER_CTX_iv_length(ctx) - iv = ctypes.create_string_buffer(iv_length) - for i in range(1000): - if evp.RAND_bytes(iv, iv_length): break - else: - raise EnvelopeError("Could not generate enough entropy for IV") - output_iv = iv.raw - - # build the randomized AES key - keysize = evp.EVP_CIPHER_key_length(cipher_object) - aes_key = ctypes.create_string_buffer(keysize) - for i in range(1000): - if evp.RAND_bytes(aes_key, keysize): break - else: - raise EnvelopeError("Could not generate enough entropy for AES key") - - # extract the RSA key - rsa_key = evp.EVP_PKEY_get1_RSA(ekey) - if not rsa_key: - raise EnvelopeError("Could not get RSA key") - - # encrypt it - buf_size = evp.RSA_size(rsa_key) - if not buf_size: - raise EnvelopeError("Invalid RSA keysize") - encrypted_aes_key = ctypes.create_string_buffer(buf_size) - # RSA_PKCS1_PADDING is defined as 1 - written = evp.RSA_public_encrypt(keysize, aes_key, encrypted_aes_key, rsa_key, 1) - if not written: - raise EnvelopeError("Could not encrypt AES key") - output_key = encrypted_aes_key.raw[:written] - - # initialize the encryption operation - if not evp.EVP_EncryptInit_ex(ctx, None, None, aes_key, iv): - raise EnvelopeError("Could not start encryption operation") - - # build the output buffer - buf = ctypes.create_string_buffer(len(data) + 16) - written = ctypes.c_int(0) - final = ctypes.c_int(0) - - # update - if not evp.EVP_EncryptUpdate(ctx, buf, ctypes.byref(written), data, len(data)): - raise EnvelopeError("Could not update ciphertext") - output = buf.raw[:written.value] - - # finalize - if not evp.EVP_EncryptFinal_ex(ctx, buf, ctypes.byref(final)): - raise EnvelopeError("Could not finalize ciphertext") - output += buf.raw[:final.value] - - # ...and go home - return output_iv, output_key, output - - -def decrypt(iv, encrypted_aes_key, data, keyfile=None, key=None): - """Decrypts the given ciphertext, raising EnvelopeError on failure. - - Usage: - >>> from evpy import envelope - >>> f = open("test/short.txt", "rb") - >>> data = f.read() - >>> public_key = "test/keys/public1.pem" - >>> private_key = "test/keys/private1.pem" - >>> iv, key, ciphertext = envelope.encrypt(data, public_key) - >>> envelope.decrypt(iv, key, ciphertext, private_key) == data - True - """ - # build and initialize the context - ctx = evp.EVP_CIPHER_CTX_new() - if not ctx: - raise EnvelopeError("Could not create context") - evp.EVP_CIPHER_CTX_init(ctx) - - # get the cipher object - cipher_object = evp.EVP_aes_192_cbc() - if not cipher_object: - raise EnvelopeError("Could not create cipher object") - - # get the key from the keyfile - if key and not keyfile: - dkey = _build_dkey_from_string(key) - elif keyfile and not key: - dkey = _build_dkey_from_file(keyfile) - else: - raise EnvelopeError("Must specify exactly one key or keyfile") - - # open the envelope - if not evp.EVP_OpenInit(ctx, cipher_object, encrypted_aes_key, len(encrypted_aes_key), iv, dkey): - raise EnvelopeError("Could not open envelope") - - # build the output buffer - buf = ctypes.create_string_buffer(len(data) + 16) - written = ctypes.c_int(0) - final = ctypes.c_int(0) - - # update - if not evp.EVP_DecryptUpdate(ctx, buf, ctypes.byref(written), data, len(data)): - raise EnvelopeError("Could not update envelope") - output = buf.raw[:written.value] - - # finalize - if not evp.EVP_DecryptFinal_ex(ctx, buf, ctypes.byref(final)): - raise EnvelopeError("Could not finalize envelope") - output += buf.raw[:final.value] - - return output diff --git a/evpy/evp.py b/evpy/evp.py deleted file mode 100755 index 55988767..00000000 --- a/evpy/evp.py +++ /dev/null @@ -1,318 +0,0 @@ -#! /usr/bin/env python - -import ctypes -import ctypes.util -import platform -from os import linesep - -def handle_errors(): - ERR_load_crypto_strings() - errno = ERR_get_error() - errbuf = ctypes.create_string_buffer(1024) - ERR_error_string_n(errno, errbuf, 1024) - return errbuf.value.decode("ascii") - -class SSLError(Exception): - def __init__(self, msg): - sslerr = handle_errors() - Exception.__init__(self, msg + linesep + sslerr) - -libraries = {} - -libraries["c"] = ctypes.CDLL(ctypes.util.find_library("c")) - -if platform.system() != "Windows": - libraries["ssl"] = ctypes.CDLL(ctypes.util.find_library("ssl")) -else: - libraries["ssl"] = ctypes.CDLL(ctypes.util.find_library("libeay32")) - - -class RSA(ctypes.Structure): - _fields_ = [ ("n", ctypes.c_void_p), - ("e", ctypes.c_void_p), - ("d", ctypes.c_void_p), - ("p", ctypes.c_void_p), - ("q", ctypes.c_void_p), - ("dmp1", ctypes.c_void_p), - ("iqmp", ctypes.c_void_p)] - -def handle_errors(): - ERR_load_crypto_strings() - errno = ERR_get_error() - errbuf = ctypes.create_string_buffer(1024) - ERR_error_string_n(errno, errbuf, 1024) - return errbuf.value.decode("ascii") - - -fopen = libraries['c'].fopen -fopen.restype = ctypes.c_void_p -fopen.argtypes = [ctypes.c_char_p, ctypes.c_char_p] - - -fclose = libraries['c'].fclose -fclose.restype = ctypes.c_int -fclose.argtypes = [ctypes.c_void_p] - - -ERR_load_crypto_strings = libraries['ssl'].ERR_load_crypto_strings -ERR_load_crypto_strings.restype = ctypes.c_int -ERR_load_crypto_strings.argtypes = [] - - -ERR_print_errors_fp = libraries['ssl'].ERR_print_errors_fp -ERR_print_errors_fp.restype = ctypes.c_int -ERR_print_errors_fp.argtypes = [ctypes.c_void_p] - - -OpenSSL_add_all_digests = libraries['ssl'].OpenSSL_add_all_digests -OpenSSL_add_all_digests.restype = ctypes.c_int -OpenSSL_add_all_digests.argtypes = [] - - -EVP_MD_CTX_create = libraries['ssl'].EVP_MD_CTX_create -EVP_MD_CTX_create.restype = ctypes.c_void_p -EVP_MD_CTX_create.argtypes = [] - - -PEM_read_PrivateKey = libraries['ssl'].PEM_read_PrivateKey -PEM_read_PrivateKey.restype = ctypes.c_void_p -PEM_read_PrivateKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - - -PEM_read_X509 = libraries['ssl'].PEM_read_X509 -PEM_read_X509.restype = ctypes.c_void_p -PEM_read_X509.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - - -PEM_read_PUBKEY = libraries['ssl'].PEM_read_PUBKEY -PEM_read_PUBKEY.restype = ctypes.c_void_p -PEM_read_PUBKEY.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - - -PEM_read_bio_PUBKEY = libraries['ssl'].PEM_read_bio_PUBKEY -PEM_read_bio_PUBKEY.restype = ctypes.c_void_p -PEM_read_bio_PUBKEY.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - - -BIO_free = libraries['ssl'].BIO_free -BIO_free.restype = ctypes.c_int -BIO_free.argtypes = [ctypes.c_void_p] - - -PEM_read_bio_PrivateKey = libraries['ssl'].PEM_read_bio_PrivateKey -PEM_read_bio_PrivateKey.restype = ctypes.c_void_p -PEM_read_bio_PrivateKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - - -EVP_get_digestbyname = libraries['ssl'].EVP_get_digestbyname -EVP_get_digestbyname.restype = ctypes.c_void_p -EVP_get_digestbyname.argtypes = [ctypes.c_char_p] - - -EVP_DigestInit = libraries['ssl'].EVP_DigestInit -EVP_DigestInit.restype = ctypes.c_int -EVP_DigestInit.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - - -EVP_DigestUpdate = libraries['ssl'].EVP_DigestUpdate -EVP_DigestUpdate.restype = ctypes.c_int -EVP_DigestUpdate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int] - - -EVP_SignFinal = libraries['ssl'].EVP_SignFinal -EVP_SignFinal.restype = ctypes.c_int -EVP_SignFinal.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_int), ctypes.c_void_p] - - -EVP_VerifyFinal = libraries['ssl'].EVP_VerifyFinal -EVP_VerifyFinal.restype = ctypes.c_int -EVP_VerifyFinal.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p] - - -EVP_PKEY_free = libraries['ssl'].EVP_PKEY_free -EVP_PKEY_free.restype = ctypes.c_int -EVP_PKEY_free.argtypes = [ctypes.c_void_p] - - -EVP_MD_CTX_cleanup = libraries['ssl'].EVP_MD_CTX_cleanup -EVP_MD_CTX_cleanup.restype = ctypes.c_int -EVP_MD_CTX_cleanup.argtypes = [ctypes.c_void_p] - - -EVP_MD_CTX_destroy = libraries['ssl'].EVP_MD_CTX_destroy -EVP_MD_CTX_destroy.restype = ctypes.c_int -EVP_MD_CTX_destroy.argtypes = [ctypes.c_void_p] - - -EVP_CIPHER_CTX_new = libraries['ssl'].EVP_CIPHER_CTX_new -EVP_CIPHER_CTX_new.restype = ctypes.c_void_p -EVP_CIPHER_CTX_new.argtypes = [] - - -EVP_CIPHER_CTX_init = libraries['ssl'].EVP_CIPHER_CTX_init -EVP_CIPHER_CTX_init.restype = ctypes.c_int -EVP_CIPHER_CTX_init.argtypes = [ctypes.c_void_p] - -try: - EVP_CIPHER_CTX_iv_length = libraries['ssl'].EVP_CIPHER_CTX_iv_length - EVP_CIPHER_CTX_iv_length.restype = ctypes.c_int - EVP_CIPHER_CTX_iv_length.argtypes = [ctypes.c_void_p] -except: - EVP_CIPHER_CTX_iv_length = lambda(x): 16 - - -EVP_aes_192_cbc = libraries['ssl'].EVP_aes_192_cbc -EVP_aes_192_cbc.restype = ctypes.c_void_p -EVP_aes_192_cbc.argtypes = [] - - -EVP_PKEY_size = libraries['ssl'].EVP_PKEY_size -EVP_PKEY_size.restype = ctypes.c_int -EVP_PKEY_size.argtypes = [ctypes.c_void_p] - - -EVP_SealInit = libraries['ssl'].EVP_SealInit -EVP_SealInit.restype = ctypes.c_int -EVP_SealInit.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_int), ctypes.c_char_p, ctypes.c_void_p, ctypes.c_int] - - -EVP_EncryptInit_ex = libraries['ssl'].EVP_EncryptInit_ex -EVP_EncryptInit_ex.restype = ctypes.c_int -EVP_EncryptInit_ex.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p] - - -EVP_EncryptUpdate = libraries['ssl'].EVP_EncryptUpdate -EVP_EncryptUpdate.restype = ctypes.c_int -EVP_EncryptUpdate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_int), ctypes.c_char_p, ctypes.c_int] - - -EVP_DecryptUpdate = libraries['ssl'].EVP_DecryptUpdate -EVP_DecryptUpdate.restype = ctypes.c_int -EVP_DecryptUpdate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_int), ctypes.c_char_p, ctypes.c_int] - - -EVP_EncryptFinal_ex = libraries['ssl'].EVP_EncryptFinal_ex -EVP_EncryptFinal_ex.restype = ctypes.c_int -EVP_EncryptFinal_ex.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_int)] - - -EVP_SealFinal = libraries['ssl'].EVP_SealFinal -EVP_SealFinal.restype = ctypes.c_int -EVP_SealFinal.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_int)] - - -EVP_DecryptFinal_ex = libraries['ssl'].EVP_DecryptFinal_ex -EVP_DecryptFinal_ex.restype = ctypes.c_int -EVP_DecryptFinal_ex.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_int)] - - -RAND_bytes = libraries['ssl'].RAND_bytes -RAND_bytes.restype = ctypes.c_int -RAND_bytes.argtypes = [ctypes.c_char_p, ctypes.c_int] - - -BIO_new_mem_buf = libraries['ssl'].BIO_new_mem_buf -BIO_new_mem_buf.restype = ctypes.c_void_p -BIO_new_mem_buf.argtypes = [ctypes.c_char_p, ctypes.c_int] - - -EVP_CIPHER_CTX_rand_key = libraries['ssl'].EVP_CIPHER_CTX_rand_key -EVP_CIPHER_CTX_rand_key.restype = ctypes.c_int -EVP_CIPHER_CTX_rand_key.argtypes = [ctypes.c_void_p, ctypes.c_char_p] - - -try: - EVP_CIPHER_key_length = libraries['ssl'].EVP_CIPHER_key_length - EVP_CIPHER_key_length.restype = ctypes.c_int - EVP_CIPHER_key_length.argtypes = [ctypes.c_void_p] -except: - EVP_CIPHER_key_length = lambda(x): 24 - - -EVP_PKEY_encrypt = libraries['ssl'].EVP_PKEY_encrypt -EVP_PKEY_encrypt.restype = ctypes.c_int -EVP_PKEY_encrypt.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p] - - -EVP_OpenInit = libraries['ssl'].EVP_OpenInit -EVP_OpenInit.restype = ctypes.c_int -EVP_OpenInit.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p] - - -EVP_PKEY_size = libraries['ssl'].EVP_PKEY_size -EVP_PKEY_size.restype = ctypes.c_int -EVP_PKEY_size.argtypes = [ctypes.c_void_p] - - -RSA_public_encrypt = libraries['ssl'].RSA_public_encrypt -RSA_public_encrypt.restype = ctypes.c_int -RSA_public_encrypt.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_int] - - -EVP_PKEY_get1_RSA = libraries['ssl'].EVP_PKEY_get1_RSA -EVP_PKEY_get1_RSA.restype = ctypes.c_void_p -EVP_PKEY_get1_RSA.argtypes = [ctypes.c_void_p] - - -EVP_DecryptInit_ex = libraries['ssl'].EVP_DecryptInit_ex -EVP_DecryptInit_ex.restype = ctypes.c_int -EVP_DecryptInit_ex.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p] - - -EVP_CIPHER_CTX_set_key_length = libraries['ssl'].EVP_CIPHER_CTX_set_key_length -EVP_CIPHER_CTX_set_key_length.restype = ctypes.c_int -EVP_CIPHER_CTX_set_key_length.argtypes = [ctypes.c_void_p, ctypes.c_int] - - -EVP_BytesToKey = libraries['ssl'].EVP_BytesToKey -EVP_BytesToKey.restype = ctypes.c_int -EVP_BytesToKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p] - - -ERR_get_error = libraries['ssl'].ERR_get_error -ERR_get_error.restype = ctypes.c_long -ERR_get_error.argtypes = [] - - -ERR_error_string_n = libraries['ssl'].ERR_error_string_n -ERR_error_string_n.restype = ctypes.c_void_p -ERR_error_string_n.argtypes = [ctypes.c_long, ctypes.c_char_p, ctypes.c_int] - -RSA_size = libraries['ssl'].RSA_size -RSA_size.restype = ctypes.c_int -RSA_size.argtypes = [ctypes.c_void_p] - -RSA_new = libraries['ssl'].RSA_new -RSA_new.restype = ctypes.POINTER(RSA) - -RSA_generate_key = libraries['ssl'].RSA_generate_key -RSA_generate_key.restype = ctypes.POINTER(RSA) -RSA_generate_key.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_void_p, ctypes.c_void_p] - -RSA_print = libraries['ssl'].RSA_print -RSA_print.restype = ctypes.c_int -RSA_print.argtypes = [ctypes.c_void_p, ctypes.POINTER(RSA), ctypes.c_int] - -PEM_write_bio_RSA_PUBKEY = libraries['ssl'].PEM_write_bio_RSA_PUBKEY - -PEM_write_bio_RSAPrivateKey = libraries['ssl'].PEM_write_bio_RSAPrivateKey -PEM_write_bio_RSAPrivateKey.restype = ctypes.c_int -PEM_write_bio_RSAPrivateKey.argtypes = [ctypes.c_void_p, ctypes.POINTER(RSA), ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] - -PEM_write_RSAPrivateKey = libraries['ssl'].PEM_write_RSAPrivateKey -PEM_write_RSAPrivateKey.restype = ctypes.c_int -PEM_write_RSAPrivateKey.argtypes = [ctypes.c_void_p, ctypes.POINTER(RSA), ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] - -i2d_RSAPrivateKey = libraries['ssl'].i2d_RSAPrivateKey - -BIO_read = libraries['ssl'].BIO_read - -BIO_s_mem = libraries['ssl'].BIO_s_mem -BIO_s_mem.restype = ctypes.c_void_p - -BIO_new = libraries['ssl'].BIO_new -BIO_new.restype = ctypes.c_void_p -BIO_new.argtypes = [ctypes.c_void_p] - -RAND_seed = libraries['ssl'].RAND_seed -RAND_seed.argtypes = [ctypes.c_void_p, ctypes.c_int] diff --git a/evpy/signature.py b/evpy/signature.py deleted file mode 100755 index 29a6beac..00000000 --- a/evpy/signature.py +++ /dev/null @@ -1,215 +0,0 @@ -#! /usr/bin/env python - -""" -signature.py - -Written by Geremy Condra -Released on 18 March 2010 -Licensed under MIT License - -This module provides a basic interface to OpenSSL's EVP -signature functions. - -All functions in this module will raise a SignatureError -in the event of a malfunction. - -The goal of cryptographic signatures is to provide some -degree of assurance that the data you are processing is -both coming from the person you think is sending it and -is what they sent. - -Note that this does not encrypt data in the sense that -it does not provide secrecy for it, while evpy.cipher -and evpy.envelope provide secrecy but no other security -properties. - -Usage: - >>> from evpy import signature - >>> data = b"abcdefg" - >>> public_key = "test/keys/public1.pem" - >>> private_key = "test/keys/private1.pem" - >>> s = signature.sign(data, private_key) - >>> signature.verify(data, s, public_key) - True -""" - -import ctypes - -import evp - - -class SignatureError(evp.SSLError): - pass - - -def sign(data, keyfile=None, key=None): - """Signs the given data, raising SignatureError on failure. - - Exactly one of keyfile, key should be given; if key is not - defined, then the key will be read from the given file. - - Usage: - >>> from evpy import signature - >>> f = open("test/short.txt", "rb") - >>> data = f.read() - >>> public_key = "test/keys/public1.pem" - >>> private_key = "test/keys/private1.pem" - >>> s = signature.sign(data, private_key) - >>> signature.verify(data, s, public_key) - True - """ - # add the digests - evp.OpenSSL_add_all_digests() - - # build the context - ctx = evp.EVP_MD_CTX_create() - if not ctx: - raise SignatureError("Could not create context") - - # get the signing key - if key and not keyfile: - skey = _build_skey_from_string(key) - elif keyfile and not key: - skey = _build_skey_from_file(keyfile) - else: - raise SignatureError("Exactly one of key, keyfile must be specified") - - # build the hash object - evp_hash = _build_hash() - if not evp.EVP_DigestInit(ctx, evp_hash): - _cleanup(skey, ctx) - raise SignatureError("Could not initialize signature") - - # update - if not evp.EVP_DigestUpdate(ctx, data, len(data)): - _cleanup(skey, ctx) - raise SignatureError("Could not update signature") - - # finalize - output_buflen = ctypes.c_int(evp.EVP_PKEY_size(skey)) - output = ctypes.create_string_buffer(output_buflen.value) - if not evp.EVP_SignFinal(ctx, output, ctypes.byref(output_buflen), skey): - _cleanup(skey, ctx) - raise SignatureError("Could not finalize signature") - - # cleanup - _cleanup(skey, ctx) - - # and go home - return ctypes.string_at(output, output_buflen) - - -def verify(data, sig, keyfile=None, key=None): - """Verifies the given signature, returning a boolean. - - Exactly one of keyfile, key should be specified. - - This function raises SignatureError on error. - - Usage: - >>> from evpy import signature - >>> f = open("test/short.txt", "rb") - >>> data = f.read() - >>> public_key = "test/keys/public1.pem" - >>> private_key = "test/keys/private1.pem" - >>> s = signature.sign(data, private_key) - >>> signature.verify(data, s, public_key) - True - """ - # add the digests - evp.OpenSSL_add_all_digests() - - # build the context - ctx = evp.EVP_MD_CTX_create() - if not ctx: - raise SignatureError("Could not create context") - - # get the vkey - if key and not keyfile: - vkey = _build_vkey_from_string(key) - elif keyfile and not key: - vkey = _build_vkey_from_file(keyfile) - else: - raise SignatureError("Exactly one of key, keyfile must be specified") - - # build the hash object - evp_hash = _build_hash() - if not evp.EVP_DigestInit(ctx, evp_hash): - _cleanup(vkey, ctx) - raise SignatureError("Could not initialize verifier") - - # update - if not evp.EVP_DigestUpdate(ctx, data, len(data)): - _cleanup(vkey, ctx) - raise SignatureError("Could not update verifier") - - # finalize - retcode = evp.EVP_VerifyFinal(ctx, sig, len(sig), vkey) - - # cleanup - _cleanup(vkey, ctx) - - # and go home - if retcode == 1: - return True - elif retcode == 0: - return False - else: - raise SignatureError("Error verifying signature") - -def _cleanup(key, ctx): - evp.EVP_PKEY_free(key) - evp.EVP_MD_CTX_cleanup(ctx) - evp.EVP_MD_CTX_destroy(ctx) - -def _string_to_bio(s): - return evp.BIO_new_mem_buf(s, len(s)) - -def _build_skey_from_file(keyfile): - fp = evp.fopen(keyfile, "r") - if not fp: - raise SignatureError("Could not open keyfile") - # get the signing key - skey = evp.PEM_read_PrivateKey(fp, None, None, None) - if not skey: - evp.fclose(fp) - raise SignatureError("Could not read signing key") - # close the file - evp.fclose(fp) - return skey - -def _build_skey_from_string(key): - buf = ctypes.create_string_buffer(key) - bio = evp.BIO_new_mem_buf(buf, len(buf.value)) - skey = evp.PEM_read_bio_PrivateKey(bio, None, None, None, None) - if not skey: - raise SignatureError("Could not construct signing key from the given string") - evp.BIO_free(bio) - return skey - -def _build_vkey_from_file(keyfile): - fp = evp.fopen(keyfile, "r") - if not fp: - raise SignatureError("Could not open keyfile") - # get the verification key - vkey = evp.PEM_read_PUBKEY(fp, None, None, None) - if not vkey: - evp.fclose(fp) - raise SignatureError("Could not read verification key") - # close the file - evp.fclose(fp) - return vkey - -def _build_vkey_from_string(key): - buf = ctypes.create_string_buffer(key) - bio = evp.BIO_new_mem_buf(buf, len(buf.value)) - vkey = evp.PEM_read_bio_PUBKEY(bio, None, None, None) - if not vkey: - raise SignatureError("Could not construct verification key from the given string") - return vkey - -def _build_hash(): - evp_hash = evp.EVP_get_digestbyname("sha512") - if not evp_hash: - raise SignatureError("Could not create hash object") - return evp_hash diff --git a/evpy/test.py b/evpy/test.py deleted file mode 100755 index f9d39f1e..00000000 --- a/evpy/test.py +++ /dev/null @@ -1,668 +0,0 @@ -#! /usr/bin/env python - -""" -test.py - -Written by Geremy Condra -Licensed under MIT License -Released 21 March 2010 - -Simple unit tests for evpy -""" - -import unittest - -from evpy import evp -from evpy import cipher -from evpy import signature -from evpy import envelope - -# locations for test data -TEST_DATA_DIR = "test/" -TEST_KEYS = TEST_DATA_DIR + "keys/" - -# test text -STRING = open(TEST_DATA_DIR + "long.txt").read() -LONG = open(TEST_DATA_DIR + "long.txt", 'rb').read() -SHORT = open(TEST_DATA_DIR + "short.txt", 'rb').read() -UNICODE = open(TEST_DATA_DIR + "unicode.txt", 'rb').read() -NULL = open(TEST_DATA_DIR + "null.txt", 'rb').read() -TEST_TEXTS = { "long": LONG, - "short": SHORT, - "unicode": UNICODE, - "null": NULL} - -# test keys -SHORT_SYMMETRIC = open(TEST_KEYS + "short_symmetric.txt", 'rb').read() -LONG_SYMMETRIC = open(TEST_KEYS + "long_symmetric.txt", 'rb').read() -SYMMETRIC_KEYS = [LONG_SYMMETRIC, SHORT_SYMMETRIC] -SYMMETRIC_STRING = open(TEST_KEYS + "short_symmetric.txt").read() - -KEY_1 = TEST_KEYS + "private1.pem", TEST_KEYS + "public1.pem" -KEY_2 = TEST_KEYS + "private2.pem", TEST_KEYS + "public2.pem" -MISMATCH_1 = KEY_1[0], KEY_2[1] -MISMATCH_2 = KEY_2[0], KEY_1[1] -MISSING_PRIVATE_KEY = "notakey.pem", KEY_1[1] -MISSING_PUBLIC_KEY = KEY_1[0], "notakey.pem" -BLANK_PRIVATE_KEY = KEY_1[0], TEST_KEYS + "blank.pem" -BLANK_PUBLIC_KEY = TEST_KEYS + "blank.pem", KEY_1[1] - -def run_n_times(f, g, n): - def err(*args, **kwargs): - if err.n > 0: - err.n -= 1 - return f(*args, **kwargs) - else: - return g(*args, **kwargs) - err.n = n - return err - -class TestCipher(unittest.TestCase): - - def round_trip(self, key, text): - salt, iv, enc = cipher.encrypt(text, key) - output = cipher.decrypt(salt, iv, enc, key) - self.assertEqual(output, text, "Failed to round trip") - - def test_round_trip_short_zero(self): - self.assertRaises(cipher.CipherError, self.round_trip, SHORT_SYMMETRIC, '') - - def test_round_trip_short_long(self): - self.round_trip(SHORT_SYMMETRIC, LONG) - - def test_round_trip_short_short(self): - self.round_trip(SHORT_SYMMETRIC, SHORT) - - def test_round_trip_short_unicode(self): - self.round_trip(SHORT_SYMMETRIC, UNICODE) - - def test_round_trip_short_null(self): - self.round_trip(SHORT_SYMMETRIC, NULL) - - def test_round_trip_long_zero(self): - self.assertRaises(cipher.CipherError, self.round_trip, LONG_SYMMETRIC, '') - - def test_round_trip_long_long(self): - self.round_trip(LONG_SYMMETRIC, LONG) - - def test_round_trip_long_short(self): - self.round_trip(LONG_SYMMETRIC, SHORT) - - def test_round_trip_long_unicode(self): - self.round_trip(LONG_SYMMETRIC, UNICODE) - - def test_round_trip_long_null(self): - self.round_trip(LONG_SYMMETRIC, NULL) - - def test_no_data(self): - iv, salt, enc = cipher.encrypt(UNICODE, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, iv, salt, '', SHORT_SYMMETRIC) - - def test_short_salt(self): - salt, iv, enc = cipher.encrypt(UNICODE, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, salt[:-1], iv, enc, SHORT_SYMMETRIC) - - def test_long_salt(self): - salt, iv, enc = cipher.encrypt(UNICODE, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, salt+salt[:-1], iv, enc, SHORT_SYMMETRIC) - - def test_short_iv(self): - salt, iv, enc = cipher.encrypt(UNICODE, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, salt, iv[:-1], enc, SHORT_SYMMETRIC) - - def test_long_iv(self): - salt, iv, enc = cipher.encrypt(UNICODE, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, salt, iv+iv[:-1], enc, SHORT_SYMMETRIC) - - def test_round_trip_no_password(self): - iv, salt, enc = cipher.encrypt(UNICODE, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.encrypt, UNICODE, '') - self.assertRaises(cipher.CipherError, cipher.decrypt, iv, salt, enc, '') - - def test_round_trip_failure(self): - salt, iv, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, salt, iv, enc, LONG_SYMMETRIC) - salt, iv, enc = cipher.encrypt(NULL, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, salt, iv, enc, LONG_SYMMETRIC) - - def test_bad_rand_bytes(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - rand_bytes = cipher.evp.RAND_bytes - cipher.evp.RAND_bytes = lambda a,b: 0 - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - cipher.evp.RAND_bytes = rand_bytes - - def test_bad_rand_bytes_1(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - rand_bytes = cipher.evp.RAND_bytes - cipher.evp.RAND_bytes = lambda a,b: 0 - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - cipher.evp.RAND_bytes = rand_bytes - - def test_bad_rand_bytes_2(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - rand_bytes = cipher.evp.RAND_bytes - cipher.evp.RAND_bytes = run_n_times(cipher.evp.RAND_bytes, lambda a,b:0, 1) - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - cipher.evp.RAND_bytes = rand_bytes - - def test_bad_hash_by_name(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - hash_by_name = cipher.evp.EVP_get_digestbyname - cipher.evp.EVP_get_digestbyname = lambda a: None - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - cipher.evp.EVP_get_digestbyname = hash_by_name - - def test_bad_bytes_to_key(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - bytes_to_key = cipher.evp.EVP_BytesToKey - cipher.evp.EVP_BytesToKey = lambda a,b,c,d,e,f,g,h: 0 - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, iv, salt, enc, SHORT_SYMMETRIC) - cipher.evp.EVP_BytesToKey = bytes_to_key - - def test_bad_ctx_new(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - new_ctx = cipher.evp.EVP_CIPHER_CTX_new - cipher.evp.EVP_CIPHER_CTX_new = lambda: None - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, iv, salt, enc, SHORT_SYMMETRIC) - cipher.evp.EVP_CIPHER_CTX_new = new_ctx - - def test_bad_cipher_object(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - cipher_getter = cipher.evp.EVP_aes_192_cbc - cipher.evp.EVP_aes_192_cbc= lambda: None - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - self.assertRaises(cipher.CipherError, cipher.decrypt, iv, salt, enc, SHORT_SYMMETRIC) - cipher.evp.EVP_aes_192_cbc = cipher_getter - - def test_bad_encrypt_init_1(self): - encrypt_init = cipher.evp.EVP_EncryptInit_ex - cipher.evp.EVP_EncryptInit_ex = lambda a,b,c,d,e: None - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - cipher.evp.EVP_EncryptInit_ex = encrypt_init - - def test_bad_encrypt_init_2(self): - encrypt_init = cipher.evp.EVP_EncryptInit_ex - cipher.evp.EVP_EncryptInit_ex = run_n_times(cipher.evp.EVP_EncryptInit_ex, lambda a,b,c,d,e: None, 1) - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - cipher.evp.EVP_EncryptInit_ex = encrypt_init - - def test_bad_encrypt_update(self): - encrypt_update = cipher.evp.EVP_EncryptUpdate - cipher.evp.EVP_EncryptUpdate = lambda a,b,c,d,e: None - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - cipher.evp.EVP_EncryptUpdate = encrypt_update - - def test_bad_encrypt_final(self): - encrypt_final = cipher.evp.EVP_EncryptFinal_ex - cipher.evp.EVP_EncryptFinal_ex = lambda a,b,c: None - self.assertRaises(cipher.CipherError, cipher.encrypt, SHORT, SHORT_SYMMETRIC) - cipher.evp.EVP_EncryptFinal_ex = encrypt_final - - def test_bad_decrypt_init(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - decrypt_init = cipher.evp.EVP_DecryptInit_ex - cipher.evp.EVP_DecryptInit_ex = lambda a,b,c,d,e: None - self.assertRaises(cipher.CipherError, cipher.decrypt, iv, salt, enc, SHORT_SYMMETRIC) - cipher.evp.EVP_DecryptInit_ex = decrypt_init - - def test_bad_decrypt_update(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - decrypt_update = cipher.evp.EVP_DecryptUpdate - cipher.evp.EVP_DecryptUpdate = lambda a,b,c,d,e: None - self.assertRaises(cipher.CipherError, cipher.decrypt, iv, salt, enc, SHORT_SYMMETRIC) - cipher.evp.EVP_DecryptUpdate = decrypt_update - - def test_bad_decrypt_final(self): - iv, salt, enc = cipher.encrypt(SHORT, SHORT_SYMMETRIC) - decrypt_final = cipher.evp.EVP_DecryptFinal_ex - cipher.evp.EVP_DecryptFinal_ex = lambda a,b,c: None - self.assertRaises(cipher.CipherError, cipher.decrypt, iv, salt, enc, SHORT_SYMMETRIC) - cipher.evp.EVP_DecryptFinal_ex = decrypt_final - - -class TestSignature(unittest.TestCase): - - def round_trip(self, keys, text): - s = signature.sign(text, keys[0]) - return signature.verify(text, s, keys[1]) - - def round_trip_strings(self, keys, text): - s = signature.sign(text, key=open(keys[0], 'rb').read()) - v = signature.verify(text, s, key=open(keys[1], 'rb').read()) - return v - - def round_trip_all_keys(self, text): - self.assertTrue(self.round_trip(KEY_1, text)) - self.assertTrue(self.round_trip(KEY_2, text)) - self.assertFalse(self.round_trip(MISMATCH_1, text)) - self.assertFalse(self.round_trip(MISMATCH_2, text)) - self.assertTrue(self.round_trip_strings(KEY_1, text)) - self.assertTrue(self.round_trip_strings(KEY_2, text)) - self.assertFalse(self.round_trip_strings(MISMATCH_1, text)) - self.assertFalse(self.round_trip_strings(MISMATCH_2, text)) - - def test_round_trip_long(self): - self.round_trip_all_keys(LONG) - - def test_round_trip_short(self): - self.round_trip_all_keys(SHORT) - - def test_round_trip_unicode(self): - self.round_trip_all_keys(UNICODE) - - def test_round_trip_null(self): - self.round_trip_all_keys(NULL) - - def test_round_trip_zero(self): - self.round_trip_all_keys('') - - def test_arguments(self): - text = SHORT - keys = KEY_1 - self.failUnlessRaises(signature.SignatureError, signature.sign, text, key=open(keys[0], 'rb').read(), keyfile=keys[0]) - self.failUnlessRaises(signature.SignatureError, signature.sign, text) - s = signature.sign(text, keyfile=keys[0]) - self.failUnlessRaises(signature.SignatureError, signature.verify, text, s, key=open(keys[1], 'rb').read(), keyfile=keys[1]) - self.failUnlessRaises(signature.SignatureError, signature.verify, text, s) - - def test_bad_ctx(self): - s = signature.sign(SHORT, KEY_1[0]) - new_ctx = signature.evp.EVP_MD_CTX_create - signature.evp.EVP_MD_CTX_create = lambda: None - self.assertRaises(signature.SignatureError, signature.sign, SHORT, KEY_1[0]) - self.assertRaises(signature.SignatureError, signature.verify, SHORT, s, KEY_1[1]) - signature.evp.EVP_MD_CTX_create = new_ctx - - def test_bad_hash(self): - s = signature.sign(SHORT, KEY_1[0]) - new_hash = signature.evp.EVP_get_digestbyname - signature.evp.EVP_get_digestbyname = lambda a: None - self.assertRaises(signature.SignatureError, signature.sign, SHORT, KEY_1[0]) - self.assertRaises(signature.SignatureError, signature.verify, SHORT, s, KEY_1[1]) - signature.evp.EVP_get_digestbyname = new_hash - - def test_bad_digest_init(self): - s = signature.sign(SHORT, KEY_1[0]) - init = signature.evp.EVP_DigestInit - signature.evp.EVP_DigestInit = lambda a,b: None - self.assertRaises(signature.SignatureError, signature.sign, SHORT, KEY_1[0]) - self.assertRaises(signature.SignatureError, signature.verify, SHORT, s, KEY_1[1]) - signature.evp.EVP_DigestInit = init - - def test_bad_digest_update(self): - s = signature.sign(SHORT, KEY_1[0]) - update = signature.evp.EVP_DigestUpdate - signature.evp.EVP_DigestUpdate = lambda a,b,c: None - self.assertRaises(signature.SignatureError, signature.sign, SHORT, KEY_1[0]) - self.assertRaises(signature.SignatureError, signature.verify, SHORT, s, KEY_1[1]) - signature.evp.EVP_DigestUpdate = update - - def test_bad_sign_final(self): - final = signature.evp.EVP_SignFinal - signature.evp.EVP_SignFinal = lambda a,b,c,d: None - self.assertRaises(signature.SignatureError, signature.sign, SHORT, KEY_1[0]) - signature.evp.EVP_SignFinal = final - - def test_bad_verify_final(self): - s = signature.sign(SHORT, KEY_1[0]) - final = signature.evp.EVP_VerifyFinal - signature.evp.EVP_VerifyFinal = lambda a,b,c,d: None - self.assertRaises(signature.SignatureError, signature.verify, SHORT, s, KEY_1[1]) - signature.evp.EVP_VerifyFinal = final - - def test_bad_read_privatekey(self): - read_private_key = signature.evp.PEM_read_PrivateKey - signature.evp.PEM_read_PrivateKey = lambda a,b,c,d: None - self.assertRaises(signature.SignatureError, signature.sign, SHORT, KEY_1[0]) - signature.evp.PEM_read_PrivateKey = read_private_key - - def test_bad_read_publickey(self): - s = signature.sign(SHORT, KEY_1[0]) - read_public_key = signature.evp.PEM_read_PUBKEY - signature.evp.PEM_read_PUBKEY = lambda a,b,c,d: None - self.assertRaises(signature.SignatureError, signature.verify, SHORT, s, KEY_1[1]) - signature.evp.PEM_read_PUBKEY = read_public_key - - def test_bad_read_bio_privatekey(self): - read_private_key = signature.evp.PEM_read_bio_PrivateKey - signature.evp.PEM_read_bio_PrivateKey = lambda a,b,c,d,e: None - self.assertRaises(signature.SignatureError, signature.sign, SHORT, key=open(KEY_1[0], 'rb').read()) - signature.evp.PEM_read_bio_PrivateKey = read_private_key - - def test_bad_read_bio_publickey(self): - s = signature.sign(SHORT, KEY_1[0]) - read_public_key = signature.evp.PEM_read_bio_PUBKEY - signature.evp.PEM_read_bio_PUBKEY = lambda a,b,c,d: None - self.assertRaises(signature.SignatureError, signature.verify, SHORT, s, key=open(KEY_1[1], 'rb').read()) - signature.evp.PEM_read_bio_PUBKEY = read_public_key - - def test_bad_fopen(self): - s = signature.sign(SHORT, KEY_1[0]) - file_open = signature.evp.fopen - signature.evp.fopen = lambda a,b: None - self.assertRaises(signature.SignatureError, signature.sign, SHORT, KEY_1[1]) - self.assertRaises(signature.SignatureError, signature.verify, SHORT, s, KEY_1[0]) - signature.evp.fopen = file_open - - -class TestEnvelope(unittest.TestCase): - - def round_trip(self, keys, text): - iv, sym_key, enc = envelope.encrypt(text, keys[1]) - return envelope.decrypt(iv, sym_key, enc, keys[0]) - - def round_trip_strings(self, keys, text): - iv, sym_key, enc = envelope.encrypt(text, key=open(keys[1], 'rb').read()) - return envelope.decrypt(iv, sym_key, enc, key=open(keys[0], 'rb').read()) - - def test_round_trip_zero(self): - self.assertRaises(envelope.EnvelopeError, self.round_trip, KEY_1, '') - self.assertRaises(envelope.EnvelopeError, self.round_trip, KEY_2, '') - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_1, '') - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_2, '') - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, KEY_1, '') - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, KEY_2, '') - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_1, '') - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_2, '') - - def test_round_trip_long(self): - self.assertEqual(self.round_trip_strings(KEY_1, LONG), LONG, "Failed to round trip") - self.assertEqual(self.round_trip_strings(KEY_2, LONG), LONG, "Failed to round trip") - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_1, LONG) - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_2, LONG) - self.assertEqual(self.round_trip(KEY_1, LONG), LONG, "Failed to round trip") - self.assertEqual(self.round_trip(KEY_2, LONG), LONG, "Failed to round trip") - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_1, LONG) - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_2, LONG) - - def test_round_trip_short(self): - self.assertEqual(self.round_trip(KEY_1, SHORT), SHORT, "Failed to round trip") - self.assertEqual(self.round_trip(KEY_2, SHORT), SHORT, "Failed to round trip") - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_1, SHORT) - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_2, SHORT) - self.assertEqual(self.round_trip_strings(KEY_1, SHORT), SHORT, "Failed to round trip") - self.assertEqual(self.round_trip_strings(KEY_2, SHORT), SHORT, "Failed to round trip") - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_1, SHORT) - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_2, SHORT) - - def test_round_trip_unicode(self): - self.assertEqual(self.round_trip(KEY_1, UNICODE), UNICODE, "Failed to round trip") - self.assertEqual(self.round_trip(KEY_2, UNICODE), UNICODE, "Failed to round trip") - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_1, UNICODE) - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_2, UNICODE) - self.assertEqual(self.round_trip_strings(KEY_1, UNICODE), UNICODE, "Failed to round trip") - self.assertEqual(self.round_trip_strings(KEY_2, UNICODE), UNICODE, "Failed to round trip") - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_1, UNICODE) - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_2, UNICODE) - - def test_round_trip_null(self): - self.assertEqual(self.round_trip(KEY_1, NULL), NULL, "Failed to round trip") - self.assertEqual(self.round_trip(KEY_2, NULL), NULL, "Failed to round trip") - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_1, NULL) - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISMATCH_2, NULL) - self.assertEqual(self.round_trip_strings(KEY_1, NULL), NULL, "Failed to round trip") - self.assertEqual(self.round_trip_strings(KEY_2, NULL), NULL, "Failed to round trip") - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_1, NULL) - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, MISMATCH_2, NULL) - - def test_bad_keys(self): - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISSING_PUBLIC_KEY, SHORT) - self.assertRaises(envelope.EnvelopeError, self.round_trip, MISSING_PRIVATE_KEY, SHORT) - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, BLANK_PUBLIC_KEY, SHORT) - self.assertRaises(envelope.EnvelopeError, self.round_trip_strings, BLANK_PRIVATE_KEY, SHORT) - - def test_bad_call(self): - # neither key nor keyfile - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT) - # both - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[0], open(KEY_1[0], 'rb').read()) - # string key instead of bytes - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, key=open(KEY_1[0], 'r').read()) - # string data instead of bytes - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, str(SHORT), KEY_1[0], open(KEY_1[0], 'rb').read()) - # get valid encryption data - iv, aes_key, enc = envelope.encrypt(SHORT, KEY_1[1]) - # neither key nor keyfile - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, aes_key, enc) - # both - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, aes_key, enc, KEY_1[1], key=open(KEY_1[1], 'rb').read()) - # string key instead of bytes - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, aes_key, enc, key=open(KEY_1[1], 'r').read()) - # string data instead of bytes - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, aes_key, str(enc), KEY_1[1], key=open(KEY_1[1], 'rb').read()) - # string iv instead of bytes - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, str(iv), aes_key, enc, KEY_1[1], key=open(KEY_1[1], 'rb').read()) - # string key instead of bytes - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, str(aes_key), enc, KEY_1[1], key=open(KEY_1[1], 'rb').read()) - - def test_bad_rand_bytes_1(self): - rand_bytes = envelope.evp.RAND_bytes - envelope.evp.RAND_bytes = lambda a,b:0 - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.RAND_bytes = rand_bytes - - def test_bad_rand_bytes_2(self): - rand_bytes = envelope.evp.RAND_bytes - envelope.evp.RAND_bytes = run_n_times(evp.RAND_bytes, lambda a,b:0, 1) - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.RAND_bytes = rand_bytes - - def test_bad_rsa_size(self): - rsa_size = envelope.evp.RSA_size - envelope.evp.RSA_size = lambda a:0 - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.RSA_size = rsa_size - - def test_bad_read_privatekey(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - read_private_key = envelope.evp.PEM_read_PrivateKey - envelope.evp.PEM_read_PrivateKey = lambda a,b,c,d: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - envelope.evp.PEM_read_PrivateKey = read_private_key - - def test_bad_read_publickey(self): - read_public_key = envelope.evp.PEM_read_PUBKEY - envelope.evp.PEM_read_PUBKEY = lambda a,b,c,d: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.PEM_read_PUBKEY = read_public_key - - def test_bad_read_bio_privatekey(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - read_private_key = envelope.evp.PEM_read_bio_PrivateKey - envelope.evp.PEM_read_bio_PrivateKey = lambda a,b,c,d: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, key=open(KEY_1[0], 'rb').read()) - envelope.evp.PEM_read_bio_PrivateKey = read_private_key - - def test_bad_read_bio_publickey(self): - read_public_key = envelope.evp.PEM_read_bio_PUBKEY - envelope.evp.PEM_read_bio_PUBKEY = lambda a,b,c,d: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, key=open(KEY_1[0], 'rb').read()) - envelope.evp.PEM_read_bio_PUBKEY = read_public_key - - def test_bad_fopen(self): - iv, aes_key, enc = envelope.encrypt(SHORT, KEY_1[1]) - file_open = envelope.evp.fopen - envelope.evp.fopen = lambda a,b: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, aes_key, enc, KEY_1[0]) - envelope.evp.fopen = file_open - - def test_bad_ctx_new(self): - iv, aes_key, enc = envelope.encrypt(SHORT, KEY_1[1]) - new_ctx = envelope.evp.EVP_CIPHER_CTX_new - envelope.evp.EVP_CIPHER_CTX_new = lambda: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, aes_key, enc, KEY_1[0]) - envelope.evp.EVP_CIPHER_CTX_new = new_ctx - - def test_bad_cipher_object(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - cipher_getter = envelope.evp.EVP_aes_192_cbc - envelope.evp.EVP_aes_192_cbc= lambda: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - envelope.evp.EVP_aes_192_cbc = cipher_getter - - def test_bad_encrypt_init_1(self): - encrypt_init = envelope.evp.EVP_EncryptInit_ex - envelope.evp.EVP_EncryptInit_ex = lambda a,b,c,d,e: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_EncryptInit_ex = encrypt_init - - def test_bad_encrypt_init_2(self): - encrypt_init = envelope.evp.EVP_EncryptInit_ex - envelope.evp.EVP_EncryptInit_ex = run_n_times(envelope.evp.EVP_EncryptInit_ex, lambda a,b,c,d,e: None, 1) - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_EncryptInit_ex = encrypt_init - - def test_bad_encrypt_update(self): - encrypt_update = envelope.evp.EVP_EncryptUpdate - envelope.evp.EVP_EncryptUpdate = lambda a,b,c,d,e: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_EncryptUpdate = encrypt_update - - def test_bad_encrypt_final(self): - encrypt_final = envelope.evp.EVP_EncryptFinal_ex - envelope.evp.EVP_EncryptFinal_ex = lambda a,b,c: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_EncryptFinal_ex = encrypt_final - - def test_bad_open_init(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - decrypt_init = envelope.evp.EVP_OpenInit - envelope.evp.EVP_OpenInit = lambda a,b,c,d,e,f: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - envelope.evp.EVP_OpenInit = decrypt_init - - def test_bad_decrypt_update(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - decrypt_update = envelope.evp.EVP_DecryptUpdate - envelope.evp.EVP_DecryptUpdate = lambda a,b,c,d,e: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - envelope.evp.EVP_DecryptUpdate = decrypt_update - - def test_bad_decrypt_final(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - decrypt_final = envelope.evp.EVP_DecryptFinal_ex - envelope.evp.EVP_DecryptFinal_ex = lambda a,b,c: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - cipher.evp.EVP_DecryptFinal_ex = decrypt_final - - def test_bad_rsa_get(self): - get_rsa = envelope.evp.EVP_PKEY_get1_RSA - envelope.evp.EVP_PKEY_get1_RSA = lambda a: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_PKEY_get1_RSA = get_rsa - - def test_bad_rsa_encrypt(self): - encrypt_rsa = envelope.evp.RSA_public_encrypt - envelope.evp.RSA_public_encrypt = lambda a,b,c,d,e: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.RSA_public_encrypt = encrypt_rsa - - def test_bad_read_privatekey(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - read_private_key = envelope.evp.PEM_read_PrivateKey - envelope.evp.PEM_read_PrivateKey = lambda a,b,c,d: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - envelope.evp.PEM_read_PrivateKey = read_private_key - - def test_bad_read_publickey(self): - read_public_key = envelope.evp.PEM_read_PUBKEY - envelope.evp.PEM_read_PUBKEY = lambda a,b,c,d: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.PEM_read_PUBKEY = read_public_key - - def test_bad_read_bio_privatekey(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - read_private_key = envelope.evp.PEM_read_bio_PrivateKey - envelope.evp.PEM_read_bio_PrivateKey = lambda a,b,c,d: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, key=open(KEY_1[0], 'rb').read()) - envelope.evp.PEM_read_bio_PrivateKey = read_private_key - - def test_bad_read_bio_publickey(self): - read_public_key = envelope.evp.PEM_read_bio_PUBKEY - envelope.evp.PEM_read_bio_PUBKEY = lambda a,b,c,d: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, key=open(KEY_1[0], 'rb').read()) - envelope.evp.PEM_read_bio_PUBKEY = read_public_key - - def test_bad_fopen(self): - iv, aes_key, enc = envelope.encrypt(SHORT, KEY_1[1]) - file_open = envelope.evp.fopen - envelope.evp.fopen = lambda a,b: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, aes_key, enc, KEY_1[0]) - envelope.evp.fopen = file_open - - def test_bad_ctx_new(self): - iv, aes_key, enc = envelope.encrypt(SHORT, KEY_1[1]) - new_ctx = envelope.evp.EVP_CIPHER_CTX_new - envelope.evp.EVP_CIPHER_CTX_new = lambda: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, aes_key, enc, KEY_1[0]) - envelope.evp.EVP_CIPHER_CTX_new = new_ctx - - def test_bad_cipher_object(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - cipher_getter = envelope.evp.EVP_aes_192_cbc - envelope.evp.EVP_aes_192_cbc= lambda: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - envelope.evp.EVP_aes_192_cbc = cipher_getter - - def test_bad_encrypt_init(self): - encrypt_init = envelope.evp.EVP_EncryptInit_ex - envelope.evp.EVP_EncryptInit_ex = lambda a,b,c,d,e: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_EncryptInit_ex = encrypt_init - - def test_bad_encrypt_update(self): - encrypt_update = envelope.evp.EVP_EncryptUpdate - envelope.evp.EVP_EncryptUpdate = lambda a,b,c,d,e: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_EncryptUpdate = encrypt_update - - def test_bad_encrypt_final(self): - encrypt_final = envelope.evp.EVP_EncryptFinal_ex - envelope.evp.EVP_EncryptFinal_ex = lambda a,b,c: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_EncryptFinal_ex = encrypt_final - - def test_bad_open_init(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - decrypt_init = envelope.evp.EVP_OpenInit - envelope.evp.EVP_OpenInit = lambda a,b,c,d,e,f: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - envelope.evp.EVP_OpenInit = decrypt_init - - def test_bad_decrypt_update(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - decrypt_update = envelope.evp.EVP_DecryptUpdate - envelope.evp.EVP_DecryptUpdate = lambda a,b,c,d,e: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - envelope.evp.EVP_DecryptUpdate = decrypt_update - - def test_bad_decrypt_final(self): - iv, key, enc = envelope.encrypt(SHORT, KEY_1[1]) - decrypt_final = envelope.evp.EVP_DecryptFinal_ex - envelope.evp.EVP_DecryptFinal_ex = lambda a,b,c: None - self.assertRaises(envelope.EnvelopeError, envelope.decrypt, iv, key, enc, KEY_1[0]) - cipher.evp.EVP_DecryptFinal_ex = decrypt_final - - def test_bad_rsa_get(self): - get_rsa = envelope.evp.EVP_PKEY_get1_RSA - envelope.evp.EVP_PKEY_get1_RSA = lambda a: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.EVP_PKEY_get1_RSA = get_rsa - - def test_bad_rsa_encrypt(self): - encrypt_rsa = envelope.evp.RSA_public_encrypt - envelope.evp.RSA_public_encrypt = lambda a,b,c,d,e: None - self.assertRaises(envelope.EnvelopeError, envelope.encrypt, SHORT, KEY_1[1]) - envelope.evp.RSA_public_encrypt = encrypt_rsa - -if __name__ == "__main__": - unittest.main() diff --git a/setup.py b/setup.py index f53495b9..d035942f 100755 --- a/setup.py +++ b/setup.py @@ -67,9 +67,8 @@ author='https://www.updateframework.com', author_email='info@updateframework.com', url='https://www.updateframework.com', - install_requires=['pycrypto>2.0'], + install_requires=['pycrypto>=2.6'], packages=[ - 'evpy', 'tuf', 'tuf.client', 'tuf.compatibility', diff --git a/tuf/tests/integration/slow_retrieval_server.py b/tests/integration/slow_retrieval_server.py similarity index 100% rename from tuf/tests/integration/slow_retrieval_server.py rename to tests/integration/slow_retrieval_server.py diff --git a/tuf/tests/integration/test_arbitrary_package_attack.py b/tests/integration/test_arbitrary_package_attack.py similarity index 100% rename from tuf/tests/integration/test_arbitrary_package_attack.py rename to tests/integration/test_arbitrary_package_attack.py diff --git a/tuf/tests/integration/test_delegations.py b/tests/integration/test_delegations.py similarity index 100% rename from tuf/tests/integration/test_delegations.py rename to tests/integration/test_delegations.py diff --git a/tuf/tests/integration/test_endless_data_attack.py b/tests/integration/test_endless_data_attack.py similarity index 100% rename from tuf/tests/integration/test_endless_data_attack.py rename to tests/integration/test_endless_data_attack.py diff --git a/tuf/tests/integration/test_extraneous_dependencies_attack.py b/tests/integration/test_extraneous_dependencies_attack.py similarity index 100% rename from tuf/tests/integration/test_extraneous_dependencies_attack.py rename to tests/integration/test_extraneous_dependencies_attack.py diff --git a/tuf/tests/integration/test_indefinite_freeze_attack.py b/tests/integration/test_indefinite_freeze_attack.py similarity index 100% rename from tuf/tests/integration/test_indefinite_freeze_attack.py rename to tests/integration/test_indefinite_freeze_attack.py diff --git a/tuf/tests/integration/test_mix_and_match_attack.py b/tests/integration/test_mix_and_match_attack.py similarity index 100% rename from tuf/tests/integration/test_mix_and_match_attack.py rename to tests/integration/test_mix_and_match_attack.py diff --git a/tuf/tests/integration/test_replay_attack.py b/tests/integration/test_replay_attack.py similarity index 100% rename from tuf/tests/integration/test_replay_attack.py rename to tests/integration/test_replay_attack.py diff --git a/tuf/tests/integration/test_slow_retrieval_attack.py b/tests/integration/test_slow_retrieval_attack.py similarity index 100% rename from tuf/tests/integration/test_slow_retrieval_attack.py rename to tests/integration/test_slow_retrieval_attack.py diff --git a/tuf/tests/unit/aggregate_tests.py b/tests/unit/aggregate_tests.py similarity index 100% rename from tuf/tests/unit/aggregate_tests.py rename to tests/unit/aggregate_tests.py diff --git a/tuf/tests/unit/repository_setup.py b/tests/unit/repository_setup.py similarity index 100% rename from tuf/tests/unit/repository_setup.py rename to tests/unit/repository_setup.py diff --git a/tuf/tests/unit/simple_server.py b/tests/unit/simple_server.py similarity index 100% rename from tuf/tests/unit/simple_server.py rename to tests/unit/simple_server.py diff --git a/tuf/tests/unit/statement_coverage.py b/tests/unit/statement_coverage.py similarity index 100% rename from tuf/tests/unit/statement_coverage.py rename to tests/unit/statement_coverage.py diff --git a/tuf/tests/unit/test_download.py b/tests/unit/test_download.py similarity index 100% rename from tuf/tests/unit/test_download.py rename to tests/unit/test_download.py diff --git a/tuf/tests/unit/test_formats.py b/tests/unit/test_formats.py similarity index 100% rename from tuf/tests/unit/test_formats.py rename to tests/unit/test_formats.py diff --git a/tuf/tests/unit/test_hash.py b/tests/unit/test_hash.py similarity index 100% rename from tuf/tests/unit/test_hash.py rename to tests/unit/test_hash.py diff --git a/tuf/tests/unit/test_keydb.py b/tests/unit/test_keydb.py similarity index 100% rename from tuf/tests/unit/test_keydb.py rename to tests/unit/test_keydb.py diff --git a/tuf/tests/unit/test_keystore.py b/tests/unit/test_keystore.py similarity index 100% rename from tuf/tests/unit/test_keystore.py rename to tests/unit/test_keystore.py diff --git a/tuf/tests/unit/test_mirrors.py b/tests/unit/test_mirrors.py similarity index 100% rename from tuf/tests/unit/test_mirrors.py rename to tests/unit/test_mirrors.py diff --git a/tuf/tests/unit/test_push.py b/tests/unit/test_push.py similarity index 100% rename from tuf/tests/unit/test_push.py rename to tests/unit/test_push.py diff --git a/tuf/tests/unit/test_pushtoolslib.py b/tests/unit/test_pushtoolslib.py similarity index 100% rename from tuf/tests/unit/test_pushtoolslib.py rename to tests/unit/test_pushtoolslib.py diff --git a/tuf/tests/unit/test_quickstart.py b/tests/unit/test_quickstart.py similarity index 100% rename from tuf/tests/unit/test_quickstart.py rename to tests/unit/test_quickstart.py diff --git a/tuf/tests/unit/test_roledb.py b/tests/unit/test_roledb.py similarity index 100% rename from tuf/tests/unit/test_roledb.py rename to tests/unit/test_roledb.py diff --git a/tuf/tests/unit/test_rsa_key.py b/tests/unit/test_rsa_key.py similarity index 100% rename from tuf/tests/unit/test_rsa_key.py rename to tests/unit/test_rsa_key.py diff --git a/tuf/tests/unit/test_schema.py b/tests/unit/test_schema.py similarity index 100% rename from tuf/tests/unit/test_schema.py rename to tests/unit/test_schema.py diff --git a/tuf/tests/unit/test_sig.py b/tests/unit/test_sig.py similarity index 100% rename from tuf/tests/unit/test_sig.py rename to tests/unit/test_sig.py diff --git a/tuf/tests/unit/test_signercli.py b/tests/unit/test_signercli.py similarity index 100% rename from tuf/tests/unit/test_signercli.py rename to tests/unit/test_signercli.py diff --git a/tuf/tests/unit/test_signerlib.py b/tests/unit/test_signerlib.py similarity index 100% rename from tuf/tests/unit/test_signerlib.py rename to tests/unit/test_signerlib.py diff --git a/tuf/tests/unit/test_updater.py b/tests/unit/test_updater.py similarity index 100% rename from tuf/tests/unit/test_updater.py rename to tests/unit/test_updater.py diff --git a/tuf/tests/unit/test_util.py b/tests/unit/test_util.py similarity index 100% rename from tuf/tests/unit/test_util.py rename to tests/unit/test_util.py diff --git a/tuf/tests/test_util_test_tools.py b/tests/unit/test_util_test_tools.py similarity index 99% rename from tuf/tests/test_util_test_tools.py rename to tests/unit/test_util_test_tools.py index 8ca8ffa8..f4ae5871 100755 --- a/tuf/tests/test_util_test_tools.py +++ b/tests/unit/test_util_test_tools.py @@ -20,7 +20,8 @@ import os import urllib import unittest -import util_test_tools + +from tuf.tests import util_test_tools diff --git a/tuf/tests/unit/unittest_toolbox.py b/tests/unit/unittest_toolbox.py similarity index 100% rename from tuf/tests/unit/unittest_toolbox.py rename to tests/unit/unittest_toolbox.py diff --git a/tuf/tests/__init__.py b/tuf/tests/__init__.py index 582fc3b3..3235da9f 100644 --- a/tuf/tests/__init__.py +++ b/tuf/tests/__init__.py @@ -1 +1,4 @@ +# The tuf.tests package is intended to export as little utility as possible to +# enable easier testing of TUF. + __all__ = ['util_test_tools'] From 864d3065a8aa54620ad340c4c8110fe544432daa Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 10 Sep 2013 00:56:30 -0400 Subject: [PATCH 096/119] Merge unit test fix from @zanefisher; fix a unit test bug. --- tests/unit/test_download.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index c93038dc..07715b3c 100755 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -110,20 +110,17 @@ def test_download_url_to_tempfileobj_and_lengths(self): download.safe_download(self.url, self.target_data_length - 1) download.unsafe_download(self.url, self.target_data_length - 1) - # NOTE: We catch tuf.DownloadError here because the STRICT_REQUIRED_LENGTH, - # which is True by default, mandates that we must download exactly what is - # required. - exception_message = 'Downloaded '+str(self.target_data_length)+\ - ' bytes, but expected '+\ - str(self.target_data_length+1)+\ - ' bytes. There is a difference of 1 bytes!' - self.assertRaisesRegexp(tuf.DownloadError, exception_message, - download.safe_download, self.url, - self.target_data_length + 1) + # NOTE: We catch tuf.DownloadLengthMismatchError here because the + # STRICT_REQUIRED_LENGTH, which is True by default, mandates that we must + # download exactly what is required. + self.assertRaises(tuf.DownloadLengthMismatchError, download.safe_download, + self.url, self.target_data_length + 1) - # NOTE: However, we do not catch a tuf.DownloadError here for the same test - # as the previous one because we have disabled STRICT_REQUIRED_LENGTH. - temp_fileobj = download.unsafe_download(self.url, self.target_data_length + 1) + # NOTE: However, we do not catch a tuf.DownloadLengthMismatchError here for + # the same test as the previous one because we have disabled + # STRICT_REQUIRED_LENGTH. + temp_fileobj = download.unsafe_download(self.url, + self.target_data_length + 1) self.assertEquals(self.target_data, temp_fileobj.read()) self.assertEquals(self.target_data_length, len(temp_fileobj.read())) temp_fileobj.close_temp_file() From 230b157c22f7656b21d8df7533c09780999b0174 Mon Sep 17 00:00:00 2001 From: vladdd Date: Tue, 10 Sep 2013 13:23:46 -0400 Subject: [PATCH 097/119] Review and update crypto comments in rsa_key.py --- tuf/rsa_key.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tuf/rsa_key.py b/tuf/rsa_key.py index 1b825640..b61b7d51 100755 --- a/tuf/rsa_key.py +++ b/tuf/rsa_key.py @@ -34,10 +34,10 @@ get the keyid of a key object by simply accessing the dictionary's 'keyid' key (i.e., rsakey['keyid']). -""" + """ -# Required for hexadecimal conversions. +# Required for hexadecimal conversions. Signatures are hexlified. import binascii # Crypto.PublicKey (i.e., PyCrypto public-key cryptography) provides algorithms @@ -47,8 +47,15 @@ # PyCrypto requires 'Crypto.Hash' hash objects to generate PKCS#1 PSS # signatures (i.e., Crypto.Signature.PKCS1_PSS). -# https://tools.ietf.org/html/rfc3447 import Crypto.Hash.SHA256 + +# RSA's probabilistic signature scheme with appendix (RSASSA-PSS). +# PKCS#1 v1.5 is provided for compatability with existing applications, but +# RSASSA-PSS is encouraged for newer applications. RSASSA-PSS generates +# a random salt to ensure the signature generated is probabilistic rather than +# deterministic, like PKCS#1 v1.5. +# http://en.wikipedia.org/wiki/RSA-PSS#Schemes +# https://tools.ietf.org/html/rfc3447#section-8.1 import Crypto.Signature.PKCS1_PSS import tuf @@ -89,6 +96,7 @@ def generate(bits=_DEFAULT_RSA_KEY_BITS): ValueError, if an exception occurs after calling the RSA key generation routine. 'bits' must be 1024, or greater, and a multiple of 256. + Raised by Cryptography library. tuf.FormatError, if 'bits' does not contain the correct format. @@ -148,17 +156,18 @@ def generate(bits=_DEFAULT_RSA_KEY_BITS): def create_in_metadata_format(key_value, private=False): """ - Return a dictionary conformant to tuf.formats.KEY_SCHEMA. + Return a dictionary conformant to 'tuf.formats.KEY_SCHEMA'. If 'private' is True, include the private key. The dictionary returned has the form: {'keytype': 'rsa', 'keyval': {'public': '-----BEGIN RSA PUBLIC KEY----- ...', 'private': '-----BEGIN RSA PRIVATE KEY----- ...'}} - or + + or if 'private' is False: {'keytype': 'rsa', 'keyval': {'public': '-----BEGIN RSA PUBLIC KEY----- ...', - 'private': ''}} if 'private' is False. + 'private': ''}} The private and public keys are in PEM format. @@ -172,7 +181,7 @@ def create_in_metadata_format(key_value, private=False): {'public': '-----BEGIN RSA PUBLIC KEY----- ...', 'private': '-----BEGIN RSA PRIVATE KEY----- ...'}}, - conformat to tuf.formats.KEYVAL_SCHEMA. + conformat to 'tuf.formats.KEYVAL_SCHEMA'. private: Indicates if the private key should be included in the @@ -197,7 +206,7 @@ def create_in_metadata_format(key_value, private=False): # Raise 'tuf.FormatError' if the check fails. tuf.formats.KEYVAL_SCHEMA.check_match(key_value) - if private and key_value['private']: + if private is True and key_value['private']: return {'keytype': 'rsa', 'keyval': key_value} else: public_key_value = {'public': key_value['public'], 'private': ''} @@ -262,6 +271,8 @@ def create_from_metadata_format(key_metadata): keytype = 'rsa' key_value = key_metadata['keyval'] + # Convert 'key_value' to 'tuf.formats.KEY_SCHEMA' and generate its hash + # The hash is in hexdigest form. keyid = _get_keyid(key_value) # We now have all the required key values. Build 'rsakey_dict'. @@ -563,6 +574,8 @@ def create_from_encrypted_pem(encrypted_pem, passphrase): where the private part of the RSA key is encrypted. PyCrypto's importKey method is used, where a passphrase is specified. PyCrypto uses PBKDF1+MD5 to strengthen 'passphrase', and 3DES with CBC mode for encryption/decryption. + Alternatively, key data may be encrypted with AES-CTR-Mode and the passphrase + strengthened with PBKDF2+SHA256. See 'keystore.py'. encrypted_pem: @@ -609,6 +622,9 @@ def create_from_encrypted_pem(encrypted_pem, passphrase): except (ValueError, IndexError, TypeError), e: message = 'An RSA key object could not be generated from the encrypted '+\ 'PEM string.' + # Raise 'tuf.CryptoError' instead of PyCrypto's exception to avoid + # revealing sensitive error, such as a decryption error due to an + # invalid passphrase. raise tuf.CryptoError(message) # Extract the public & private halves of the RSA key and generate their From 59dfa1f5ba4980dd00e3eb34e3fc94bf45c1448f Mon Sep 17 00:00:00 2001 From: dachshund Date: Tue, 10 Sep 2013 17:33:00 -0400 Subject: [PATCH 098/119] Replace relative with absolute imports for tests. --- tests/integration/test_arbitrary_package_attack.py | 4 ++-- tests/integration/test_delegations.py | 2 +- tests/integration/test_endless_data_attack.py | 4 ++-- tests/integration/test_extraneous_dependencies_attack.py | 6 +++--- tests/integration/test_indefinite_freeze_attack.py | 4 ++-- tests/integration/test_mix_and_match_attack.py | 4 ++-- tests/integration/test_replay_attack.py | 4 ++-- tests/integration/test_slow_retrieval_attack.py | 4 ++-- tests/unit/test_download.py | 2 +- tests/unit/test_mirrors.py | 2 +- tests/unit/test_quickstart.py | 2 +- tests/unit/test_signercli.py | 2 +- tests/unit/test_signerlib.py | 2 +- tests/unit/test_updater.py | 4 ++-- tests/unit/test_util.py | 2 +- tests/unit/test_util_test_tools.py | 2 +- tuf/tests/__init__.py | 2 +- {tests/unit => tuf/tests}/repository_setup.py | 2 +- {tests/unit => tuf/tests}/unittest_toolbox.py | 0 19 files changed, 27 insertions(+), 27 deletions(-) rename {tests/unit => tuf/tests}/repository_setup.py (99%) rename {tests/unit => tuf/tests}/unittest_toolbox.py (100%) diff --git a/tests/integration/test_arbitrary_package_attack.py b/tests/integration/test_arbitrary_package_attack.py index 67ac627e..8164c678 100755 --- a/tests/integration/test_arbitrary_package_attack.py +++ b/tests/integration/test_arbitrary_package_attack.py @@ -35,8 +35,8 @@ import tempfile import tuf -from tuf.interposition import urllib_tuf -from tuf.tests import util_test_tools +import tuf.interposition.urllib_tuf as urllib_tuf +import tuf.tests.util_test_tools as util_test_tools diff --git a/tests/integration/test_delegations.py b/tests/integration/test_delegations.py index d84127d0..94f84917 100755 --- a/tests/integration/test_delegations.py +++ b/tests/integration/test_delegations.py @@ -30,7 +30,7 @@ import tuf.repo.keystore as keystore import tuf.repo.signercli as signercli import tuf.repo.signerlib as signerlib -from tuf.tests import util_test_tools +import tuf.tests.util_test_tools as util_test_tools version = 1 # Modify the number of iterations (from the higher default count) so the unit diff --git a/tests/integration/test_endless_data_attack.py b/tests/integration/test_endless_data_attack.py index 4d445daf..3f1e0ac8 100755 --- a/tests/integration/test_endless_data_attack.py +++ b/tests/integration/test_endless_data_attack.py @@ -37,8 +37,8 @@ import urllib import tuf -from tuf.interposition import urllib_tuf -from tuf.tests import util_test_tools +import tuf.interposition.urllib_tuf as urllib_tuf +import tuf.tests.util_test_tools as util_test_tools diff --git a/tests/integration/test_extraneous_dependencies_attack.py b/tests/integration/test_extraneous_dependencies_attack.py index f07b4cde..2813bdfd 100755 --- a/tests/integration/test_extraneous_dependencies_attack.py +++ b/tests/integration/test_extraneous_dependencies_attack.py @@ -43,8 +43,8 @@ import tempfile import tuf -import tuf.interposition -from tuf.tests import util_test_tools +import tuf.interposition.urllib_tuf as urllib_tuf +import tuf.tests.util_test_tools as util_test_tools class ExtraneousDependencyAlert(Exception): @@ -57,7 +57,7 @@ class ExtraneousDependencyAlert(Exception): def _download(url, filename, directory, TUF=False): destination = os.path.join(directory, filename) if TUF: - tuf.interposition.urllib_tuf.urlretrieve(url, destination) + urllib_tuf.urlretrieve(url, destination) else: urllib.urlretrieve(url, destination) diff --git a/tests/integration/test_indefinite_freeze_attack.py b/tests/integration/test_indefinite_freeze_attack.py index 10887433..b422fae5 100755 --- a/tests/integration/test_indefinite_freeze_attack.py +++ b/tests/integration/test_indefinite_freeze_attack.py @@ -28,9 +28,9 @@ import tuf import tuf.formats +import tuf.interposition.urllib_tuf as urllib_tuf import tuf.repo.signerlib as signerlib -from tuf.interposition import urllib_tuf -from tuf.tests import util_test_tools +import tuf.tests.util_test_tools as util_test_tools class IndefiniteFreezeAttackAlert(Exception): diff --git a/tests/integration/test_mix_and_match_attack.py b/tests/integration/test_mix_and_match_attack.py index ae6efd59..da8e170c 100755 --- a/tests/integration/test_mix_and_match_attack.py +++ b/tests/integration/test_mix_and_match_attack.py @@ -40,8 +40,8 @@ import tempfile import tuf -from tuf.interposition import urllib_tuf -from tuf.tests import util_test_tools +import tuf.interposition.urllib_tuf as urllib_tuf +import tuf.tests.util_test_tools as util_test_tools class MixAndMatchAttackAlert(Exception): diff --git a/tests/integration/test_replay_attack.py b/tests/integration/test_replay_attack.py index f2b7fec6..b9088264 100755 --- a/tests/integration/test_replay_attack.py +++ b/tests/integration/test_replay_attack.py @@ -38,8 +38,8 @@ import urllib import tempfile -from tuf.interposition import urllib_tuf -from tuf.tests import util_test_tools +import tuf.interposition.urllib_tuf as urllib_tuf +import tuf.tests.util_test_tools as util_test_tools class TestSetupError(Exception): diff --git a/tests/integration/test_slow_retrieval_attack.py b/tests/integration/test_slow_retrieval_attack.py index b80328ef..9d515be2 100755 --- a/tests/integration/test_slow_retrieval_attack.py +++ b/tests/integration/test_slow_retrieval_attack.py @@ -49,8 +49,8 @@ import urllib -from tuf.interposition import urllib_tuf -from tuf.tests import util_test_tools +import tuf.interposition.urllib_tuf as urllib_tuf +import tuf.tests.util_test_tools as util_test_tools class SlowRetrievalAttackAlert(Exception): diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index 07715b3c..75690717 100755 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -37,7 +37,7 @@ import tuf.conf as conf import tuf.download as download import tuf.log -import unittest_toolbox as unittest_toolbox +import tuf.tests.unittest_toolbox as unittest_toolbox logger = logging.getLogger('tuf.test_download') diff --git a/tests/unit/test_mirrors.py b/tests/unit/test_mirrors.py index f762a809..8aac13a4 100755 --- a/tests/unit/test_mirrors.py +++ b/tests/unit/test_mirrors.py @@ -21,7 +21,7 @@ import tuf import tuf.formats as formats import tuf.mirrors as mirrors -import unittest_toolbox +import tuf.tests.unittest_toolbox as unittest_toolbox diff --git a/tests/unit/test_quickstart.py b/tests/unit/test_quickstart.py index 42ef8771..fd7e5515 100755 --- a/tests/unit/test_quickstart.py +++ b/tests/unit/test_quickstart.py @@ -30,7 +30,7 @@ import tuf.log import tuf.repo.quickstart as quickstart import tuf.util -import unittest_toolbox +import tuf.tests.unittest_toolbox as unittest_toolbox logger = logging.getLogger('tuf.test_quickstart') unit_tbox = unittest_toolbox.Modified_TestCase diff --git a/tests/unit/test_signercli.py b/tests/unit/test_signercli.py index 378cc0f7..297eee18 100755 --- a/tests/unit/test_signercli.py +++ b/tests/unit/test_signercli.py @@ -52,7 +52,7 @@ class guarantees the order of unit tests. So that, 'test_something_A' import tuf.repo.signercli as signercli # Helper module unittest_toolbox.py -import unittest_toolbox as unittest_toolbox +import tuf.tests.unittest_toolbox as unittest_toolbox logger = logging.getLogger('tuf.test_signercli') diff --git a/tests/unit/test_signerlib.py b/tests/unit/test_signerlib.py index 5efdfa3c..a09f78fa 100755 --- a/tests/unit/test_signerlib.py +++ b/tests/unit/test_signerlib.py @@ -60,8 +60,8 @@ import tuf.formats as formats import tuf.repo.signerlib as signerlib import tuf.repo.keystore +import tuf.tests.unittest_toolbox as unittest_toolbox -import unittest_toolbox logger = logging.getLogger('tuf.test_signerlib') diff --git a/tests/unit/test_updater.py b/tests/unit/test_updater.py index 797b2e92..770e36e6 100755 --- a/tests/unit/test_updater.py +++ b/tests/unit/test_updater.py @@ -53,8 +53,8 @@ class guarantees the order of unit tests. So that, 'test_something_A' import tuf.repo.keystore as keystore import tuf.repo.signerlib as signerlib import tuf.roledb -import repository_setup as setup -import unittest_toolbox as unittest_toolbox +import tuf.tests.repository_setup as setup +import tuf.tests.unittest_toolbox as unittest_toolbox import tuf.util logger = logging.getLogger('tuf.test_updater') diff --git a/tests/unit/test_util.py b/tests/unit/test_util.py index c568eea7..fd203c50 100755 --- a/tests/unit/test_util.py +++ b/tests/unit/test_util.py @@ -30,7 +30,7 @@ import tuf.log import tuf.hash import tuf.util as util -import unittest_toolbox as unittest_toolbox +import tuf.tests.unittest_toolbox as unittest_toolbox logger = logging.getLogger('tuf.test_util') diff --git a/tests/unit/test_util_test_tools.py b/tests/unit/test_util_test_tools.py index f4ae5871..1ecfa219 100755 --- a/tests/unit/test_util_test_tools.py +++ b/tests/unit/test_util_test_tools.py @@ -21,7 +21,7 @@ import urllib import unittest -from tuf.tests import util_test_tools +import tuf.tests.util_test_tools as util_test_tools diff --git a/tuf/tests/__init__.py b/tuf/tests/__init__.py index 3235da9f..ac4c493e 100644 --- a/tuf/tests/__init__.py +++ b/tuf/tests/__init__.py @@ -1,4 +1,4 @@ # The tuf.tests package is intended to export as little utility as possible to # enable easier testing of TUF. -__all__ = ['util_test_tools'] +__all__ = ['repository_setup', 'unittest_toolbox', 'util_test_tools'] diff --git a/tests/unit/repository_setup.py b/tuf/tests/repository_setup.py similarity index 99% rename from tests/unit/repository_setup.py rename to tuf/tests/repository_setup.py index b79e5b57..066ca61e 100755 --- a/tests/unit/repository_setup.py +++ b/tuf/tests/repository_setup.py @@ -28,7 +28,7 @@ import tuf.repo.keystore as keystore import tuf.repo.signerlib as signerlib import tuf.repo.signercli as signercli -import unittest_toolbox as unittest_toolbox +import tuf.tests.unittest_toolbox as unittest_toolbox diff --git a/tests/unit/unittest_toolbox.py b/tuf/tests/unittest_toolbox.py similarity index 100% rename from tests/unit/unittest_toolbox.py rename to tuf/tests/unittest_toolbox.py From 3a7638ececa0f394a6ae9d9a83bc8e1dba2ea2b3 Mon Sep 17 00:00:00 2001 From: vladdd Date: Wed, 11 Sep 2013 11:03:52 -0400 Subject: [PATCH 099/119] Continue self-review of PyCrypto changes to keystore.py --- tuf/repo/keystore.py | 109 +++++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 36 deletions(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index 68019de8..4bfa067a 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -30,6 +30,11 @@ have only one key for the entire keystore, and as a result, we're requiring that the password be set at the point where the key is added. + The .key files are encrypted with the AES-256-CTR-Mode symmetric key + algorithm. User passwords are strengthened with PBKDF2, currently set to + 100,000 passphrase iterations. The previous evpy implementation used 1,000 + iterations. + """ import os @@ -50,10 +55,16 @@ import Crypto.Cipher.AES # 'Crypto.Random' is a cryptographically strong version of Python's standard -# "random" module. Random bits of data are needed for salts and +# "random" module. Random bits of data is needed for salts and # initialization vectors suitable for the encryption algorithms used in # 'keystore.py'. import Crypto.Random + +# The mode of operation is presently set to CTR (CounTeR Mode) for symmetric +# block encryption (AES-256). PyCrypto provides a callable stateful block +# counter that can update successive blocks when needed. The initial random +# block (IV) can be set to begin the process of incrementing the 128-bit blocks +# and allowing the AES algorithm to perform cipher block operations on them. import Crypto.Util.Counter import tuf.rsa_key @@ -78,15 +89,20 @@ # to protect against dictionary attacks) is generated for PBKDF2. _SALT_SIZE = 16 -# Default PBKDF2 passphrase iterations. -_PBKDF2_ITERATIONS = 90510 +# Default PBKDF2 passphrase iterations. The current (2013) "good enough" number +# of passphrase iterations. We recommend that important keys, such as root, +# be kept offline. Are we going overboard with respect to our use case? +# http://security.stackexchange.com/questions/3959/recommended-of-iterations-when-using-pkbdf2-sha256 +_PBKDF2_ITERATIONS = 100000 -# A password is set for each key added to the keystore. -# The passwords dict has the form: {keyid: 'password', ...} +# A user password is read and a derived key generated. The derived key and +# salt returned by the key derivation function (PBKDF2) is saved in +# '_derived_keys', which has the form: +# {keyid: {'salt': ..., 'derived_key': ...}} _derived_keys = {} # The keystore database, which has the form: -# {keyid: key, ...}. +# {keyid: key, keyid2: key2, ...} _keystore = {} @@ -162,9 +178,9 @@ def add_rsakey(rsakey_dict, password, keyid=None): message = 'Keyid: '+repr(keyid)+' already exists.' raise tuf.KeyAlreadyExistsError(message) - # The _derived_keys dictionary does not store the user's password. A key + # The '_derived_keys' dictionary does not store the user's password. A key # derivation function is applied to 'password' prior to storing it in - # _derived_keys. + # _derived_key and may then be used as a symmetric key. salt, derived_key = _generate_derived_key(password) _derived_keys[keyid] = {'salt': salt, 'derived_key': derived_key} _keystore[keyid] = rsakey_dict @@ -178,7 +194,8 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): Populate the keystore database with the key files found in 'directory_name'. Use the user-supplied passwords in 'passwords' to - decrypt the key files. Each key file has a corresponding password. + decrypt the key files. Each '.key' file has a corresponding + password. directory_name: @@ -219,8 +236,8 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): # Raise 'tuf.FormatError' if the check fails. tuf.formats.PASSWORDS_SCHEMA.check_match(passwords) - # Keep a list of the keys loaded. - loaded_keys = [] + # Keep a list of the keyids loaded, which is returned to the caller. + loaded_keyids = [] logger.info('Loading private key(s) from '+repr(directory_name)) @@ -233,7 +250,7 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): full_filepath = os.path.join(directory_name, keyfilename) raw_contents = open(full_filepath, 'rb').read() except: - logger.warn('Could not find key '+repr(full_filepath)+'!') + logger.warn('Could not find key '+repr(full_filepath)+'.') else: # Try to decrypt the file using one of the passwords in 'passwords'. for password in passwords: @@ -271,7 +288,7 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): logger.info('Loaded key: '+rsa_key['keyid']) except tuf.KeyAlreadyExistsError, e: logger.info('Key already loaded: '+rsa_key['keyid']) - loaded_keys.append(rsa_key['keyid']) + loaded_keyids.append(rsa_key['keyid']) continue else: logger.warn(repr(full_filepath)+' contains an invalid key type.') @@ -282,7 +299,7 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): logger.info('Done.') - return loaded_keys + return loaded_keyids @@ -291,15 +308,16 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): def save_keystore_to_keyfiles(directory_name): """ - Save all the keys found in the keystore to separate files. The password - for each key is stored when it is added to the keystore. Use these - passwords to encrypt the key files and save them in encrypted form to - 'directory_name'. + Save all the keys found in the keystore to individual files. The derived + symmetric key and salt for each key is stored when it is added to the + keystore. Use the symmetric key to encrypt the key files and save the + the ciphertext to 'directory_name' (Note: salt, IV, etc. is also appended + to the generated '.key'). directory_name: The name of the directory containing the key files ('.key'), - conformant to tuf.formats.RELPATH_SCHEMA. + conformant to 'tuf.formats.RELPATH_SCHEMA'. tuf.FormatError if 'directory_name is incorrectly formatted. @@ -325,7 +343,7 @@ def save_keystore_to_keyfiles(directory_name): logger.info('...no such directory. The directory will be created.') os.mkdir(directory_name) - # Iterate through the keystore keys and save them individually to a file. + # Iterate the keystore keys and save them individually to a file. for keyid, key in _keystore.items(): basefilename = os.path.join(directory_name, str(keyid)+'.key') file_object = open(basefilename, 'w') @@ -339,7 +357,8 @@ def save_keystore_to_keyfiles(directory_name): continue # Encrypt 'key_metadata_format' and save it. - encrypted_key = _encrypt(json.dumps(key_metadata_format), _derived_keys[keyid]) + encrypted_key = _encrypt(json.dumps(key_metadata_format), + _derived_keys[keyid]) file_object.write(encrypted_key) file_object.close() logger.info(repr(basefilename)+' saved.') @@ -353,7 +372,8 @@ def save_keystore_to_keyfiles(directory_name): def clear_keystore(): """ - Clear the keystore and key passwords. + Clear '_keystore', containing the all key data, and '_derived_keys', + containing the salt and symmetric keys. None. @@ -379,7 +399,9 @@ def clear_keystore(): def change_password(keyid, old_password, new_password): """ - Change the password for 'keyid'. + Change the password for 'keyid'. 'old_password' is verified prior to + any changes. Since user passwords are not stored, the derived key + information generated from these passwords is what's verified and updated. keyid: @@ -396,10 +418,11 @@ def change_password(keyid, old_password, new_password): keystore. tuf.BadPasswordError, if 'old_password' is invalid or - 'new_password' does not have to correct format. + 'new_password' does not have the correct format. - The old password for 'keyid' is changed to 'new_password'. + The old key information generated from the user password for 'keyid' + is changed for the new key information from 'new_password'. None. @@ -420,8 +443,7 @@ def change_password(keyid, old_password, new_password): message = repr(keyid)+' not recognized.' raise tuf.UnknownKeyError(message) - - # Check if the old password matches. The _derived_keys dictionary + # Check if the old password is valid. The _derived_keys dictionary # stores derived keys instead of user passwords, according to the # key derivation function used by _generate_derived_key(). salt = _derived_keys[keyid]['salt'] @@ -484,7 +506,8 @@ def _generate_derived_key(password, salt=None): """ Generate a derived key by feeding 'password' to the Password-Based Key Derivation Function (PBKDF2). PyCrypto's PBKDF2 implementation is - currently used. + currently used. 'salt' may be specified so that a previous derived key + may be regenerated. """ @@ -494,8 +517,9 @@ def _generate_derived_key(password, salt=None): def pseudorandom_function(password, salt): """ - PyCrypto's PBKDF2() expects a callable function for the optional - 'prf' argument. 'prf' is set to HMAC SHA1 by default. + PyCrypto's PBKDF2() expects a callable function for its optional + 'prf' argument. 'prf' is set to HMAC-SHA1 (in PyCrypto's PBKDF2 function) + by default. 'pseudorandom_function' instead sets 'prf' to HMAC-SHA256. """ return Crypto.Hash.HMAC.new(password, salt, Crypto.Hash.SHA256).digest() @@ -518,8 +542,8 @@ def pseudorandom_function(password, salt): def _encrypt(key_data, derived_key_information): """ Encrypt 'key_data' using the Advanced Encryption Standard (AES-256) algorithm. - 'derived_key_information' should have been strengthened by PBKDF2. The key - size is 256 bits and AES's mode of operation is set to CTR (Counter Mode). + 'derived_key_information' should contain a key strengthened by PBKDF2. The + key size is 256 bits and AES's mode of operation is set to CTR (CounTeR Mode). The HMAC of the ciphertext is generated to ensure the ciphertext has not been modified. @@ -540,7 +564,13 @@ def _encrypt(key_data, derived_key_information): # Generate a random initialization vector (IV). The 'iv' is treated as the # initial counter block to a stateful counter block function (i.e., # PyCrypto's 'Crypto.Util.Counter'. The AES block cipher operates on 128-bit - # blocks, so generate a random 16-byte initialization block. + # blocks, so generate a random 16-byte initialization block. PyCrypto expects + # the initial value of the stateful counter to be an integer. + # Follow the provably secure encrypt-then-MAC approach, which affords the + # ability to verify ciphertext without needing to decrypt it and preventing + # an attacker from feeding the block cipher malicious data. Modes like GCM + # provide both encryption and authentication, whereas CTR only provides + # encryption. iv = Crypto.Random.new().read(16) stateful_counter_128bit_blocks = Crypto.Util.Counter.new(128, initial_value=long(iv.encode('hex'), 16)) @@ -549,7 +579,8 @@ def _encrypt(key_data, derived_key_information): Crypto.Cipher.AES.MODE_CTR, counter=stateful_counter_128bit_blocks) - # Use AES-256 to encrypt 'key_data'. + # Use AES-256 to encrypt 'key_data'. The key size determines how many cycle + # repetitions are performed by AES, 14 cycles for 256-bit keys. try: ciphertext = aes_cipher.encrypt(key_data) except: @@ -557,6 +588,8 @@ def _encrypt(key_data, derived_key_information): raise tuf.CryptoError(message) # Generate the hmac of the ciphertext to ensure it has not been modified. + # The decryption routine may verify a ciphertext without having to perform + # a decryption operation. salt = derived_key_information['salt'] derived_key = derived_key_information['derived_key'] hmac_object = Crypto.Hash.HMAC.new(derived_key, ciphertext, Crypto.Hash.SHA256) @@ -588,15 +621,19 @@ def _decrypt(file_contents, password): # This delimiter is arbitrarily chosen and should not occur in the # hexadecimal representations of the fields it is separating. salt, hmac, iv, ciphertext = file_contents.split(_ENCRYPTION_DELIMITER) + + # Ensure we have the expected raw data for the delimited cryptographic data. salt = binascii.unhexlify(salt) hmac = binascii.unhexlify(hmac) iv = binascii.unhexlify(iv) ciphertext = binascii.unhexlify(ciphertext) - # Generate derived key from 'password'. + # Generate derived key from 'password'. The salt is specified so that + # the expected derived key is regenerated correctly. junk, derived_key = _generate_derived_key(password, salt) - # Verify the hmac to ensure the ciphertext is valid or has not been altered. + # Verify the hmac to ensure the ciphertext is valid and has not been altered. + # See the encryption routine for why we use the encrypt-then-MAC approach. generated_hmac_object = Crypto.Hash.HMAC.new(derived_key, ciphertext, Crypto.Hash.SHA256) generated_hmac = generated_hmac_object.hexdigest() From 7e98cb8269e4202b819e87078bf02f5ad214aa5c Mon Sep 17 00:00:00 2001 From: vladdd Date: Wed, 11 Sep 2013 12:36:44 -0400 Subject: [PATCH 100/119] Fix unit test failure in test_roledb.py Ensure roledb is empty prior to running the test cases. roledb.py may have been modified by other unit tests that did not properly clean up. --- tests/unit/test_roledb.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_roledb.py b/tests/unit/test_roledb.py index 6d068054..baecab91 100755 --- a/tests/unit/test_roledb.py +++ b/tests/unit/test_roledb.py @@ -38,7 +38,7 @@ class TestRoledb(unittest.TestCase): def setUp(self): - pass + pass @@ -403,6 +403,21 @@ def _test_rolename(self, test_function): +def setUpModule(): + # setUpModule is called before any test cases run. + # Ensure the roledb has not been modified by a previous test, which may + # affect assumptions (i.e., empty roledb) made by the tests cases in this + # unit test. + tuf.roledb.clear_roledb() + +def tearDownModule(): + # tearDownModule is called after all the tests have run. + # Ensure we clean up roledb. Courtesy is contagious, and it begins with + # test_roledb.py. + tuf.roledb.clear_roledb() + + + # Run the unit tests. if __name__ == '__main__': unittest.main() From a1132af5afe27483f6ff3a0df2aa11712c4f885b Mon Sep 17 00:00:00 2001 From: vladdd Date: Wed, 11 Sep 2013 13:33:28 -0400 Subject: [PATCH 101/119] Fix test_keystore.py unit test failure Ensure the keystore is empty prior to running the test cases. The courtesy bug is spreading. --- tests/unit/test_keystore.py | 13 +++++++++++++ tests/unit/test_roledb.py | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_keystore.py b/tests/unit/test_keystore.py index d00b812b..8d49c182 100755 --- a/tests/unit/test_keystore.py +++ b/tests/unit/test_keystore.py @@ -330,6 +330,19 @@ def test_internal_decrypt(self): +def setUpModule(): + # setUpModule() is called before any test cases run. + # Ensure the keystore has not been modified by a previous test, which may + # affect assumptions (i.e., empty keystore) made by the tests cases in this + # unit test. + tuf.repo.keystore.clear_keystore() + +def tearDownModule(): + # tearDownModule() is called after all the tests have run. + # Ensure we clean up the keystore. They say courtesy is contagious. + tuf.repo.keystore.clear_keystore() + + # Run the unit tests. if __name__ == '__main__': unittest.main() diff --git a/tests/unit/test_roledb.py b/tests/unit/test_roledb.py index baecab91..0b0259b5 100755 --- a/tests/unit/test_roledb.py +++ b/tests/unit/test_roledb.py @@ -404,14 +404,14 @@ def _test_rolename(self, test_function): def setUpModule(): - # setUpModule is called before any test cases run. + # setUpModule() is called before any test cases run. # Ensure the roledb has not been modified by a previous test, which may # affect assumptions (i.e., empty roledb) made by the tests cases in this # unit test. tuf.roledb.clear_roledb() def tearDownModule(): - # tearDownModule is called after all the tests have run. + # tearDownModule() is called after all the tests have run. # Ensure we clean up roledb. Courtesy is contagious, and it begins with # test_roledb.py. tuf.roledb.clear_roledb() From f026a998a9447eab2873d6fe741941d8f9dbc81a Mon Sep 17 00:00:00 2001 From: dachshund Date: Wed, 11 Sep 2013 17:46:29 -0400 Subject: [PATCH 102/119] Fix #102. --- tuf/__init__.py | 9 ++- tuf/client/updater.py | 167 +++++++++++++++++++++++++----------------- tuf/util.py | 16 ++-- 3 files changed, 116 insertions(+), 76 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index f44657e6..503e4fcf 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -176,7 +176,14 @@ class UnsupportedLibraryError(Error): class DecompressionError(Error): """Indicate that some error happened while decompressing a file.""" - pass + + def __init__(self, exception): + # Store the original exception. + self.exception = exception + + def __str__(self): + # Show the original exception. + return str(self.exception) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index e2e90207..1d924f8c 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -636,20 +636,22 @@ def __check_hashes(self, file_object, trusted_hashes): - def __hard_check_length(self, file_object, trusted_length): + def __hard_check_compressed_file_length(self, file_object, + compressed_file_length): """ - A helper function that checks the expected length of a file-like object. - The length of the file must be strictly equal to the expected length. - This is a deliberately redundant implementation designed to complement - tuf.download._check_downloaded_length(). + A helper function that checks the expected compressed length of a + file-like object. The length of the file must be strictly equal to the + expected length. This is a deliberately redundant implementation designed + to complement tuf.download._check_downloaded_length(). file_object: A file-like object. - trusted_length: - A nonnegative integer that is the expected length of the file. + compressed_file_length: + A nonnegative integer that is the expected compressed length of the + file. tuf.DownloadLengthMismatchError, if the lengths don't match. @@ -662,28 +664,34 @@ def __hard_check_length(self, file_object, trusted_length): """ - observed_length = len(file_object) - if observed_length != trusted_length: - raise tuf.DownloadLengthMismatchError(trusted_length, observed_length) + observed_length = file_object.get_compressed_length() + if observed_length != compressed_file_length: + raise tuf.DownloadLengthMismatchError(compressed_file_length, + observed_length) + else: + logger.debug('file length ('+str(observed_length)+\ + ') == trusted length ('+str(compressed_file_length)+')') - def __soft_check_length(self, file_object, trusted_length): + def __soft_check_compressed_file_length(self, file_object, + compressed_file_length): """ - A helper function that checks the expected length of a file-like object. - The length of the file must be less than or equal to the expected length. - This is a deliberately redundant implementation designed to complement - tuf.download._check_downloaded_length(). + A helper function that checks the expected compressed length of a + file-like object. The length of the file must be less than or equal to + the expected length. This is a deliberately redundant implementation + designed to complement tuf.download._check_downloaded_length(). file_object: A file-like object. - trusted_length: - A nonnegative integer that is the expected length of the file. + compressed_file_length: + A nonnegative integer that is the expected compressed length of the + file. tuf.DownloadLengthMismatchError, if the lengths don't match. @@ -696,15 +704,20 @@ def __soft_check_length(self, file_object, trusted_length): """ - observed_length = len(file_object) - if observed_length > trusted_length: - raise tuf.DownloadLengthMismatchError(trusted_length, observed_length) + observed_length = file_object.get_compressed_length() + if observed_length > compressed_file_length: + raise tuf.DownloadLengthMismatchError(compressed_file_length, + observed_length) + else: + logger.debug('file length ('+str(observed_length)+\ + ') <= trusted length ('+str(compressed_file_length)+')') - def get_target_file(self, target_filepath, file_length, file_hashes): + def get_target_file(self, target_filepath, compressed_file_length, + uncompressed_file_hashes): """ Safely download a target file up to a certain length, and check its @@ -714,10 +727,11 @@ def get_target_file(self, target_filepath, file_length, file_hashes): target_filepath: The relative target filepath obtained from TUF targets metadata. - file_length: - The expected length of the target file. + compressed_file_length: + The expected compressed length of the target file. If the file is not + compressed, then it will simply be its uncompressed length. - file_hashes: + uncompressed_file_hashes: The expected hashes of the target file. @@ -735,24 +749,25 @@ def get_target_file(self, target_filepath, file_length, file_hashes): """ - def verify_decompressed_target_file(target_file_object): + def verify_uncompressed_target_file(target_file_object): # Every target file must have its length and hashes inspected. - self.__hard_check_length(target_file_object, file_length) - self.__check_hashes(target_file_object, file_hashes) + self.__hard_check_compressed_file_length(target_file_object, + compressed_file_length) + self.__check_hashes(target_file_object, uncompressed_file_hashes) - return self.__get_file(target_filepath, verify_decompressed_target_file, - 'target', file_length, download_safely=True, - compression=None) + return self.__get_file(target_filepath, verify_uncompressed_target_file, + 'target', compressed_file_length, + download_safely=True, compression=None) - def __verify_decompressed_metadata_file(self, metadata_file_object, + def __verify_uncompressed_metadata_file(self, metadata_file_object, metadata_role): """ - A private helper function to verify a decompressed downloaded metadata + A private helper function to verify an uncompressed downloaded metadata file. @@ -827,7 +842,7 @@ def __verify_decompressed_metadata_file(self, metadata_file_object, def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, - file_length): + compressed_file_length): """ Unsafely download a metadata file up to a certain length. The actual file @@ -841,8 +856,9 @@ def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, metadata_filepath: The relative metadata filepath. - file_length: - The expected length of the metadata file. + compressed_file_length: + The expected compressed length of the metadata file. If the file is not + compressed, then it will simply be its uncompressed length. tuf.NoWorkingMirrorError: @@ -859,14 +875,15 @@ def unsafely_get_metadata_file(self, metadata_role, metadata_filepath, """ - def unsafely_verify_decompressed_metadata_file(metadata_file_object): - self.__soft_check_length(metadata_file_object, file_length) - self.__verify_decompressed_metadata_file(metadata_file_object, + def unsafely_verify_uncompressed_metadata_file(metadata_file_object): + self.__soft_check_compressed_file_length(metadata_file_object, + compressed_file_length) + self.__verify_uncompressed_metadata_file(metadata_file_object, metadata_role) return self.__get_file(metadata_filepath, - unsafely_verify_decompressed_metadata_file, 'meta', - file_length, download_safely=False, + unsafely_verify_uncompressed_metadata_file, 'meta', + compressed_file_length, download_safely=False, compression=None) @@ -874,7 +891,8 @@ def unsafely_verify_decompressed_metadata_file(metadata_file_object): def safely_get_metadata_file(self, metadata_role, metadata_filepath, - file_length, file_hashes, compression): + compressed_file_length, + uncompressed_file_hashes, compression): """ Safely download a metadata file up to a certain length, and check its @@ -887,10 +905,11 @@ def safely_get_metadata_file(self, metadata_role, metadata_filepath, metadata_filepath: The relative metadata filepath. - file_length: - The expected length of the metadata file. + compressed_file_length: + The expected compressed length of the metadata file. If the file is not + compressed, then it will simply be its uncompressed length. - file_hashes: + uncompressed_file_hashes: The expected hashes of the metadata file. compression: @@ -911,15 +930,16 @@ def safely_get_metadata_file(self, metadata_role, metadata_filepath, """ - def safely_verify_decompressed_metadata_file(metadata_file_object): - self.__hard_check_length(metadata_file_object, file_length) - self.__check_hashes(metadata_file_object, file_hashes) - self.__verify_decompressed_metadata_file(metadata_file_object, + def safely_verify_uncompressed_metadata_file(metadata_file_object): + self.__hard_check_compressed_file_length(metadata_file_object, + compressed_file_length) + self.__check_hashes(metadata_file_object, uncompressed_file_hashes) + self.__verify_uncompressed_metadata_file(metadata_file_object, metadata_role) return self.__get_file(metadata_filepath, - safely_verify_decompressed_metadata_file, 'meta', - file_length, download_safely=True, + safely_verify_uncompressed_metadata_file, 'meta', + compressed_file_length, download_safely=True, compression=compression) @@ -929,8 +949,8 @@ def safely_verify_decompressed_metadata_file(metadata_file_object): # TODO: Instead of the more fragile 'download_safely' switch, unroll the # function into two separate ones: one for "safe" download, and the other one # for "unsafe" download? This should induce safer and more readable code. - def __get_file(self, filepath, verify_decompressed_file, file_type, - file_length, download_safely, compression): + def __get_file(self, filepath, verify_uncompressed_file, file_type, + compressed_file_length, download_safely, compression): """ Try downloading, up to a certain length, a metadata or target file from a @@ -941,9 +961,9 @@ def __get_file(self, filepath, verify_decompressed_file, file_type, filepath: The relative metadata or target filepath. - verify_decompressed_file: - A function which expects a decompressed file-like object and which will - raise an exception in case the file is not valid for any reason. + verify_uncompressed_file: + A function which expects an uncompressed file-like object and which + will raise an exception in case the file is not valid for any reason. file_type: Type of data needed for download, must correspond to one of the strings @@ -951,8 +971,9 @@ def __get_file(self, filepath, verify_decompressed_file, file_type, 'target' for target file type. It should correspond to NAME_SCHEMA format. - file_length: - The expected length of the metadata or target file. + compressed_file_length: + The expected compressed length of the target or metadata file. If the + file is not compressed, then it will simply be its uncompressed length. download_safely: A boolean switch to toggle safe or unsafe download of the file. @@ -984,9 +1005,11 @@ def __get_file(self, filepath, verify_decompressed_file, file_type, for file_mirror in file_mirrors: try: if download_safely: - file_object = tuf.download.safe_download(file_mirror, file_length) + file_object = tuf.download.safe_download(file_mirror, + compressed_file_length) else: - file_object = tuf.download.unsafe_download(file_mirror, file_length) + file_object = tuf.download.unsafe_download(file_mirror, + compressed_file_length) if compression: logger.debug('Decompressing '+str(file_mirror)) @@ -994,7 +1017,7 @@ def __get_file(self, filepath, verify_decompressed_file, file_type, else: logger.debug('Not decompressing '+str(file_mirror)) - verify_decompressed_file(file_object) + verify_uncompressed_file(file_object) except Exception, exception: # Remember the error from this mirror, and "reset" the target file. @@ -1033,6 +1056,9 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): A dictionary containing length and hashes of the metadata file. Ex: {"hashes": {"sha256": "3a5a6ec1f353...dedce36e0"}, "length": 1340} + The length must be that of the compressed metadata file if it is + compressed, or uncompressed metadata file if it is uncompressed. + The hashes must be that of the uncompressed metadata file. STRICT_REQUIRED_LENGTH: A Boolean indicator used to signal whether we should perform strict @@ -1074,8 +1100,8 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): # Extract file length and file hashes. They will be passed as arguments # to 'download_file' function. - file_length = fileinfo['length'] - file_hashes = fileinfo['hashes'] + compressed_file_length = fileinfo['length'] + uncompressed_file_hashes = fileinfo['hashes'] # Attempt a file download from each mirror until the file is downloaded and # verified. If the signature of the downloaded file is valid, proceed, @@ -1100,11 +1126,12 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): if metadata_role == 'timestamp': metadata_file_object = \ self.unsafely_get_metadata_file(metadata_role, metadata_filename, - file_length) + compressed_file_length) else: metadata_file_object = \ self.safely_get_metadata_file(metadata_role, metadata_filename, - file_length, file_hashes, + compressed_file_length, + uncompressed_file_hashes, compression=compression) # The metadata has been verified. Move the metadata file into place. @@ -1128,9 +1155,11 @@ def _update_metadata(self, metadata_role, fileinfo, compression=None): # 'metadata_file_object' is an instance of tuf.util.TempFile. metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) if compression == 'gzip': - current_uncompressed_filepath = os.path.join(self.metadata_directory['current'], - uncompressed_metadata_filename) - current_uncompressed_filepath = os.path.abspath(current_uncompressed_filepath) + current_uncompressed_filepath = \ + os.path.join(self.metadata_directory['current'], + uncompressed_metadata_filename) + current_uncompressed_filepath = \ + os.path.abspath(current_uncompressed_filepath) metadata_file_object.move(current_uncompressed_filepath) else: metadata_file_object.move(current_filepath) @@ -1251,7 +1280,7 @@ def _update_metadata_if_changed(self, metadata_role, referenced_metadata='releas compressed_fileinfo = self.metadata['current'][referenced_metadata] \ ['meta'][gzip_metadata_filename] # NOTE: When we download the compressed file, we care about its - # compressed length. However, we check the hash of the decompressed + # compressed length. However, we check the hash of the uncompressed # file; therefore we use the hashes of the uncompressed file. fileinfo = {'length': compressed_fileinfo['length'], 'hashes': uncompressed_fileinfo['hashes']} @@ -2236,7 +2265,7 @@ def _visit_child_role(self, child_role, target_filepath): Ensure that we explore only delegated roles trusted with the target. We assume conservation of delegated paths in the complete tree of delegations. Note that the call to _ensure_all_targets_allowed in - __verify_decompressed_metadata_file should already ensure that all + __verify_uncompressed_metadata_file should already ensure that all targets metadata is valid; i.e. that the targets signed by a delegatee is a proper subset of the targets delegated to it by the delegator. Nevertheless, we check it again here for performance and safety reasons. diff --git a/tuf/util.py b/tuf/util.py index 2c2f01d4..1460e838 100755 --- a/tuf/util.py +++ b/tuf/util.py @@ -93,10 +93,11 @@ def __init__(self, prefix='tuf_temp_'): - def __len__(self): + def get_compressed_length(self): """ - Initializes TempFile. + Get the compressed length of the file. This will be correct information + even when the file is read as an uncompressed one. None. @@ -105,10 +106,12 @@ def __len__(self): OSError. - Nonnegative integer representing file size. + Nonnegative integer representing compressed file size. """ + # Even if we read a compressed file with the gzip standard library module, + # the original file will remain compressed. return os.stat(self.temporary_file.name).st_size @@ -295,9 +298,10 @@ def decompress_temp_file_object(self, compression): self._orig_file = self.temporary_file try: - self.temporary_file = gzip.GzipFile(fileobj=self.temporary_file, mode='rb') - except: - raise tuf.DecompressionError(self.temporary_file) + self.temporary_file = gzip.GzipFile(fileobj=self.temporary_file, + mode='rb') + except Exception, exception: + raise tuf.DecompressionError(exception) From 3111c22c634a637e2a2cd423b6db5e7d05bfe70d Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 12 Sep 2013 08:14:18 -0400 Subject: [PATCH 103/119] Fix test_util_test_tools.py test case failure Clear the keystore before initializing the util_test_tools.py repository and again after every test case exits. --- tests/unit/test_util_test_tools.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_util_test_tools.py b/tests/unit/test_util_test_tools.py index 1ecfa219..825f4aab 100755 --- a/tests/unit/test_util_test_tools.py +++ b/tests/unit/test_util_test_tools.py @@ -22,13 +22,17 @@ import unittest import tuf.tests.util_test_tools as util_test_tools - +import tuf.repo.keystore class test_UtilTestTools(unittest.TestCase): def setUp(self): unittest.TestCase.setUp(self) + # Ensure the keystore is empty prior to initializing the repository + # generated by 'util_test_tools'. + tuf.repo.keystore.clear_keystore() + # Unpacking necessary parameters returned from init_repo() essential_params = util_test_tools.init_repo(tuf=True) self.root_repo = essential_params[0] @@ -42,11 +46,16 @@ def setUp(self): def tearDown(self): unittest.TestCase.tearDown(self) + + # 'util_test_tools.cleanup()' should clear the keystore... util_test_tools.cleanup(self.root_repo, self.server_proc) + # Clear the keystore here just in case. + tuf.repo.keystore.clear_keystore() + #================================================# -# Below are few quick tests to make sure that # +# Below are a few quick tests that make sure # # everything works smoothly in util_test_tools. # #================================================# From 8c7bee515ad3d28b982cfc4e2ca92997df213636 Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 12 Sep 2013 12:50:11 -0400 Subject: [PATCH 104/119] Relocate the default PBKDF2 iterations to tuf.conf --- tuf/conf.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tuf/conf.py b/tuf/conf.py index de9ad7f7..249ab870 100755 --- a/tuf/conf.py +++ b/tuf/conf.py @@ -54,4 +54,14 @@ # The time (in seconds) we ignore a server with a slow initial retrieval speed. SLOW_START_GRACE_PERIOD = 30 #seconds - +# The current "good enough" number of PBKDF2 passphrase iterations. +# We recommend that important keys, such as root, be kept offline. +# 'tuf.conf.PBKDF2_ITERATIONS' should increase as CPU speeds increase, set here +# at 100,000 iterations by default (in 2013). The repository maintainer may opt +# to modify the default setting according to their security needs and +# computational restrictions. A strong user password is still important. +# Modifying the number of iterations will result in a new derived key+PBDKF2 +# combination if the key is loaded and re-saved, overriding any previous +# iteration setting used by the old '.key'. +# https://en.wikipedia.org/wiki/PBKDF2 +PBKDF2_ITERATIONS = 100000 From ffd4f485ed64b4744f3ea280e16ae063406ca785 Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 12 Sep 2013 12:58:56 -0400 Subject: [PATCH 105/119] Store in .key files the number of PBKDF2 iterations used --- tuf/repo/keystore.py | 77 ++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index 4bfa067a..43bcb3a0 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -69,7 +69,7 @@ import tuf.rsa_key import tuf.util - +import tuf.conf # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.keystore') @@ -89,11 +89,17 @@ # to protect against dictionary attacks) is generated for PBKDF2. _SALT_SIZE = 16 -# Default PBKDF2 passphrase iterations. The current (2013) "good enough" number +# Default PBKDF2 passphrase iterations. The current "good enough" number # of passphrase iterations. We recommend that important keys, such as root, -# be kept offline. Are we going overboard with respect to our use case? -# http://security.stackexchange.com/questions/3959/recommended-of-iterations-when-using-pkbdf2-sha256 -_PBKDF2_ITERATIONS = 100000 +# be kept offline. 'tuf.conf.PBKDF2_ITERATIONS' should increase as CPU +# speeds increase, set here at 100,000 iterations by default (in 2013). The +# repository maintainer may opt to modify the default setting according to +# their security needs and computational restrictions. A strong user password +# is still important. Modifying the number of iterations will result in a new +# derived key+PBDKF2 combination if the key is loaded and re-saved, overriding +# any previous iteration setting used by the old '.key'. +# https://en.wikipedia.org/wiki/PBKDF2 +_PBKDF2_ITERATIONS = tuf.conf.PBKDF2_ITERATIONS # A user password is read and a derived key generated. The derived key and # salt returned by the key derivation function (PBKDF2) is saved in @@ -171,8 +177,8 @@ def add_rsakey(rsakey_dict, password, keyid=None): 'Expected: '+repr(keyid) raise tuf.Error(message) - # Check if the keyid belonging to 'rsakey_dict' is not already - # available in the key database. + # Check if the keyid belonging to 'rsakey_dict' is not already available in + # the key database. keyid = rsakey_dict['keyid'] if keyid in _keystore: message = 'Keyid: '+repr(keyid)+' already exists.' @@ -181,8 +187,10 @@ def add_rsakey(rsakey_dict, password, keyid=None): # The '_derived_keys' dictionary does not store the user's password. A key # derivation function is applied to 'password' prior to storing it in # _derived_key and may then be used as a symmetric key. - salt, derived_key = _generate_derived_key(password) - _derived_keys[keyid] = {'salt': salt, 'derived_key': derived_key} + salt, iterations, derived_key = _generate_derived_key(password) + _derived_keys[keyid] = {'salt': salt, + 'derived_key': derived_key, + 'iterations': iterations} _keystore[keyid] = rsakey_dict @@ -447,16 +455,19 @@ def change_password(keyid, old_password, new_password): # stores derived keys instead of user passwords, according to the # key derivation function used by _generate_derived_key(). salt = _derived_keys[keyid]['salt'] - junk, old_derived_key = _generate_derived_key(old_password, salt) + iterations = _derived_keys[keyid]['iterations'] + junk, junk, old_derived_key = _generate_derived_key(old_password, + salt, iterations) if _derived_keys[keyid]['derived_key'] != old_derived_key: message = 'Old password invalid.' raise tuf.BadPasswordError(message) # Update '_derived_keys[keyid]' with the new derived key and salt. - salt, new_derived_key = _generate_derived_key(new_password) + salt, iterations, new_derived_key = _generate_derived_key(new_password) _derived_keys[keyid] = {} _derived_keys[keyid]['salt'] = salt _derived_keys[keyid]['derived_key'] = new_derived_key + _derived_keys[keyid]['iterations'] = iterations @@ -502,7 +513,7 @@ def get_key(keyid): -def _generate_derived_key(password, salt=None): +def _generate_derived_key(password, salt=None, iterations=None): """ Generate a derived key by feeding 'password' to the Password-Based Key Derivation Function (PBKDF2). PyCrypto's PBKDF2 implementation is @@ -514,6 +525,8 @@ def _generate_derived_key(password, salt=None): if salt is None: salt = Crypto.Random.new().read(_SALT_SIZE) + if iterations is None: + iterations = _PBKDF2_ITERATIONS def pseudorandom_function(password, salt): """ @@ -530,10 +543,10 @@ def pseudorandom_function(password, salt): # must be callable. derived_key = Crypto.Protocol.KDF.PBKDF2(password, salt, dkLen=_AES_KEY_SIZE, - count=_PBKDF2_ITERATIONS, + count=iterations, prf=pseudorandom_function) - return salt, derived_key + return salt, iterations, derived_key @@ -554,8 +567,9 @@ def _encrypt(key_data, derived_key_information): 'private': '-----BEGIN RSA PRIVATE KEY----- ...'}} 'derived_key_information' is a dictionary of the form: - {'salt': '...' - 'derived_key': '...'} + {'salt': '...', + 'derived_key': '...', + 'iterations': '...'} 'tuf.CryptoError' raised if the encryption fails. @@ -591,15 +605,22 @@ def _encrypt(key_data, derived_key_information): # The decryption routine may verify a ciphertext without having to perform # a decryption operation. salt = derived_key_information['salt'] - derived_key = derived_key_information['derived_key'] - hmac_object = Crypto.Hash.HMAC.new(derived_key, ciphertext, Crypto.Hash.SHA256) + hmac_object = Crypto.Hash.HMAC.new(symmetric_key, ciphertext, + Crypto.Hash.SHA256) hmac = hmac_object.hexdigest() - # Return the hmac, initialization vector, and ciphertext as a single string. - # These three values are delimited by '_ENCRYPTION_DELIMITER' to make - # extraction easier. This delimiter is arbitrarily chosen and should not - # occur in the hexadecimal representations of the fields it is separating. + # Store the number of PBKDF2 iterations used to derive the symmetric key so + # that the decryption routine can regenerate the symmetric key successfully. + # The pbkdf2 iterations are allowed to vary for the keys loaded and saved. + iterations = derived_key_information['iterations'] + + # Return the salt, iterations, hmac, initialization vector, and ciphertext + # as a single string. These five values are delimited by + # '_ENCRYPTION_DELIMITER' to make extraction easier. This delimiter is + # arbitrarily chosen and should not occur in the hexadecimal representations + # of the fields it is separating. return binascii.hexlify(salt) + _ENCRYPTION_DELIMITER + \ + binascii.hexlify(str(iterations)) + _ENCRYPTION_DELIMITER + \ binascii.hexlify(hmac) + _ENCRYPTION_DELIMITER + \ binascii.hexlify(iv) + _ENCRYPTION_DELIMITER + \ binascii.hexlify(ciphertext) @@ -620,17 +641,19 @@ def _decrypt(file_contents, password): # 'file_contents'. These three values are delimited by '_ENCRYPTION_DELIMITER'. # This delimiter is arbitrarily chosen and should not occur in the # hexadecimal representations of the fields it is separating. - salt, hmac, iv, ciphertext = file_contents.split(_ENCRYPTION_DELIMITER) + salt, iterations, hmac, iv, ciphertext = \ + file_contents.split(_ENCRYPTION_DELIMITER) # Ensure we have the expected raw data for the delimited cryptographic data. - salt = binascii.unhexlify(salt) + salt = binascii.unhexlify(salt) + iterations = int(binascii.unhexlify(iterations)) hmac = binascii.unhexlify(hmac) iv = binascii.unhexlify(iv) ciphertext = binascii.unhexlify(ciphertext) - # Generate derived key from 'password'. The salt is specified so that - # the expected derived key is regenerated correctly. - junk, derived_key = _generate_derived_key(password, salt) + # Generate derived key from 'password'. The salt and iterations are specified + # so that the expected derived key is regenerated correctly. + junk, junk, derived_key = _generate_derived_key(password, salt, iterations) # Verify the hmac to ensure the ciphertext is valid and has not been altered. # See the encryption routine for why we use the encrypt-then-MAC approach. From b89d6fb853bdefc85351a90e0534b3f74edcac4e Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 12 Sep 2013 13:02:54 -0400 Subject: [PATCH 106/119] Update test_keystore.py following .key format and keystore changes --- tests/unit/test_keystore.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_keystore.py b/tests/unit/test_keystore.py index 8d49c182..b5b2c3f8 100755 --- a/tests/unit/test_keystore.py +++ b/tests/unit/test_keystore.py @@ -36,6 +36,8 @@ # internal function. json = tuf.util.import_json() +tuf.repo.keystore._PBKDF2_ITERATIONS = 1000 + # Creating a directory string in current directory. _CURRENT_DIR = os.getcwd() _DIR = os.path.join(_CURRENT_DIR, 'test_keystore') @@ -297,8 +299,10 @@ def test_get_key(self): def test_internal_encrypt(self): # Test for valid arguments to '_encrypt()' and a valid return type. salt = Crypto.Random.new().read(16) + iterations = tuf.repo.keystore._PBKDF2_ITERATIONS derived_key = Crypto.Protocol.KDF.PBKDF2(PASSWDS[0], salt) - derived_key_information = {'salt': salt, 'derived_key': derived_key} + derived_key_information = {'salt': salt, 'derived_key': derived_key, + 'iterations': iterations} encrypted_key = KEYSTORE._encrypt(json.dumps(RSAKEYS[0]), derived_key_information) self.assertEqual(type(encrypted_key), str) @@ -310,8 +314,11 @@ def test_internal_decrypt(self): tuf.formats.KEY_SCHEMA.check_match(RSAKEYS[0]) salt = Crypto.Random.new().read(16) - salt, derived_key = tuf.repo.keystore._generate_derived_key(PASSWDS[0], salt) - derived_key_information = {'salt': salt, 'derived_key': derived_key} + salt, iterations, derived_key = \ + tuf.repo.keystore._generate_derived_key(PASSWDS[0], salt) + derived_key_information = {'salt': salt, + 'iterations': iterations, + 'derived_key': derived_key} # Getting a valid encrypted key using '_encrypt()'. encrypted_key = KEYSTORE._encrypt(json.dumps(RSAKEYS[0]), @@ -343,6 +350,7 @@ def tearDownModule(): tuf.repo.keystore.clear_keystore() + # Run the unit tests. if __name__ == '__main__': unittest.main() From 1513065a04e6fcd24d086521f4e01279b2dc46cc Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 12 Sep 2013 13:04:07 -0400 Subject: [PATCH 107/119] Update unittest_toolbox.py following .key format and keystore changes --- tuf/tests/unittest_toolbox.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tuf/tests/unittest_toolbox.py b/tuf/tests/unittest_toolbox.py index 0ff48392..7aa2126d 100755 --- a/tuf/tests/unittest_toolbox.py +++ b/tuf/tests/unittest_toolbox.py @@ -30,6 +30,7 @@ import tuf.rsa_key as rsa_key import tuf.repo.keystore as keystore + # Modify the number of iterations (from the higher default count) so the unit # tests run faster. keystore._PBKDF2_ITERATIONS = 1000 @@ -390,9 +391,10 @@ def generate_rsakey(): Modified_TestCase.rsa_keyids.append(keyid) password = Modified_TestCase.random_string() Modified_TestCase.rsa_passwords[keyid] = password - salt, derived_key = keystore._generate_derived_key(password) + salt, iterations, derived_key = keystore._generate_derived_key(password) Modified_TestCase.rsa_derived_keys[keyid] = {'salt': salt, - 'derived_key': derived_key} + 'derived_key': derived_key, + 'iterations': iterations} Modified_TestCase.rsa_keystore[keyid] = rsakey return keyid From 372bcbda3f245f0e089f4a07641ad13ed4a49db8 Mon Sep 17 00:00:00 2001 From: vladdd Date: Thu, 12 Sep 2013 14:22:08 -0400 Subject: [PATCH 108/119] Minor code updates to keystore.py --- tuf/repo/keystore.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index 43bcb3a0..a5d8b8cf 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -257,15 +257,15 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords): keyfilename = keyid+'.key' full_filepath = os.path.join(directory_name, keyfilename) raw_contents = open(full_filepath, 'rb').read() - except: - logger.warn('Could not find key '+repr(full_filepath)+'.') + except (OSError, IOError), e: + logger.warn('Could not load key file: '+repr(full_filepath)+'.') else: # Try to decrypt the file using one of the passwords in 'passwords'. for password in passwords: try: json_data = _decrypt(raw_contents, password) - except: - logger.warn(repr(full_filepath)+' contains an invalid key.') + except tuf.CryptoError, e: + logger.warn(repr(full_filepath)+' could not be decrypted.') continue try: @@ -577,7 +577,7 @@ def _encrypt(key_data, derived_key_information): # Generate a random initialization vector (IV). The 'iv' is treated as the # initial counter block to a stateful counter block function (i.e., - # PyCrypto's 'Crypto.Util.Counter'. The AES block cipher operates on 128-bit + # PyCrypto's 'Crypto.Util.Counter'). The AES block cipher operates on 128-bit # blocks, so generate a random 16-byte initialization block. PyCrypto expects # the initial value of the stateful counter to be an integer. # Follow the provably secure encrypt-then-MAC approach, which affords the @@ -597,7 +597,10 @@ def _encrypt(key_data, derived_key_information): # repetitions are performed by AES, 14 cycles for 256-bit keys. try: ciphertext = aes_cipher.encrypt(key_data) - except: + + # Raise generic exception message to avoid revealing sensitive information, + # such as invalid passwords, encryption keys, etc. + except Exception, e: message = 'The key data could not be encrypted.' raise tuf.CryptoError(message) @@ -673,7 +676,10 @@ def _decrypt(file_contents, password): counter=stateful_counter_128bit_blocks) try: key_plaintext = aes_cipher.decrypt(ciphertext) - except: + + # Raise generic exception message to avoid revealing sensitive information, + # such as invalid passwords, encryption keys, etc. + except Exception, e: raise tuf.CryptoError('Decryption failed.') return key_plaintext From 42a8c086f32b60741253a211c69080b5de0d9d30 Mon Sep 17 00:00:00 2001 From: vladdd Date: Fri, 13 Sep 2013 08:18:40 -0400 Subject: [PATCH 109/119] Continue minor edits of PyCrypto changes Improve the comments of keystore.py and rsa_key.py. --- tuf/repo/keystore.py | 47 ++++++++++++++++++++++++++------------------ tuf/rsa_key.py | 5 +++-- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index a5d8b8cf..2feb1a33 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -61,10 +61,11 @@ import Crypto.Random # The mode of operation is presently set to CTR (CounTeR Mode) for symmetric -# block encryption (AES-256). PyCrypto provides a callable stateful block -# counter that can update successive blocks when needed. The initial random -# block (IV) can be set to begin the process of incrementing the 128-bit blocks -# and allowing the AES algorithm to perform cipher block operations on them. +# block encryption (AES-256, where the symmetric is 256 bits). PyCrypto +# provides a callable stateful block counter that can update successive blocks +# when needed. The initial random block, or initialization vector (IV), can +# be set to begin the process of incrementing the 128-bit blocks and allowing +# the AES algorithm to perform cipher block operations on them. import Crypto.Util.Counter import tuf.rsa_key @@ -77,7 +78,7 @@ json = tuf.util.import_json() # The delimiter symbol used to separate the different sections -# of encrypted files (i.e., salt, IV, ciphertext, passphrase). +# of encrypted files (i.e., salt, iterations, hmac, IV, ciphertext). # This delimiter is arbitrarily chosen and should not occur in # the hexadecimal representations of the fields it is separating. _ENCRYPTION_DELIMITER = '@@@@' @@ -86,14 +87,15 @@ _AES_KEY_SIZE = 32 # Default salt size, in bytes. A 128-bit salt (i.e., a random sequence of data -# to protect against dictionary attacks) is generated for PBKDF2. +# to protect against attacks that use precomputed rainbow tables to crack +# password hashes) is generated for PBKDF2. _SALT_SIZE = 16 # Default PBKDF2 passphrase iterations. The current "good enough" number # of passphrase iterations. We recommend that important keys, such as root, # be kept offline. 'tuf.conf.PBKDF2_ITERATIONS' should increase as CPU -# speeds increase, set here at 100,000 iterations by default (in 2013). The -# repository maintainer may opt to modify the default setting according to +# speeds increase, set here at 100,000 iterations by default (in 2013). +# Repository maintainers may opt to modify the default setting according to # their security needs and computational restrictions. A strong user password # is still important. Modifying the number of iterations will result in a new # derived key+PBDKF2 combination if the key is loaded and re-saved, overriding @@ -101,14 +103,19 @@ # https://en.wikipedia.org/wiki/PBKDF2 _PBKDF2_ITERATIONS = tuf.conf.PBKDF2_ITERATIONS -# A user password is read and a derived key generated. The derived key and -# salt returned by the key derivation function (PBKDF2) is saved in -# '_derived_keys', which has the form: -# {keyid: {'salt': ..., 'derived_key': ...}} +# A user password is read and a derived key generated. The derived key returned +# by the key derivation function (PBKDF2) is saved in '_derived_keys', along +# with the salt and iterations used, which has the form: +# {keyid: {'salt': '\x9b\x90\xf1g\xb9li\x04\x8d\x10h\xb5T\xaa\xc1', +# 'iterations': 10000, +# 'derived_key': '\xda\xed\xf2\xe0\x8f\x03\xeb\xde!\xc4RJ'}, +# keyid2: ...} _derived_keys = {} # The keystore database, which has the form: -# {keyid: key, keyid2: key2, ...} +# {keyid: key, +# keyid2: key2, +# ...} _keystore = {} @@ -599,7 +606,8 @@ def _encrypt(key_data, derived_key_information): ciphertext = aes_cipher.encrypt(key_data) # Raise generic exception message to avoid revealing sensitive information, - # such as invalid passwords, encryption keys, etc. + # such as invalid passwords, encryption keys, etc., that an attacker can use + # to his advantage. except Exception, e: message = 'The key data could not be encrypted.' raise tuf.CryptoError(message) @@ -640,10 +648,10 @@ def _decrypt(file_contents, password): """ - # Extract the salt, hmac, initialization vector, and ciphertext from - # 'file_contents'. These three values are delimited by '_ENCRYPTION_DELIMITER'. - # This delimiter is arbitrarily chosen and should not occur in the - # hexadecimal representations of the fields it is separating. + # Extract the salt, iterations, hmac, initialization vector, and ciphertext + # from 'file_contents'. These five values are delimited by + # '_ENCRYPTION_DELIMITER'. This delimiter is arbitrarily chosen and should + # not occur in the hexadecimal representations of the fields it is separating. salt, iterations, hmac, iv, ciphertext = \ file_contents.split(_ENCRYPTION_DELIMITER) @@ -678,7 +686,8 @@ def _decrypt(file_contents, password): key_plaintext = aes_cipher.decrypt(ciphertext) # Raise generic exception message to avoid revealing sensitive information, - # such as invalid passwords, encryption keys, etc. + # such as invalid passwords, encryption keys, etc., that an attacker can + # use to his advantage. except Exception, e: raise tuf.CryptoError('Decryption failed.') diff --git a/tuf/rsa_key.py b/tuf/rsa_key.py index b61b7d51..c5ce9d52 100755 --- a/tuf/rsa_key.py +++ b/tuf/rsa_key.py @@ -372,8 +372,9 @@ def create_signature(rsakey_dict, data): method = 'PyCrypto-PKCS#1 PSS' sig = None - if private_key: - # Take + if private_key is not '': + # Calculate the SHA256 hash of 'data' and generate the hash's PKCS1-PSS + # signature. try: rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key) sha256_object = Crypto.Hash.SHA256.new(data) From 312e6398d6ec468a85746ed7152808ba81475c72 Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 23 Sep 2013 14:04:13 -0400 Subject: [PATCH 110/119] Fix typo: missing the word 'key' --- tuf/repo/keystore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index 9db7d812..8b20937d 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -61,7 +61,7 @@ import Crypto.Random # The mode of operation is presently set to CTR (CounTeR Mode) for symmetric -# block encryption (AES-256, where the symmetric is 256 bits). PyCrypto +# block encryption (AES-256, where the symmetric key is 256 bits). PyCrypto # provides a callable stateful block counter that can update successive blocks # when needed. The initial random block, or initialization vector (IV), can # be set to begin the process of incrementing the 128-bit blocks and allowing From cbb76d1a8007aa889697cc17b2021963ef46dbe4 Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 23 Sep 2013 14:17:36 -0400 Subject: [PATCH 111/119] Fix ignored placeholder variable names 'junk' signifies that this variable is not needed and thus discarded, however, it hides what is returned. Include the junk and what is returned so the variable name is more descriptive. --- tuf/repo/keystore.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index 8b20937d..637c1b0a 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -464,8 +464,11 @@ def change_password(keyid, old_password, new_password): # key derivation function used by _generate_derived_key(). salt = _derived_keys[keyid]['salt'] iterations = _derived_keys[keyid]['iterations'] - junk, junk, old_derived_key = _generate_derived_key(old_password, - salt, iterations) + + # Discard the old "salt" and "iterations" values, as we only need the old + # derived key. + junk_old_salt, junk_old_iterations, old_derived_key = \ + _generate_derived_key(old_password, salt, iterations) if _derived_keys[keyid]['derived_key'] != old_derived_key: message = 'Old password invalid.' raise tuf.BadPasswordError(message) @@ -665,8 +668,10 @@ def _decrypt(file_contents, password): ciphertext = binascii.unhexlify(ciphertext) # Generate derived key from 'password'. The salt and iterations are specified - # so that the expected derived key is regenerated correctly. - junk, junk, derived_key = _generate_derived_key(password, salt, iterations) + # so that the expected derived key is regenerated correctly. Discard the old + # "salt" and "iterations" values, as we only need the old derived key. + junk_old_salt, junk_old_iterations, derived_key = \ + _generate_derived_key(password, salt, iterations) # Verify the hmac to ensure the ciphertext is valid and has not been altered. # See the encryption routine for why we use the encrypt-then-MAC approach. From 2163400892ee8a401d410b452c2084cbfed9e442 Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 23 Sep 2013 14:33:55 -0400 Subject: [PATCH 112/119] Fix how the private key is checked prior to verifying a signature --- tuf/rsa_key.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tuf/rsa_key.py b/tuf/rsa_key.py index 482710e2..8d737391 100755 --- a/tuf/rsa_key.py +++ b/tuf/rsa_key.py @@ -378,8 +378,12 @@ def create_signature(rsakey_dict, data): keyid = rsakey_dict['keyid'] method = 'PyCrypto-PKCS#1 PSS' sig = None - - if private_key is not '': + + # Verify the signature, but only if the private key has been set. The private + # key is a NULL string if unset. Although it may be clearer to explicit check + # that 'private_key' is not '', we can/should check for a value and not + # compare identities with the 'is' keyword. + if len(private_key): # Calculate the SHA256 hash of 'data' and generate the hash's PKCS1-PSS # signature. try: From c65a1aaf3828ae8a9d6d55ac80c1fdc55d1f9249 Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 23 Sep 2013 19:33:25 -0400 Subject: [PATCH 113/119] Fix link that discusses setting the time for all logging formatters --- tuf/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/log.py b/tuf/log.py index e1a44752..88a11135 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -73,7 +73,7 @@ '[%(funcName)s:%(lineno)s@%(filename)s] %(message)s' # Ask all Formatter instances to talk GMT. -# http://docs.python.org/2/library/logging.html#logging.Formatter.formatException +# http://docs.python.org/2/library/logging.html#logging.Formatter.formatTime logging.Formatter.converter = time.gmtime formatter = logging.Formatter(_FORMAT_STRING) From de6a5161c47b38d17753e3d1290e3e11b1511463 Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 23 Sep 2013 19:54:47 -0400 Subject: [PATCH 114/119] Add missing docstring to remove_console_handler() and minor edits --- tuf/log.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tuf/log.py b/tuf/log.py index 88a11135..d82f1078 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -143,6 +143,7 @@ def filter(self, record): # original exception traceback. The exc_info is explained here: # http://docs.python.org/2/library/sys.html#sys.exc_info exc_type, exc_value, exc_traceback = record.exc_info + # Simply set the class name as the exception text. record.exc_text = exc_type.__name__ @@ -241,6 +242,9 @@ def set_console_log_level(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.LENGTH_SCHEMA.check_match(log_level) + # Assign to the global console_handler object. + global console_handler + if console_handler is not None: console_handler.setLevel(log_level) else: @@ -272,14 +276,14 @@ def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): None. """ - - # Assign to the global console_handler object. - global console_handler # Does 'log_level' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. tuf.formats.LENGTH_SCHEMA.check_match(log_level) + # Assign to the global console_handler object. + global console_handler + if not console_handler: # Set the console handler for the logger. The built-in console handler will # log messages to 'sys.stderr' and capture 'log_level' messages. @@ -301,7 +305,26 @@ def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): def remove_console_handler(): - # Assign to the global console_handler object. + """ + + Remove the console handler from the logger in 'log.py', if previously added. + + + None. + + + None. + + + A handler belonging to the console is removed from the 'log.py' logger + and the console handler is marked as unset. + + + None. + + """ + + # Assign to the global 'console_handler' object. global console_handler if console_handler: From c3a358ed17723e5b8bd367c0fb352a33aa0dc62d Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 23 Sep 2013 21:35:14 -0400 Subject: [PATCH 115/119] Update comments about thread safety --- tuf/log.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tuf/log.py b/tuf/log.py index d82f1078..d5816685 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -48,6 +48,13 @@ logging.DEBUG 10 logging.NOTSET 0 + The logging module is thread-safe. Logging to a single file from + multiple threads in a single process is also thread-safe. The logging + module is NOT thread-safe when logging to a single file across multiple + processes: + http://docs.python.org/2/library/logging.html#thread-safety + http://docs.python.org/2/howto/logging-cookbook.html + """ @@ -287,7 +294,6 @@ def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): if not console_handler: # Set the console handler for the logger. The built-in console handler will # log messages to 'sys.stderr' and capture 'log_level' messages. - # NOTE: This is not thread-safe. console_handler = logging.StreamHandler() # Get our filter for the console handler. console_filter = ConsoleFilter() @@ -329,7 +335,6 @@ def remove_console_handler(): if console_handler: logger.removeHandler(console_handler) - # NOTE: This is not thread-safe. console_handler = None logger.debug('Removed a console handler.') else: From 4e6c46364dd5533bf75c4a126ca1457097322247 Mon Sep 17 00:00:00 2001 From: vladdd Date: Tue, 24 Sep 2013 09:36:55 -0400 Subject: [PATCH 116/119] Fix modified (accidently) variable name in keystore.py --- tuf/repo/keystore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/repo/keystore.py b/tuf/repo/keystore.py index 637c1b0a..8f5492e5 100755 --- a/tuf/repo/keystore.py +++ b/tuf/repo/keystore.py @@ -117,7 +117,7 @@ # {keyid: key, # keyid2: key2, # ...} -keystore = {} +_keystore = {} def add_rsakey(rsakey_dict, password, keyid=None): From e07bd128b9fb7548132121ce5c260faba1b5bb0a Mon Sep 17 00:00:00 2001 From: vladdd Date: Tue, 1 Oct 2013 15:28:38 -0400 Subject: [PATCH 117/119] Add LOGLEVEL_SCHEMA to tuf.formats log.py previously checked the format of log level arguments with LENGTH_SCHEMA. LOGLEVEL_SCHEMA might be clearer. --- tuf/formats.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tuf/formats.py b/tuf/formats.py index 7142469b..038bf4a0 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -121,6 +121,10 @@ # An integer representing length. Must be 0, or greater. LENGTH_SCHEMA = SCHEMA.Integer(lo=0) +# An integer representing logger levels, such as logging.CRITICAL (=50). +# Must be between 0 and 50. +LOGLEVEL_SCHEMA = SCHEMA.Integer(lo=0, hi=50) + # A string representing a named object. NAME_SCHEMA = SCHEMA.AnyString() From 213b96dfea6678edecb1a95b4ee9125bb935c55b Mon Sep 17 00:00:00 2001 From: vladdd Date: Tue, 1 Oct 2013 15:33:39 -0400 Subject: [PATCH 118/119] Update comments and schema type checked Expand the comments on 'logging.Formatter.converter' to discuss the converter attribute. Modify LENGTH_SCHEMA to LOGLEVEL_SCHEMA. --- tuf/log.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tuf/log.py b/tuf/log.py index d5816685..6ba9ece5 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -79,8 +79,14 @@ _FORMAT_STRING = '[%(asctime)s UTC] [%(name)s] [%(levelname)s]'+\ '[%(funcName)s:%(lineno)s@%(filename)s] %(message)s' -# Ask all Formatter instances to talk GMT. +# Ask all Formatter instances to talk GMT. Set the 'converter' attribute of +# 'logging.Formatter' so that all formatters use Greenwich Mean Time. # http://docs.python.org/2/library/logging.html#logging.Formatter.formatTime +# The 2nd paragraph in the link above contains the relevant information. +# GMT = UTC (Coordinated Universal Time). TUF metadata stores timestamps in UTC. +# We previously displayed the local time but this lead to confusion when +# visually comparing logger events and metadata information. Unix time stamps +# are fine but they may be less human-readable than UTC. logging.Formatter.converter = time.gmtime formatter = logging.Formatter(_FORMAT_STRING) @@ -184,7 +190,7 @@ def set_log_level(log_level=_DEFAULT_LOG_LEVEL): # Does 'log_level' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.LENGTH_SCHEMA.check_match(log_level) + tuf.formats.LOGLEVEL_SCHEMA.check_match(log_level) logger.setLevel(log_level) @@ -215,7 +221,7 @@ def set_filehandler_log_level(log_level=_DEFAULT_FILE_LOG_LEVEL): # Does 'log_level' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.LENGTH_SCHEMA.check_match(log_level) + tuf.formats.LOGLEVEL_SCHEMA.check_match(log_level) file_handler.setLevel(log_level) @@ -247,7 +253,7 @@ def set_console_log_level(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): # Does 'log_level' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.LENGTH_SCHEMA.check_match(log_level) + tuf.formats.LOGLEVEL_SCHEMA.check_match(log_level) # Assign to the global console_handler object. global console_handler @@ -286,7 +292,7 @@ def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): # Does 'log_level' have the correct format? # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.LENGTH_SCHEMA.check_match(log_level) + tuf.formats.LOGLEVEL_SCHEMA.check_match(log_level) # Assign to the global console_handler object. global console_handler From dc84e051e6970bbc1c767f594df7d1a3c08b703a Mon Sep 17 00:00:00 2001 From: vladdd Date: Mon, 14 Oct 2013 09:20:49 -0400 Subject: [PATCH 119/119] Update the valid expiration data --- tests/unit/test_quickstart.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_quickstart.py b/tests/unit/test_quickstart.py index fd7e5515..00ddf44e 100755 --- a/tests/unit/test_quickstart.py +++ b/tests/unit/test_quickstart.py @@ -72,7 +72,7 @@ def test_2_build_repository(self): proj_files = self.make_temp_directory_with_data_files() proj_dir = os.path.join(proj_files[0], 'targets') - input_dict = {'expiration':'12/12/2013', + input_dict = {'expiration':'12/12/2020', 'root':{'threshold':1, 'password':'pass'}, 'targets':{'threshold':1, 'password':'pass'}, 'release':{'threshold':1, 'password':'pass'}, @@ -128,7 +128,7 @@ def _remove_repository_directories(repo_dir, keystore_dir, client_dir): _remove_repository_directories(repo_dir, keystore_dir, client_dir) # Restore expiration. - input_dict['expiration'] = '10/10/2013' + input_dict['expiration'] = '10/10/2020' # Supplying bogus 'root' threshold. Doing this for all roles slows # the test significantly.