diff --git a/README.md b/README.md index d4bb96fa..0a100065 100644 --- a/README.md +++ b/README.md @@ -118,3 +118,11 @@ TUF has four major classes of users: clients, for whom TUF is largely transparen * [Low-level Integration](tuf/client/README.md) * [High-level Integration](tuf/interposition/README.md) + +## Acknowledgements + +This material is based upon work supported by the National Science Foundation +under Grant No. CNS-1345049 and CNS-0959138. Any opinions, findings, and +conclusions or recommendations expressed in this material are those of the +author(s) and do not necessarily reflect the views of the National Science +Foundation. diff --git a/dev-requirements.txt b/dev-requirements.txt index 841f6216..4310cb16 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -17,3 +17,7 @@ # http://nvie.com/posts/pin-your-packages/ pycrypto==2.6.1 pynacl==0.2.3 + +# Testing requirements. The rest of the testing dependencies available in +# 'tox.ini' +tox diff --git a/docs/images/repository_tool-diagram.png b/docs/images/repository_tool-diagram.png index e8e6f289..6bfbdeb0 100644 Binary files a/docs/images/repository_tool-diagram.png and b/docs/images/repository_tool-diagram.png differ diff --git a/examples/client/example_client.py b/examples/client/example_client.py index 53b18142..21fb9922 100755 --- a/examples/client/example_client.py +++ b/examples/client/example_client.py @@ -21,7 +21,7 @@ The custom examples below demonstrate: (1) updating all targets (2) updating all the targets of a specified role - (3) updating a specific target explicitely named. + (3) updating a specific target explicitly named. It assumes a server is listening on 'http://localhost:8001'. One can be started by navigating to the 'examples/repository/' and starting: diff --git a/setup.py b/setup.py index 17867ca0..78a994d1 100755 --- a/setup.py +++ b/setup.py @@ -96,6 +96,10 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Security', 'Topic :: Software Development' diff --git a/tests/.coveragerc b/tests/.coveragerc index bb2d6cff..40958a0f 100644 --- a/tests/.coveragerc +++ b/tests/.coveragerc @@ -5,11 +5,15 @@ branch = True [report] exclude_lines = pragma: no cover - def _check_crypto_libraries + def check_crypto_libraries + def _get_password + def _prompt def __str__ if __name__ == .__main__.: omit = */tuf/interposition/* */tuf/_vendor/* - */tuf/compatibility/* + + # Command-line tool and integration example that calls core TUF. + */tuf/client/basic_client.py diff --git a/tests/aggregate_tests.py b/tests/aggregate_tests.py index 5394498a..a73e95fc 100755 --- a/tests/aggregate_tests.py +++ b/tests/aggregate_tests.py @@ -23,11 +23,26 @@ 'tuf/tests'. Use --random to run the tests in random order. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import sys import unittest import glob import random +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest + # Generate a list of pathnames that match a pattern (i.e., that begin with # 'test_' and end with '.py'. A shell-style wildcard is used with glob() to @@ -42,9 +57,9 @@ test = test[:-3] tests_without_extension.append(test) -# Provide command-line option to randomize the order in which the tests run. -# Randomization might catch errors with unit tests that do not properly clean -# up or restore monkey-patched modules. +# Randomize the order in which the tests run. Randomization might catch errors +# with unit tests that do not properly clean up or restore monkey-patched +# modules. random.shuffle(tests_without_extension) diff --git a/tests/repository_data/client/metadata/current/root.json b/tests/repository_data/client/metadata/current/root.json index 37993bf1..f4f0b55d 100644 Binary files a/tests/repository_data/client/metadata/current/root.json and b/tests/repository_data/client/metadata/current/root.json differ diff --git a/tests/repository_data/client/metadata/current/snapshot.json b/tests/repository_data/client/metadata/current/snapshot.json index 9a960dcd..3d2aa188 100644 Binary files a/tests/repository_data/client/metadata/current/snapshot.json and b/tests/repository_data/client/metadata/current/snapshot.json differ diff --git a/tests/repository_data/client/metadata/current/targets.json b/tests/repository_data/client/metadata/current/targets.json index 0a7b4a8b..624e231c 100644 Binary files a/tests/repository_data/client/metadata/current/targets.json and b/tests/repository_data/client/metadata/current/targets.json differ diff --git a/tests/repository_data/client/metadata/current/targets.json.gz b/tests/repository_data/client/metadata/current/targets.json.gz index 757cff50..874b7b77 100644 Binary files a/tests/repository_data/client/metadata/current/targets.json.gz and b/tests/repository_data/client/metadata/current/targets.json.gz differ diff --git a/tests/repository_data/client/metadata/current/targets/role1.json b/tests/repository_data/client/metadata/current/targets/role1.json index d4d3c560..af4e844e 100644 Binary files a/tests/repository_data/client/metadata/current/targets/role1.json and b/tests/repository_data/client/metadata/current/targets/role1.json differ diff --git a/tests/repository_data/client/metadata/current/timestamp.json b/tests/repository_data/client/metadata/current/timestamp.json index 55099610..61dbbcae 100644 Binary files a/tests/repository_data/client/metadata/current/timestamp.json and b/tests/repository_data/client/metadata/current/timestamp.json differ diff --git a/tests/repository_data/client/metadata/previous/root.json b/tests/repository_data/client/metadata/previous/root.json index 37993bf1..f4f0b55d 100644 Binary files a/tests/repository_data/client/metadata/previous/root.json and b/tests/repository_data/client/metadata/previous/root.json differ diff --git a/tests/repository_data/client/metadata/previous/snapshot.json b/tests/repository_data/client/metadata/previous/snapshot.json index 9a960dcd..3d2aa188 100644 Binary files a/tests/repository_data/client/metadata/previous/snapshot.json and b/tests/repository_data/client/metadata/previous/snapshot.json differ diff --git a/tests/repository_data/client/metadata/previous/targets.json b/tests/repository_data/client/metadata/previous/targets.json index 0a7b4a8b..624e231c 100644 Binary files a/tests/repository_data/client/metadata/previous/targets.json and b/tests/repository_data/client/metadata/previous/targets.json differ diff --git a/tests/repository_data/client/metadata/previous/targets.json.gz b/tests/repository_data/client/metadata/previous/targets.json.gz index 757cff50..874b7b77 100644 Binary files a/tests/repository_data/client/metadata/previous/targets.json.gz and b/tests/repository_data/client/metadata/previous/targets.json.gz differ diff --git a/tests/repository_data/client/metadata/previous/targets/role1.json b/tests/repository_data/client/metadata/previous/targets/role1.json index d4d3c560..af4e844e 100644 Binary files a/tests/repository_data/client/metadata/previous/targets/role1.json and b/tests/repository_data/client/metadata/previous/targets/role1.json differ diff --git a/tests/repository_data/client/metadata/previous/timestamp.json b/tests/repository_data/client/metadata/previous/timestamp.json index 55099610..61dbbcae 100644 Binary files a/tests/repository_data/client/metadata/previous/timestamp.json and b/tests/repository_data/client/metadata/previous/timestamp.json differ diff --git a/tests/repository_data/generate.py b/tests/repository_data/generate.py index 2bc1c67c..15e5a661 100755 --- a/tests/repository_data/generate.py +++ b/tests/repository_data/generate.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ generate.py @@ -86,13 +88,13 @@ target3_filepath = 'repository/targets/file3.txt' tuf.util.ensure_parent_dir(target2_filepath) -with open(target1_filepath, 'wb') as file_object: +with open(target1_filepath, 'wt') as file_object: file_object.write('This is an example target file.') -with open(target2_filepath, 'wb') as file_object: +with open(target2_filepath, 'wt') as file_object: file_object.write('This is an another example target file.') -with open(target3_filepath, 'wb') as file_object: +with open(target3_filepath, 'wt') as file_object: file_object.write('This is role1\'s target file.') # Add target files to the top-level 'targets.json' role. These target files @@ -106,11 +108,11 @@ # Set the top-level expiration times far into the future so that # they do not expire anytime soon, or else the tests fail. Unit tests may # modify the expiration datetimes (of the copied files), if they wish. -repository.root.expiration = datetime.datetime(2030, 01, 01, 00, 00) -repository.targets.expiration = datetime.datetime(2030, 01, 01, 00, 00) -repository.snapshot.expiration = datetime.datetime(2030, 01, 01, 00, 00) -repository.timestamp.expiration = datetime.datetime(2030, 01, 01, 00, 00) -repository.targets('role1').expiration = datetime.datetime(2030, 01, 01, 00, 00) +repository.root.expiration = datetime.datetime(2030, 1, 1, 0, 0) +repository.targets.expiration = datetime.datetime(2030, 1, 1, 0, 0) +repository.snapshot.expiration = datetime.datetime(2030, 1, 1, 0, 0) +repository.timestamp.expiration = datetime.datetime(2030, 1, 1, 0, 0) +repository.targets('role1').expiration = datetime.datetime(2030, 1, 1, 0, 0) # Compress the 'targets.json' role so that the unit tests have a pre-generated # example of compressed metadata. diff --git a/tests/repository_data/keystore/delegation_key b/tests/repository_data/keystore/delegation_key index fbdd0d1e..e4fd71f8 100644 --- a/tests/repository_data/keystore/delegation_key +++ b/tests/repository_data/keystore/delegation_key @@ -1,30 +1,30 @@ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,31815D9E16C988F5 +DEK-Info: DES-EDE3-CBC,8EB89B1037BC3FA7 -oDyggW94u5q5vkUJBzKJJWrkxxdN2EgneApAQL6AZQBnZQuOn9vbxYiX3DZVK3nO -jNQ/eU4JrUe6dueLl+xlipx6cHq0MbBNrLA15sMBj9l4KSsVtiWhz/9mSPBdOqWV -hLL34Rh0/84P6id/Xg9aFJFEb6EZkUOO99V/8Dc8kPlWaiW5LUKN0j2Gkbf3Qtci -UGOKlKfHAKiSziKtkhh9ai9qPRpxEFXTgDpkhJgcy1QxOi7M4VF5ljEr/xYyFMlo -K9N6f1ZuF39K1qc9kTrMmtIOtZPaU/kGsFlCBIUT1h4JVS+WPqOxb1QQ6d2vw0sm -VTuc6xxGbDf2r91dtoAvqdqSCJuQh5VS6sKSo4FIWz82AwNKX5OKwylJ64BCawxE -Gw/El1q3/Mxwljl8pDYow2pTfUa7c2HW+eoYZwGOPPHOnA7J4BcJPtFb7hLUXCGE -GszSZQd1SYqj6GxAqVcYsK2AWzv/IXKcZJjlQD1tQJXz8aMLbbX70S+TuUtCEHaz -4DP9gCFZLZGCwGD9kE2qVOfObQADj/B20VpoOVSWV6uvsMrjY9EnauVsWyZoB7fY -AxMY7Z4BQBzNqvhHTMgUgS18XGFKOPQfAnQWNq4DVssR8+OPeXeeLFriYhSZ6bES -hvuW0gWwlU5R6OT3SC7lr8Jo3WjAcOCpJ1iFS1VH5NljDoLzup064Jg3HUCcEMTl -zF1kMKRNGuIdEy2JVFYh538SC7DJ+04hLOvpulqnDa+OLs8s5LlAeDtTDyZiEbzH -IjDJK/ZcmG95N+hg78u4pTr5lr9Y5NAour+DXPrU02LTHRKlgqah1Va5huNRjCmh -4MEc90G2ODxs71Fg/bOGDXAg5TSt/MaDhweEzGf54CdAuSKeREmdj0cbjsBzdvyo -7+VsFozx1Sa6wHmHmQEEVM2a0lEU9PuzsOQfSBDy4n+RRuU2JOCmcFlox1q59693 -P61qvJDT+UT9pGvyJ4oJztHyh9O6nHqPALWxP1HPWwL67y2g1+NvZo3hJ9mN68oB -u6+xxEciPRsTi/Mg2YhfjoZNqkN3NWgJQ87zpeYfTwosJepbe1CzNmQPh4rWSAV2 -D5OJRpOgniOGJxtE7+wMLpoeZAu3nqLE7u9ebKVp/gBz63kk8AYxB3EbclNNFLDY -i9ECD/ZncYdkTHXx8KuDIcEautxRGeBqmQGuwstduoF/scPC+8JzNPBjDYbbSoi6 -1HiFvMYNarJO3tIPS+8fP4dk3TjI1j/XuVQp2XGTE29+po8UFAhLCtGetyzCbHpd -2qvymx0xNVW+ISDFnMFlhI8o8NQI2ml6LSlAk/A8P7ZqVsNCW4K9VglhbOpXO12G -U97vkNVqOykGRhos2iztmyOMmjQ8GRJrcNd8OsVjYIIcr730L835jJr65fp6t2Sa -RPbhMMpMsepQFAlnFlrG8i1jzeiiGT5K+kPV52EWd2GiaxHdzMOvp7wFgJJJmD+J -3uT5wgwEbZcFtrui01RYSetPEWi4JvEEadkazlGurbAeodO0/gtyeZNuOtyQvs8A -2lKAN5qagyBKy9RqKzJQKpiIyflq5S1B3wxC6WvGM/ts8jFNrfgxhxiZ8Fahoodj +UTGKs5HlOy+TxAlskIC7gTPh9CZ0SJZs9rxUOgutLQT0CL4A8StSeOShx8gGsVxz ++kAsehxrwD6MpGU8E6WpTNhGQcmgfSAgbI1PM0aZkG7DimIWz/ZlYRAkSoHQE438 +HnqaxydbpcXfb4wpriTx7bJx9zmGKysv7lb+j3Ub8LD1Dt96UiohmEYGnikur6CI +7s+HhtdGNh+EGh/XBqUjIZRf0iA+HHLidWU9zL9e3HQNUc2hgVc4DwcW1lKz4ylc +FLgiXhuRLJJVv2ciE0FXlBtoxZNz80fTuVtN8tUd7LSZ5E6radloeV90+YNzOzQx +m0cM4bVkQrVKBkZmNLNp18qa2ZxB0zWWAM86th/YCSkRTTGIayKEw+M+642F1GXZ +wSQRewjH/P2gfIwLLZre1/eZsohfmqC1FpRaGK4626oLgXAhaOuneucJdrkCgZeQ +PxekzJrvfsbMuTjRq9w8EfoCl2qsQ17tKhhxb1QC3tw4aaT8Cn9fDUMqolQ4jtTm +Rvefn8gKaDsFjnym2QV7+Of1i/rgmhE8wHEvpK6i9yQyfjCc1/5kl5abTmdoB+aa +rzD02uNfbVrp7rzP4gPTLyHUXM8k1ffKRlnf3PRyqhN63mMnUNKp7w3lDRR/66Ld +ce37Dc3/FQc/jM3fKIS3E2XAcjWKgHla1YdQZpQimvVR5YNK3j/f+p3sfphyTXOz +a0xN/1sd8yP32MLVxAnB/9fSfwUecaoU6uPb64gVbRJHozZJF0BZaMioxgBarg4r +JpcD/3aIRoB7kEUmXTEGifu+yW/Xl7JYW6gS5IdQ7V2ZFnhhlr2lg+MQ745CXgHZ +X3Hgd/jsQkGPkjDrtowQ4B6cAWs7EflD894hVnt6QPLm0wA5CUYKXybX3jm4Rv8h +LUNtglrj9WKSzt+KiH5j4eM7wcP3NSNv8nLTkd95uVyyuGFJpmyb/Rle5+X9Q5Or +UbJhF9E44CFjTE2kAPZwgRn78gLBX84znS0rV0F1t/0jc6qT61492PbT79rdNka4 +nghGculmnH4MAubDcQDfQSn9vjbeRc37Qd0SQATjzpJCJDEQh5v5htXqf2Ip2TXP +ayOPwYfxABHo0D+zkaYEPentjlFuWvNj5u6+eREIY19Opze9yY0s1A7whFvSgAjO +OIgc+ZkhR5JgmJA5Jt1DjffWYCiPzmL14S+oPd1EswBSPSKIH09CK52LoMGk4c/A +QtlEtHVR+r89NtAoU/l8Psr5dJvwkIH3cek9ec6IETT26Xe8D4eibruVfeHVILG4 +3vFPro8rWWXg78LgI5C3AyC13nA3yy25Yka/IDXYK/VbXDzkDWBGNmJGWds2Eswu +1VxXwEe0BdnxNXYubU+vJ733L5i1QIbWMCduayoifPV9Yx+gDgPbKCAywOCQomsT +3V15myDzdNajuyXt5W05CjPKq5VaUMLGCoUzBaVd3zLRCIsGf9gR77nzyJMayQwi +vgB2LzRaim5LuKQyBUAVaLPzkItq4wLE/NDHul300aR25pochGvV0vUI7IiPKoIF +kJk0+6ObmVgT5nS2cUrgi2dqsOLoVWKeUWH0VurGAh2FQ1fx+stUtzxqmNj35/j9 -----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/delegation_key.pub b/tests/repository_data/keystore/delegation_key.pub index 780c740d..cd3e75f9 100644 --- a/tests/repository_data/keystore/delegation_key.pub +++ b/tests/repository_data/keystore/delegation_key.pub @@ -1,9 +1,9 @@ -----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsD++h8KXKhrJpzYxvZTE -7GqTDKM3uAWL8qGcQnvh7CNbqR4WmZrCQj1zF9hO5OAV+lGVD+JxUXW+yOPw+RjN -i7mPVa12Mq+vCS7WuosoCoooLpnYhRRVYMgpnhbvnjS6xA+7myJ1Bob+7WUEZZlC -oWdsmjfYG82sA7TOTubPk0s9pqQllINMEsB4JTTt3P3DD9+uifCFhoAEKeAItcgA -p4PJw2ImNwJiuet5wTP1ssTclPPWB6ofkm8zXoJUywTIW+hcQhN88jQv4Egx1llj -pWZbEdkIpNxjm6BAEqfPfiuAt6MA8cmdRpDl+Jn030A1kI7NJossuvTcfHwvReZO -WwIDAQAB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqKdTRVn3mLQFUf02Rpug +wVEU4yJtechILLb6nM7+urfwLe6f7EsNCDFhkiTP7vKuQywdLYrhwZKYZMDmaVnI +q4d/tBLvb/jGY/IPFVvWbAOWtwWG7apiAFrcp3Idq6EKGaVVLn7tyv74+nisssYJ +cVKodlkzpgX1Ibrdq73BUlAxhEQNDAUM5bzyJUW0BU4OSjUoFKCgc8BSkNcSLwXO +RpyqAwDpPWiL68N1Dch7R9uD6GE9aREY9SKoYsNCvUOraIcme4fJZ3NmxpN3SVnX +tepoiJo2iAtORtEI1yTCv/dOPap/iebveeCjn667HkMezJodSR8X3pMgMKMVyxhJ +gwIDAQAB -----END PUBLIC KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/root_key b/tests/repository_data/keystore/root_key index 7445b409..883165af 100644 --- a/tests/repository_data/keystore/root_key +++ b/tests/repository_data/keystore/root_key @@ -1,30 +1,30 @@ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,820FC61841CA82E2 +DEK-Info: DES-EDE3-CBC,B90D8CBD38BD4DE2 -ij/VFR5b9yvvlcLWW3sbbAIl4GvSDke44Gi1bu/uhX68QwpEMV88me0UELP+m/D/ -+l2k6u2Er+hV2pYU+cwXGGo88b1cy//6MYMVqmK4mlDnvl6qhlUTFN8XvNd1T5WS -WMRLKcAZBjkTIy8c/SblvwqlgqvJhoKvqrCBAcM71//3J/GyTGLiUJ4TlOd3RMf/ -LxoNIhcUJMUkOswhc1mX4GiaPnsprimSiH7vEILEBmti/IE1Tg8NWCtMypLSYalW -jyh6u/5V9ELmoNOgKH2dPFB39sH5nCc/kzMhMeZMvhFZnOeA9Q7pdsCLpOBEeopa -DrZply09N/FO/1yaiKX7t4KJOd75BvfAztyIbsU1dvlh2w6f2+t33hhlw0jP88Bk -y1rlm+B/TmqQE86hLpU9FDCxbNQQfgZ/OVS3vL27nhk/0BPmqhyBSRC0yTKz/bbI -HLFiMZ+BJCWar3tX/C7XjkCILxICOdxlEPuZlv4k0IpazrIjrw43Xw4ot80z+bnp -C5zxQ8iRlxVtluaQCEzGHEEsBeA96TDMtaNMPtHMvVP3c+X+PfuNPvyCkqkauVB2 -zFioXIOUw5zVuCqWs/+5PgxgPYsDgiFxbQDoIxQQ6dUfMoCZxD46WpIFjbSVu0M3 -hkG0XFvlKxEJpk/CLNE+s1yqtsWHEBD0LluaVYFhCXqkgmrfII+1h9+MVLl8vE5t -mCTqswAS1k7t8kOFKimnWU24ykFxRGookrieOl53Hlt3XpAXIVJ6kHKIUjjKVJp4 -5AdmP4M0IFqKqtHaCzR/UrSsNuIdZlDLxS1aQRx2XS5NIZkqZ3IHkrC4wCxXOoi3 -QdIO4HsxaGViC9+Kr16NatzHPp4+kbfjWNHZizCOZNJfJax4Jvfzer3kCyfdBwxd -K+Gpo+VRuBZqXnnSndKHdYbxVHpBXji2Tm5eGXTqOx2WmtUS4yzswB7VNkxb2jS7 -PF0cI1SF3cq+lmMhNhf0I5rjMqOtyOtx0HhGyv2SiNf+7XWLSh1L+tTtCpiS6YKC -Qvh8DRMVyuINcjPRzbNRWdWxGeq+NAVV045nZy6lQf+/XKmUbk7BGD7xxF+SPeqf -pdHT+GUEJs5Xh4DRu8g8Q2UvkgC6/5ykT4FS/GWSzNxit8oPJGQEIu4joCPlnSSJ -toqVRKaTC1sNTFZKOHOLKFy/CQejKz3EXSEThuWF3/ClzHn0htD55+F8sUt8Hb9/ -zOHud5je2BBocHKpeeED45fs+Q/Se+BlZAkFE1P0STjJU0PJJLYd7RUePJI/DaNq -qPA/XuX2ttqqBoXpyMy9VDL5rcWuazZuTIUnT6rAeMIodMOipdL0Xg3sehxM3TQA -NxVG7eeGHV9Djb6ooNrwnADQT0r1TXw2fwkL0oOA1pU66QpjpdDgT50zLkwNkvQB -gby+sgGO4mi77cxB/L4LiSCk2wojkIs+gmTNFNmT69pNr5jJMRd9ev9fkqMdnE0A -BT5+ztxmwavYFP/LosRg0LMVr0AtQIOlnRgs0rK3Umn6Pv8Y1vgmudGO4gy2sbX7 -u0KjL4FltVSHe0BCaEY7m33Sa25rNPXwYMxkuVSJZs+zfIM8UWNl3pcDTjHvGaHy +1TT2SOrSKoqg4r8DryQXBevP92jYaq5kA5QoW6ufqKV3TwACRD25P9ra/3wRqGWP +OyVeE3KEdma1Zp/x5HW/6ouyhzPC6i82NvqOz61P/5B/NKManD8xj/0i6RlZTW03 +lOBrC9chQPcQkrdexjffGG+OWBqPg8H0ApjgDyyzxtvIK2SRzYhSoLtTCkznLDnC +Qo2kLMtPvxxyXf+fMwyptSQyhieoCCDHTgtvtG3EHIGgeJJk5bORFoH8XPFhLVMU +PO0asgr4WUWXrHYTgrzFvMvC3Jsm0FjuASHZsihlwn3gW22aARU2704rLSjjTEgU +F5fzKvyUbbytc1TNjT8QOc7m78mjBqVdOf3WsH5eD1BRdexAbIRtfw8TGhGtd0f1 +KyHl/7iOEQTiNtAkCigfxzUBXv0godBPZnpbHLk/cx0Xow1wo+6TzzQP58i2j0hk +TE5O/I6MgJmmn8lZ1FA1IkOn05kny9TST+ZuaJTfQGuV3AyYsBBtQ2TC9veuXpu9 +DTsf8eVNCr2J4x5sT9ihCKIChBdxj5l5CgmOkk9uy/3KuBjXH/jSlPzjGX14tURg +SfhxY47SUJGsqAxdBHcQnnAhNUAqO9TW/soVsrLLKgZgRUHx3isIEADwwGko0t35 +1m4RoU9hFr+hel2muWgFGebTZsiz9Lx1sJHlVPWc+CM8XwBBzVWMMpC0/PYRapQl +4LkA7hlebJESVG/2o4ItMWho/qDH/jZkRgzcavNzfmV+5DAKE2wquZXrc7rSjlIm +xEpqP+O6aE+NwIxI83slL7Ga0N99vIGNC0iEoBWBXIrWsVNGJssX/F8OJUC2+f66 +Rwy0DbcO2h0z9TqKxOcnd67420KifDn6icp/JMEXGHWWyS/+OR8Q5XA8dP20PlGa +WHQ+LhEAgx2kHE3Ciz3luMMmKbVg77AHofMm6zk5rfyHFpXQq4CDKa0uES9XmWeP +xuhcQ4py69gRKxVvlqNoPGdnZ9D2LB7CKIdT/MhK7G3uuMHkdLpSUbGWb6K7CNno +q7fPglxyrzAsr2P/AAdYBd+bMDTsO2p2Nleq84yhj2hZfZHXwztMmdvHRhhAmEyF +pobpENclV075bRtZqCBHS/8dewkM6LE9vnQQJ79IqUXv3fd6oEewwtK7b7++EsQL +LdKx43CPQ0sjJYkjaWgzFKqh8s5udsmrRadmdmMDh5UMUtzCGlS3QEHBECBwkJex +Vgddw6zZy2bmJfANN0HUIKMzyvJ+wzho5FdGN+hkafdVa/dHR4GhAzMTbt78SuKS +5nqV70hgubsDRIJYIeUYa7nt0CC8a1eybbARCMNYQ4NsSzKbel10Ge0WoSUyAHJw +VWbPSlmfc2D4N/8wPXsNtR9AU6fm+z4CzudIh9HI6V/muPBJzBpUahgPpGR88RNK +5zdxymOJPjeKSGvf8WfysBO3Uz0ClKOXPIlJERPJ4msLoD7SYixdTnZnw4NaNZW/ +ceHUmBbqwoTTp96+cp6zZIaTeYuAKYbcznRpmb7K/15u3+Rrkb8lX+cdQDw0KCxL +DgRPhuMdApuz3LCHA5ztm3vYuQGMZshrkIg7Bwg122+7VQ0NAaIdIA== -----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/root_key.pub b/tests/repository_data/keystore/root_key.pub index 9fdf180e..40ae3768 100644 --- a/tests/repository_data/keystore/root_key.pub +++ b/tests/repository_data/keystore/root_key.pub @@ -1,9 +1,9 @@ -----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA72PwhMhTpDZ/wuB5Bdst -gYsfEI931GcIZ46Iq2dqMNWEg2Qt9w6W0xEQj8M5R99XFwbhXL6U7hKGDt958FzT -OL6CnjrnnBgzjbFm1vT380Qi5DaUbJkPcNmjzV45gGZkJ6LnohnBtnWUM/IdbbwR -PdWaqBxWRJHECPHjgbKt6Y9kDwaO6tJQdUIDGwt2V9hz9orPqwiX+c6uO6qJ0naU -F31ZvI4AtHUDaesbyp2j2X2dfCKNiM2t2sgxz79/G6VQvKG30PXxVPXvOhCDowsk -5NdM67bWIkFyf1yNArrhw0D/c0aSGZhYZs+FqvBzKjCy+9+uEfLZsRra6zvx8Jw9 -TwIDAQAB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7J15ZaeDQPrhQsRj29wB +PhibH+Do59xsT2396L+uCg793gZlar5wZN2eHSh725cNQWyTAa9LwG+lXaKMukQ+ +8176CKR2J5sv3DezrGVu3x8V1qhyJyy79FlNZRVYTVqNaYzvJzxsVnFPpg7f8B7C +ffiqWJr9XkpqwRlCpxooXm4hplZ7uek5Ku21CzQ4OWg7hbuc+ZjCGzpXfm8NuosU +7TipnKGpEt0Agiph5g6TB2/scoeFar1CKMONIl80maxzAQk+xkWgiJ00+Z2qFCsx +ESfis/YkILS6RMFyZz7oa1WwMtUjYmrsRuz+jlFcbNuxZpIkaISiG9a2YdGcJ1Aj +3QIDAQAB -----END PUBLIC KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/snapshot_key b/tests/repository_data/keystore/snapshot_key index 5fe431e4..266801ae 100644 --- a/tests/repository_data/keystore/snapshot_key +++ b/tests/repository_data/keystore/snapshot_key @@ -1,30 +1,30 @@ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,0E7D0C9F6987D107 +DEK-Info: DES-EDE3-CBC,9DA302AB20EFABC8 -XrHRVUpQ1wkE2qxhS6BlvK7KkI+d61vg3zfNbKzjE/TpBz/PW0N8wak7Y/CrpsMQ -JHHmaiFx3nGzy4NSq0tBR+ek8e0E+mUrvpICC7M4kyquxB2QA70Tn9cyCN1VfzEt -4p+VgZiNRPEj4gXmgE8eP8OVG7Kn9I0dfVwVAb+bsGeWitK+DdYuyW13fyhVnjlF -we7gpnAcqUYUrR6HpSvjLgo8WHnXTRlOHKzulduwF4udbLQmhUuRUYVupvI47AJi -syHgSkGWwoT76CXtO0cCJf17ykh6M++vB5xDB0Qlf3xepmvxQxbohFsrwFzJKiql -YZ4OzquasMKfdVKSWrskkCK62fLygmABx5zRXPOR5bgZiLBj/CxHhS7aUoYRAmYJ -1PUGN5n21YLl6NqAPKvSq20fc753e5anWopTysbJXpKQiihdmTxX5/uSSuBL9E+O -O8HC6l4LXXnCwuw4LVdWIN6gKRG7Urf3AT6966b9AeDwe2lHyvKgfyOjmwLkkm1q -5Oh+5lob5uL+1wzaQiQnRDnBpb5XGoi7MjJ2azcas2neOm0jWK3j7UhrM2JqIVd5 -G2dk3kLsX3+Oj8G9YlS/O0xYEZUqajT+ktY6jku9uQfDYM7V1ZesbKrj2Zfyc/zD -+xjojZ4NJjL9T+4q5exTH5oC+n7zUPByNMggVbYt2xEOJYBrCIjw2If11Tmcucht -Cw2S3iv6mJsd50H6sOpSPH52usbi9NrAXgU4U3GmRSUiNS9/6wH8So3GPkIGRl36 -fN/GxGGLQvUXyvP3nTJIpCdcbRPM38cGEoa3GorSquUMKHcKjwYHgn2MJVHfibai -RcLKqILbwZd2hJt84HskF/znwN69NHg+e+muNs/diPlhR5h0uvIjCw9tEA9CxMt9 -++P26Hx1YNlyGtv6Sg/7m1EIVg8XcdkDe/qVDzHSAoKDRg3UoM/v4y7gXcoM1o4i -uO1bRAlEG837Oh77es3/qR0cIcirUMhPd4PGI+rHIUfIxdZe2YnpLQpHHCHMjzVi -hUungj3nYBxnwc93xA2zWEvyYIrKS8GtfKiuRVrQ8mjymL34sIWpK847pJQHYnFH -iLRPsLOUYR3k2eCxsw53yVXhgCJPslxwg/TGTIBz3ye+lXp+w6FZp1gBpz+8pqCl -tTTOPhu8H3aL1UTSQCJ8Ew6zcXSyQVFPGjmhKvvJIQH5OoUCVGcwOOICqNobc5Gz -6IGZ10TUWBlrsxgk17n8a5+4PKngrOwaU7Z+YUUqG62boLrmfpDL5laoDNcRPuVx -nh9vXgBurkvt1MmvHXDFsSyNmEs6G1NSjQnb6Ij6GbRRvsbvbWM/DuJ4iLK5BSUN -fnkfpiHyyBYoA6GNgHMhDmPuVQqyfgmikeX+haGDIgyg6P1H6mV9/5pxDg3arSNS -ivFbBhY9lCjmrr4yMHHH+ddFaWvSOA8IH5daM0Zanei2V8A4fzKEMFm2frUBDNTe -HhMcNzUf29lTO190L5ojdtDJEn72YZFW7xhA0UvQof85nyFNCE2TkhLGGyjtcuLK -kNucHkjRrfX1o6Ut8MypgCIFso+MPEJ+Pt0lrK7VHuMJjcHmeJ2abUPmrJNfvrb6 +s9SuQg1zhblJCXnj6mXm3sKerFlJFLo11BeKi7k/kJ80IgJLwKmhnE9n8vulo+ix +6/TM90P1ybn0Qgj18Y5jHixvQUFVgGfBDZno7WgKONoHm6v0e3QlMo5hSe2vea6Y +B+QciU1jzNcI9/y0x7+lghX7BFrtsp9If2xCyI5/gFqQOCYq+4RWvPfUhDR7DUvj +yfsYfx9TzGne2FpvK817gNClKpfgcPoliMVu47Vtlfo+Hye4x/NJWnxCmT4yc3IM +XpFEZ2PgSFbq8CIEObHiwxemI1HPWIK+PxkBrNW5+J7yaNWgkhZVlflQJvx+CQCP +aLgODNLUitD/iD5GQrnQEnc6dYfK28lc4Z6kpFOE6/le41m3K2an8zu98PAZnDuo +DZxobB0IhRgIM3aSkFHjKpFS5lz1Y3serZn2OxScJnHGpAsBgEXnXBA3AmyNArsR +Z2R1Iw+GFbqPDRpVOARkhoYS0VGV0gZ4dlDjnR3Nl9DF6yhpbQDCRnib0E4Wj5pS +fQT0B+o7qSe9eZ4UXVIZuBlJUrz/hn0wIq0tpdmFBswb8VWAKPaNY6sI13qP3WEX +UsxMHFjt9qlCJ4WfegrwLDmUQ7ZicS0DXO7fNNElwbERMXX8K+YR0SIAHT24smsg +FJ5MXRs1jEmu2E/lLMOewR+kiGACp9KrTjWGjb6Hoaftda/69uG3xjhkveprIls1 +ar2nGZXwwBqaWoDKIc5N0zxtIglY5Cq5mssmWjbl6/Oj8UKETYqsuXl2S3+pnrA3 +OjvNMrSAE0EDRcZCBpX5+o4MUy/IwlOOJ+aNR4dK5HfTSXXdmDqoFIERsU+BXeRz +wq29dwoVy8L9m76y+BpuQwO5Os7F89v7JETFyL7vDvJdSjX2EoMiLv+f8x6GiG/O +uJb8ODYVlYzCYf0piIkRXrZfkG9AGTI+yOgZrCu/nlCZpURcONO41btax3IHACdt +BIRgcxPacAsN4RZRdXAPpW5Z68GLZwqKozRoFM9SSnEmnB0u07i5LSeaIt1CGNJm +FtyR+w50RenByAKScc1Jo2x5D+7jkIViH9pogm/WnaEylNYYi7v+KIvp2fZ8p49i +BggUAtXZEMMHVJojJFiVLs+W0VCT8YXj2quwqrDfcAdKa8PazgVdYSXdVj+ii9rx +FIdpaJ8b48Z8CcYubd9Omlz2H0cVjrmjlcXuvJalqM3K19NRtc+wp9XYCsKMHFXj +KdQ1Buva9ZZWcBBMeb+vMXCLIvlbegcToZcXMZZpBLjs7kAD33yhq2tKpeh7Hk45 +E+NrtwiALOWqyVjTagZyYFOD5knOPVOET+DYDdq1A9HcWuZeZwRv201wKz/92K/f +9xuvO3VWBL71FbUhulh2NuknwihmFnzTTJ0nyS/Zg9XA440f2KtlKL61Jonun9eN +RaJTBBKbcfZuRJZH804mq0tZiyaRBf0+wIBgwAgk0oADG77W7A0y2pIm3un7MEHf +XrWGwUktQlHFUkbicae3JGb8/hyHZLrH7yZHWrYf12MvZRT8BjCw3PCJv/lACTiE +g+tLawCAn5Xd7LBNQWRBY5zgexx5maGjq2zcbzFIsqrlHQJ+5ndZtvQaaUuBLzcQ -----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/snapshot_key.pub b/tests/repository_data/keystore/snapshot_key.pub index f892e210..140ca251 100644 --- a/tests/repository_data/keystore/snapshot_key.pub +++ b/tests/repository_data/keystore/snapshot_key.pub @@ -1,9 +1,9 @@ -----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs0uAxc1XwUIayVPNt9U/ -gLM5hjG/6AJ86XiHVIq6BuzUtiuKONZXq3SJwY5eAxdjylNt/A3FrJwylQbVMZkh -wLj0eU1IMh23xV1wOGu63Q9ARkmBAaksM+6apo2CHjtQaNXorhJ3/WDti2hST/cc -HjP81+JwJ+T6lXGKvGDbh3h6Car99MWlMAeYODdNqvRaVkkiQ20HgNB+RSiyGZPi -bzqlMe+qCQU/vktmmy9Zw3bjY7b8GFBix+7PHzFCpX15xKwB4dITPmsiL2OOo/w/ -1Vqu3wP7Ct170oK+bH9eIk6wSAQX3IljKhgzkiYFRyCC33XHRpWmSjMgAGErCcl6 -cwIDAQAB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA9XqJohXw46tOKUiOMzPx +lDtrSlpy3WLH2zFSppN0eLIqByD4mk5nbyWKOzzGetQYgv9FzyER4AbmG40kD9bT +2jm3zxjoTnCoM+1Qt7khZm3LxcKBa7q1yrAlvSfNLauIC220kauVRn4Kehd+IqeS +/LhfOT6YyHUMH9SjZKM8XVHU1ehxTiA69eos4AosMK1Gf7jr042FzfiBTygqV1h5 +LXxO0IUYXiI4eCYTwzK4ChfQBmG3DGFGh2G8yrgqQZ5ERaBQPYG9rqQnfF8T8RUQ +o4n7yKpEKSWLOr6Uz9Y1pnHZG4YiKKbTe9EKGtrRbDMIfI+Mv5f3+n600nwZrN7K +OwIDAQAB -----END PUBLIC KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/targets_key b/tests/repository_data/keystore/targets_key index 107f8ffa..a4f2c771 100644 --- a/tests/repository_data/keystore/targets_key +++ b/tests/repository_data/keystore/targets_key @@ -1,30 +1,30 @@ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,D32B3B70C5CB2933 +DEK-Info: DES-EDE3-CBC,807D96406E906425 -qW+li52UfAVfYj3ibhUW/PDHMOrBQyqUa9zVYChG4dH1AXBjIXx1PvCyse6o+3XC -Rr2XkCVOftlZDD8Zi4KievIyfPbDEbTDnprOE6jcL0rpo2dlkqfkIvaweWrtPy+O -pEn2kNyxSscovqgKCo8t1DKT8DHqJdNG7HseBmTfO5zh+RupXQwAv+4KdZfNmGNA -qsyo3OzdqyZlczaUt5e4yBD1lmQ2aL0CIgFEOAg/2Wy7TXp9wTpnZm0wwj6F03Xi -dAHUI4+PMQXvty2OpTfp9aHbx14IBefziW6ziSHEpd+MBoJjTebykTazBKdVUbI0 -d71SoCcqLfXOcXmNiGZtT97iXhf0+kvby2hNinmnnDWdXWuC4RZAAXJ5J634BO2l -8L2GIWogNXPP+Zl9VbmDbfiUjvyS3kivPdKeSdIjtl98IuQmZFsOaqidxad13pZ9 -XS4IihKbDLdNeWy2xWVt5Di3nzTuEMtbNzsohbjWNdj4NjGOPHl4aXsp/POkUBs9 -MeYA7L2CdLBE/Vc486tafY6zuVVZiX/z1w6b51hFWvygia/G1F0mirKyUBIgZCNm -ZcGS4d0RLWqbQs+vLhUIsr30Q3xRRHaxAUNJF0VGwtG/CqyjyutVd6ZE9koU84S2 -frZ0lK5Pk4KhAsYalIiLSxdE+JJh7yDoVREbupMrgdtRnWzkINfer3Uv2w8Ot8yq -fg7VRnYufm4LzdweimgQ1WfhTkZ/kTGZOhmWU9q/cPpRxlmd4U1xecaWRPc/OHFj -w6yI0wAg9bwIFf5Xi3WUhPOUL6+uc1SRChPINDo7dbcBsuCXYzCERlNL5HBZLTvy -zvpQBCaalaRmHk+l39iwtAstasM2Svwp5JIL3Yrl5xkrybeTisUyu+TbRA8RfdGg -3apE1iiWee4j9U1W0SMVIjNuVRpIoYN/a/UdPnpwSkmcb+yaiQB5AcXdr4k8eSLd -vwaKTbqeJyMtbGgJeGjQH4xTRRZ3fTKq9kn/XeSIkzwvOksip2/kwwUaUu8VhWnZ -CQl9P6UssbAvKaNWokDwKhMZXJynAC1G2NcaTds2s+PecbB+dHXGb3pzaNRVfxLJ -PzKg4m2qhIqVCIpO177MmO1MjwptEf+g7zZH0gMI6rwpKhs5BCX0zXbUOiG/rv6B -dztvrnykKeZMaEqajsP0LCij7MEgcYEnh3GYvvW3EwHMhp7ZdRDl7hFnzXMzymQs -okiVuA8hqTTZPF6o9Y/KwWTgYobLAHRcX/qjJEBuXitwMxu8mWcDOEScj8abZPG6 -A3rvRAZRxAWy4jMFW3Ri+BxE65KYUkgusX43hsHgZxygO28QqNTQW9xU6WaQTEaM -Wbq9NDAXIk+3R7dAAy7sXWP5RX85gduHaftLGhQn+AhLwyPOhgQFXekqmmN6a1y5 -uzlRgglcrVswLalBUPZG2IqiNrpvEizIyPX6CifvN7ymsevg6mOyv8WjtZqiPm8L -GtbJMbAQVWfS/+jw+lKWpsqx8IJcnPR5x1XBw7FJow93QAC81Pm5XwLbtnJl9BtH -hCg/2xK1jCYntTyxPZRSstOk8NpwcjaInN2LimKug8pDnOJntIQ5jQ== +omm708pgsHSKHXcafBiqj685vNbnbK1Ea9XPZYE3oQ/lwS0vBRlIFcDVanswN/u0 +BsuiXlQriCycfLFCi+Vbj3Am/PAWTBibvLsoAWlF0ymxzHUmQ4n3rGldkFBPCv8N +8DQz/kRmVwC9e/kboRRwmWymCV/HaUsQ43/XjMf3bG5gps1ygwAdGdfIIkTc1FcA +yzunF3f73edC2IuP1AyYIXI37pYauljI5dUsAwnYTqfxkh01XMdaRjoIU1THk3DP +1Tr6H3XBoa7YYS/Y29LpD/FEaoWmYPcQw4TQE84p2cfUoYmGLS6ohN9m/4YSbghG +0sL5nZRVfHvdZOQoD1n4FwNlcOTHwj20wlUhY0Uh33dD5xEeBYiMndeMisfBOFG2 +bheqVtefQFMRQP4Kdin0JJEKch4AXcMeCB8+RqcfCIPF/6A+IOK48bhyiIbl06J9 +AF2fBkcbCpmzhK49Ou101LCgQvJG49+ZE6jT0sFu+Vij2JT0+zpE+6Z9fvczutRI +8VZWYh2k78WmXVuD5IOvH/srqrZKIzFUiVVDVhFb9fV+SitBpBl5Ui/YyH2WP2tu +uEGatgqZui6YZBBCFDdR2kq29rorAz1x3RyPybfOtQgZWgrzeXMUE5EzCONnIekM +B4NNG5Yz7WJXIEc1aJXNpMT/HfLSXojWoJjBLXjJClUYMr7IomJnNggWNiGJ4lkL +cOmIBZ/z4zsbUlMWe7IrjXcXR5CQj306P+q2kMtI/ACn6X6a36AASpF8hwConEiA +c5YXMLTAHtJYqtL5YE727TcePJlUZFUh2rajO6RxHbz68Hx500E3Ml9tVPO60kvD +rkrIWVsIgpyfRtr8jBpCL+XOcXjddjNQQCB7y2ta0MfX3lJMa5cjb+RqpgoafvcI +dAzkA7/ELCQ1BVpXtuZsnQB1pzfv6aA1ctv9CEJAwpZ45sin7plYJ1Z0gpqcHpbr +sjUGJm22a823sUQYM2lHZsRX4Tx4uA1cQFTz4G/N2wjJeSPV7v/F6FpFRtPBx0S9 +AB2MJNZHzi+UE/w243bdYa5hqd+39HuTkLPpSRfINyjy5OE8+pJx6G8ODRD1I09+ +jE2GKmDguT3kVCF03Sw9IBF3qMvlAtVRqNyvqbIbdqd1tqF7TJWPisobVuWDCNOT +/HULgS+1vcB7w/74GYhniFGIoAokXdpfQ0T5JSDPlhfH4ARjJBYlbfQ4Yd20ag9j +wawMFprnBVcRz7z8NPQIbozdouqxBmgy4HGoHFxv3H6E7L/m49lhk8q/XdJHP9/n +1LULUZ4jFNtm674O7duyaCTWWJTHs1hdmK/Zjm+aTj4qVini+ep/T9nYi2Kux52i +X6lSI+pgixJAHKigb+9QmTxaxqFzVGBQ1Fs166et0CibHSPSSxNoKH2STyZvKp3T +K79Yup3CdXe9qe4995rcNdyB/sXxhuXQlZCJlPdTCtrQ7jrQKbNM9tHqCJ4MTNxG +cU5XOJiQsZTh9ps84wRULz2iHToAC6RaHsiY7Qy8/nWZGihbVwreZuwrdI8UYlPO +GpiSYREOFhBiHQ+hW6sIkLpiUgaOamQBY554hb+xnCpKspf/oGyGIQ== -----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/targets_key.pub b/tests/repository_data/keystore/targets_key.pub index a7bd1580..8080a88a 100644 --- a/tests/repository_data/keystore/targets_key.pub +++ b/tests/repository_data/keystore/targets_key.pub @@ -1,9 +1,9 @@ -----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTXKgkuaAyfvjkInfVic -lJnwLvG9mCXOcxR3HPwHK/8k/E/DBx8YGIGk7NWCXQFYnAeMZZdx8v3dBmH7oGAn -1pW+LnaAI7Csark8sbgmwYqwTd2oLHHmp/fOZ1vNDWjqvMLwi1YUllR6wPWKAvP2 -8i7q7GCT/MBHDyZ899FCR4f7HvlCW5EYNt+wjxdm79T++Ix7iqvs4iUhvllsfdty -cVPWc+wh60qqCCbnr1Fow8d2j42a8mHoIHWgDvEF9ch+ChDOzQ+53jVmXS3CC+a2 -H6EWNvKMc/wXvIAwA/y6cIjCG+Kep9AvVgz1blHf4ReTlNJWu2OkanszuboAR6nt -+wIDAQAB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqF3xKdM9FySLLp0PiwIx +2O9CFikFBo4Xfm4Z0HW69lu2X5WQFx8/GL9kmo1QjZrOwYToYDe287nidbbLs/rT +lq3buN5wPMiD1GbVgGN/nknkkzv9KkJtrSF4RLbKrUnKo7/9C6IUmMt30wBk4GpJ +RZ+8wFfRhUE6859/f6Xl4XbtBJofbIGwV/OBdIzO5zIgB3uBktbbqBVjJb8Oj6Oi +YYskEIacP+TUrpa1iuC6nONj6ahI5NnEjt2B4/pLaUcEPm43kktJTabznkfNZXOa +2nMjngY8v8EbNLBpG6Y7MqZwwLZ4wnaTOe5Bp323YN9eVONXfU2gtT2MBoWExvFV +FwIDAQAB -----END PUBLIC KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/timestamp_key b/tests/repository_data/keystore/timestamp_key index eb8c5bc5..1222b6f4 100644 --- a/tests/repository_data/keystore/timestamp_key +++ b/tests/repository_data/keystore/timestamp_key @@ -1,30 +1,30 @@ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,D5B52C0993E278D0 +DEK-Info: DES-EDE3-CBC,261BF7F755E46B73 -BF+QhAzOH4XkGczsMFpWjDyMGhJVyxOdYa4y5Ik2jldvo/0pXjHFRbuYpsHjRQWf -F/mAdaCNZ6LcoggCMkN6AOeEq8fc7l1gDvn8H+wFgZb5pfdjcViBh0xvqHQRDMqj -t69QVJnywWhlbtK1967nqKWxKIeVh4EWnWkVO2ep2ZYDbnfX0o2yL8lGFgduWFKy -wpWPmeFYpHFo24lEAFQVN9tv0/1TvQGdJ5GEOYhd/Xnjmg+rDAm3qkZ54wUB1sPq -cR55/nC5iSySAtEsXud4lyx16vRtjsY5kKkF5lIDscLxcGaTUEsLRUcQq9NQmk7J -7VCvH1/wtB+h36hFltjr7N8AeEgfboRmquid6a2aNzcY7qTeQrLCzMeEqSC1FXNK -2KUGvRsvroLMAwNXme8WeBzC6EAsF7ffwQENS+IqwA35h+mecFDBDxmeuFZLvlOD -K0o6UF0t0wV6B83qu/ooWkJgeNlPOaYh6XcTrBdC0jpYfBJfdGS6TDxKSYgxCSNH -Y344hhmXoTUvMPLOydyrnQjaISSZY4W9F3cVPTAUg84uK8ZHyhnknKkKDj3sPfNC -/VLXIbDqa5COGpuY/oFo3IZGi5mb7vaKFQtaScu8HDIOE1PEkipMUrIMNebUxBhm -9VyG4kFcQB/jLzZQpOwyIwkuYERVhVrBBWqlWf3TOn7dyq+anD6WUfveSIqjWdyy -NolLMnvdZxk6R3Dm2eKBMRBv91VfJihh2gqu7SlnlK4wmDmu6hB42laPXlkA6DNP -EOuS0zsz+1bkwwrRZzAbxlhexykL+a3NnLGJuoJBPW0larO0xSqOd0/UmJ06A5L3 -h8gVLjwWssqfHftRlxiAPB2KcCA0XHcXLbqdy/XVR4ttW2f9vp8Xb9W3qMxIJWjI -n+bTSOjameg5AYeYQlowXNY+W+P50xFmt/1Bbc5kWyzftQLT3b/SCiGAuLeJiJOP -13osdj3fL6ANj8QjSU6HSfV+oF6EYh9jmAt3fUmYq6eLTTFCFq1cszd093wy1weC -RO8/tB/x2y2jUoKZL2XNDcmfHSvrd93LHhc9BwDQ2XVd61/+iwOAmMuYcLjX717S -IAtKpICyfyiflc/WwPaCXFhaTlafzQKQHbER+b1Y61gvKhDkMxcrAKm+dwh7k7PA -DHcT8+3uhyJbXnK9YXqgvp3aj8ddDL7D5ooLGVhHkhsQNdowx7xfEXXGOTib0sec -FL722JXKaLY2PreL5rFj/sZLKpZZ3St2lIc9uAMOO6XVLbTpxB989B51wIZXXXe2 -jlFzIC2oDx6XPSiOqZS5o/KnBuddEL4JXBRBPDvhPQiHWSh3iM0mDnOiCyYWoo+N -HjGioJZ1IvlMC7M+qyyWZjqPA35Y6kZMNmkg+VgvAfPO4lbgOSQBaenBvAk2AmJ4 -S+3AAijBWJxKN9nIYnkb7DZ/Bpt/rdxdXfZUTbZg0C3LgE/JSvPAhj700+BVZekm -Bip9tuZ3fARxT6K5stVBepTVFM0ueR/97j/krFNQFEJopCmsiqJjM1pQPv+TRmfH -yLkmGDTO63P8xaunrOtoX8NkmcYAl2xwKXhVjSkCxscZj3kSkljYieiC0Sg3YMGu +2faJuP6DIhZHuOcnCLgG1H2C076VFivK4Sz4NQipXCzLrMJvUWhAuYZrAAgDy4Z0 +wSQBONeduwVUdNm5McpOOIOgcIn2p6DDVs09OGY5BOMi1J0MlBw9d667jpYLuMbj +CFUHvXRGLyCagg6eyJyFY3JB1ppQl5EJxh295iIz+4FouMpYQ5C0+7ub9r6rc0iR +1Y2AYpr1XqVS9599sabwMM3IQ0d28h+o7UeleEjZQYG9u+7OF2YddULjB5CfNXcn +yJnmHxsQwQb15YMf4pvc02RGC51BIjnXtYGG2mlyxo9Wg+HZsU7AZByqSh0RLzri ++Us3PKKgsVFleo3V+9L6zS/pXn8KU4X6BMEDLX/t++z0VseNVxaYqMFG3um2Qw6C +WIHtbMonOaub7VzRJx3mvTD+/xLi1HYQ0k037f4z890/HPW2eP3aE9EE+jZJH+M8 +3RAY22qQ5RWtt9oZhNhOPqVBuRJz+ZqMWNY819HrAWR72msUXItTIemwABJI7Wpy +V0LwA03NLcTms5+z6XcdzUMnlcXSnGa9+YPIz7dRP/YNFTAwDgfbfBybaue/7wYC +XABD9WyIx+/7jEN0trJaTKADPBUzNDrKFnUWxSqnnj05b84YeB7gA4TXemin+n80 +rHqAudscj4CqGx1dsYgjoNHa7nlbd5YAE/pUovWR6KJSRwR+jqySO8n7HvwXgxia +AVk3jwq0GVo5MLUMMbkE61gBzTzPVyAF24q8AS2YRG8hFyQzFxTibcCjs4zwK6ZW +WjaIHOU5pP9SX3Vz3WBWRz0KcGG/ebTq/JSOFPIxFbhPm/qyZehXRtYLTiZyvE5w +RvuNqGulA3Zv61+5wdy+Wt43hyF36MhePXU3MeNPBRSupIAaBj+I9fWHyBTGbrQq +DomEcSXExCdm2LKCn875QYNyaxFownKlwObQsKevQhG+DIGR12pzcvXDQLoerfIy +gZH+oWKTipM2BwQzj1fZlh/4nZNflo4q+jPzJrSWrqkPBBiZPrHL53D7coWVxI3v ++qtSI3Go1OoBEZAkplef9buFH+KXrksLriyxIJvzKuzY/y+JeepaBuVfb+jdeoV9 +lyWX4tLMkHLp2Of5rQM0bU0ngT32pnYhCzOUXBmdaF5krCN626sUCtdTpJpCZbOK +0v+ssj4wUnpaIFOwrA+n68eRe5d3izXOABy4WPc66P+k9swIpUFqpBuhikcolqE0 +qDQQg2bBg5lRqCAbcNcjm/59Ozi257SaUDgB/zUBmxE917+rLFpQ1+Y78TBcQXN+ +TXgSsJ/D8oIPEnpOEBIiXdCeOkZwchJAsfH/vdpUf/cB45wqzx3vZiDnWp71vNNO +9V00wOmz4G5yZFmScVrUwsX1dfsJtwUb3Vafa7wsBSjSrWasvGT8FbuygYUG8sGM +rqAsUCvXt3XWY8at75zLRuFwqNlUSbeMOuJxDzvsVRvTZHW7PZPX8tNJXf92TanS +bVHYJTEuwCienKRALp+Uyqa3tUqb/IpwN8wR5NDStr0bO7PvGLFS0Ha6O5Aj/n6s +HksamVs6rrZwaXaoxUFQE8ig9o8bi05RjWSUWcjIbfNvwehJ19aQE31KNze2rsXA -----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/tests/repository_data/keystore/timestamp_key.pub b/tests/repository_data/keystore/timestamp_key.pub index ee92796c..7163fbef 100644 --- a/tests/repository_data/keystore/timestamp_key.pub +++ b/tests/repository_data/keystore/timestamp_key.pub @@ -1,9 +1,9 @@ -----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAueyWz2lDPzyywmWegFTP -l0DDXVNr9XIYLNffLRC24Y9ZKJCqb6eIz3XlF/3On5dCCqKFOfneHm55SxyCzoQb -RAtr5+SmoXfsds7ZAnxv48iGLtk7aZEvHchhEJ+1HsKy5hMeqpXtOAMPJXqBfqgT -ZzXagdCHNnbLEvhyJmRAaK88I/93s57KRNvPp6NIUbQ8EHaX1jaWt+LhGfsA3C34 -qcxlk16LXF45Wm2AgMFCtZ2BecUvSQds24b/ShxfeVRtXSSUMDd+dM+MbwclPsoP -eTeegQATmA4pu4dDaBeYUnS/hstUZS5QPUuvVw6K0Q2JHUWv6CS2kziUjPXGI44H -LwIDAQAB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyw8QAC0PNyyf0AV3qzSp +DgT74sGKC/72HIO1EskP4VEkdfh67PROHCm0YJTYLch9zH+uHIsmyyuzNr6go2Nv +GPSRwYEP34LJlmqr699zkjSXw79T/t244keFiL8SFWTlWmQyTPDdn+N2v4acAmvW +xSFcjTl8cVIGyGuU2s/vHrBn0zoOJ7ZIGAFzzCGAm0j6VvGvkxy3mymE+8VjzrAV +9P1aOMdRVmlqCyPlGVW66Lvz7wkQKcp7rf0CEKkBGlYMtgTqiiagHJy0Sv6qAapw +LXzE6ZdM40E1J1rT9GUitd0K4LhpSjW1lfipSbNQDLiZTG9R2EhnDMl5suaIaFh0 +UQIDAQAB -----END PUBLIC KEY----- \ No newline at end of file diff --git a/tests/repository_data/repository/metadata.staged/root.json b/tests/repository_data/repository/metadata.staged/root.json index 37993bf1..f4f0b55d 100644 Binary files a/tests/repository_data/repository/metadata.staged/root.json and b/tests/repository_data/repository/metadata.staged/root.json differ diff --git a/tests/repository_data/repository/metadata.staged/snapshot.json b/tests/repository_data/repository/metadata.staged/snapshot.json index 9a960dcd..3d2aa188 100644 Binary files a/tests/repository_data/repository/metadata.staged/snapshot.json and b/tests/repository_data/repository/metadata.staged/snapshot.json differ diff --git a/tests/repository_data/repository/metadata.staged/targets.json b/tests/repository_data/repository/metadata.staged/targets.json index 0a7b4a8b..624e231c 100644 Binary files a/tests/repository_data/repository/metadata.staged/targets.json and b/tests/repository_data/repository/metadata.staged/targets.json differ diff --git a/tests/repository_data/repository/metadata.staged/targets.json.gz b/tests/repository_data/repository/metadata.staged/targets.json.gz index 757cff50..874b7b77 100644 Binary files a/tests/repository_data/repository/metadata.staged/targets.json.gz and b/tests/repository_data/repository/metadata.staged/targets.json.gz differ diff --git a/tests/repository_data/repository/metadata.staged/targets/role1.json b/tests/repository_data/repository/metadata.staged/targets/role1.json index d4d3c560..af4e844e 100644 Binary files a/tests/repository_data/repository/metadata.staged/targets/role1.json and b/tests/repository_data/repository/metadata.staged/targets/role1.json differ diff --git a/tests/repository_data/repository/metadata.staged/timestamp.json b/tests/repository_data/repository/metadata.staged/timestamp.json index 55099610..61dbbcae 100644 Binary files a/tests/repository_data/repository/metadata.staged/timestamp.json and b/tests/repository_data/repository/metadata.staged/timestamp.json differ diff --git a/tests/repository_data/repository/metadata/root.json b/tests/repository_data/repository/metadata/root.json index 37993bf1..f4f0b55d 100644 Binary files a/tests/repository_data/repository/metadata/root.json and b/tests/repository_data/repository/metadata/root.json differ diff --git a/tests/repository_data/repository/metadata/snapshot.json b/tests/repository_data/repository/metadata/snapshot.json index 9a960dcd..3d2aa188 100644 Binary files a/tests/repository_data/repository/metadata/snapshot.json and b/tests/repository_data/repository/metadata/snapshot.json differ diff --git a/tests/repository_data/repository/metadata/targets.json b/tests/repository_data/repository/metadata/targets.json index 0a7b4a8b..624e231c 100644 Binary files a/tests/repository_data/repository/metadata/targets.json and b/tests/repository_data/repository/metadata/targets.json differ diff --git a/tests/repository_data/repository/metadata/targets.json.gz b/tests/repository_data/repository/metadata/targets.json.gz index 757cff50..874b7b77 100644 Binary files a/tests/repository_data/repository/metadata/targets.json.gz and b/tests/repository_data/repository/metadata/targets.json.gz differ diff --git a/tests/repository_data/repository/metadata/targets/role1.json b/tests/repository_data/repository/metadata/targets/role1.json index d4d3c560..af4e844e 100644 Binary files a/tests/repository_data/repository/metadata/targets/role1.json and b/tests/repository_data/repository/metadata/targets/role1.json differ diff --git a/tests/repository_data/repository/metadata/timestamp.json b/tests/repository_data/repository/metadata/timestamp.json index 55099610..61dbbcae 100644 Binary files a/tests/repository_data/repository/metadata/timestamp.json and b/tests/repository_data/repository/metadata/timestamp.json differ diff --git a/tests/simple_server.py b/tests/simple_server.py index 217bdeed..57e7d7f7 100755 --- a/tests/simple_server.py +++ b/tests/simple_server.py @@ -1,12 +1,14 @@ +#!/usr/bin/env python + """ simple_server.py - Konstantin Andrianov + Konstantin Andrianov. - February 15, 2012 + February 15, 2012. See LICENSE for licensing information. @@ -15,16 +17,23 @@ This is a basic server that was designed to be used in conjunction with test_download.py to test download.py module. - + SimpleHTTPServer: http://docs.python.org/library/simplehttpserver.html#module-SimpleHTTPServer - """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import sys import random -import SimpleHTTPServer -import SocketServer + +import tuf._vendor.six as six PORT = 0 @@ -36,13 +45,14 @@ def _port_gen(): PORT = int(sys.argv[1]) if PORT < 30000 or PORT > 45000: raise ValueError + except ValueError: PORT = _port_gen() + else: PORT = _port_gen() -Handler = SimpleHTTPServer.SimpleHTTPRequestHandler -httpd = SocketServer.TCPServer(("", PORT), Handler) +Handler = six.moves.SimpleHTTPServer.SimpleHTTPRequestHandler +httpd = six.moves.socketserver.TCPServer(('', PORT), Handler) -#print "PORT: ", PORT httpd.serve_forever() diff --git a/tests/slow_retrieval_server.py b/tests/slow_retrieval_server.py index 6e5cc3fc..c8dfcaba 100755 --- a/tests/slow_retrieval_server.py +++ b/tests/slow_retrieval_server.py @@ -18,25 +18,33 @@ interval 'DELAY'). The server is used in 'test_slow_retrieval_attack.py'. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os import sys import time import random -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +import tuf._vendor.six as six # Modify the HTTPServer class to pass the 'test_mode' argument to # do_GET() function. -class HTTPServer_Test(HTTPServer): +class HTTPServer_Test(six.moves.BaseHTTPServer.HTTPServer): def __init__(self, server_address, Handler, test_mode): - HTTPServer.__init__(self, server_address, Handler) + six.moves.BaseHTTPServer.HTTPServer.__init__(self, server_address, Handler) self.test_mode = test_mode # HTTP request handler. -class Handler(BaseHTTPRequestHandler): +class Handler(six.moves.BaseHTTPServer.BaseHTTPRequestHandler): # Overwrite do_GET. def do_GET(self): @@ -44,7 +52,7 @@ def do_GET(self): try: filepath = os.path.join(current_dir, self.path.lstrip('/')) data = None - with open(filepath, 'rb') as fileobj: + with open(filepath, 'r') as fileobj: data = fileobj.read() self.send_response(200) @@ -68,12 +76,12 @@ def do_GET(self): # 'tuf.conf.SLOW_START_GRACE_PERIOD' seconds before triggering a # potential slow retrieval error. for i in range(len(data)): - self.wfile.write(data[i]) + self.wfile.write(data[i].encode('utf-8')) time.sleep(DELAY) return - except IOError, e: + except IOError as e: self.send_error(404, 'File Not Found!') diff --git a/tests/test_arbitrary_package_attack.py b/tests/test_arbitrary_package_attack.py index f533ef7c..83061e97 100755 --- a/tests/test_arbitrary_package_attack.py +++ b/tests/test_arbitrary_package_attack.py @@ -25,23 +25,30 @@ There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatability, where the print statement is a function, an +# Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals import os -import urllib import tempfile import random import time import shutil import json import subprocess -import unittest import logging +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf import tuf.formats @@ -49,6 +56,7 @@ import tuf.log import tuf.client.updater as updater import tuf.unittest_toolbox as unittest_toolbox +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_arbitrary_package_attack') @@ -81,7 +89,7 @@ def setUpClass(cls): # NOTE: Following error is raised if a delay is not applied: # - time.sleep(.2) + time.sleep(1) @@ -173,7 +181,7 @@ def test_without_tuf(self): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'targets', 'file1.txt') - urllib.urlretrieve(url_file, client_target_path) + six.moves.urllib.request.urlretrieve(url_file, client_target_path) self.assertTrue(os.path.exists(client_target_path)) length, hashes = tuf.util.get_file_details(client_target_path) @@ -181,12 +189,12 @@ def test_without_tuf(self): self.assertEqual(fileinfo, download_fileinfo) # Test: Download a target file that has been modified by an attacker. - with open(target_path, 'wb') as file_object: + with open(target_path, 'wt') as file_object: file_object.write('add malicious content.') length, hashes = tuf.util.get_file_details(target_path) malicious_fileinfo = tuf.formats.make_fileinfo(length, hashes) - urllib.urlretrieve(url_file, client_target_path) + six.moves.urllib.request.urlretrieve(url_file, client_target_path) length, hashes = tuf.util.get_file_details(client_target_path) download_fileinfo = tuf.formats.make_fileinfo(length, hashes) @@ -211,13 +219,13 @@ def test_with_tuf(self): # Modify 'file1.txt' and confirm that the TUF client rejects it. target_path = os.path.join(self.repository_directory, 'targets', 'file1.txt') - with open(target_path, 'wb') as file_object: + with open(target_path, 'wt') as file_object: file_object.write('add malicious content.') try: self.repository_updater.download_target(file1_fileinfo, destination) - except tuf.NoWorkingMirrorError, exception: + except tuf.NoWorkingMirrorError as exception: url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'targets', 'file1.txt') @@ -241,7 +249,7 @@ def test_with_tuf_and_metadata_tampering(self): # An attacker modifies 'file1.txt'. target_path = os.path.join(self.repository_directory, 'targets', 'file1.txt') - with open(target_path, 'wb') as file_object: + with open(target_path, 'wt') as file_object: file_object.write('add malicious content.') # An attacker also tries to add the malicious target's length and digest @@ -258,7 +266,7 @@ def test_with_tuf_and_metadata_tampering(self): tuf.formats.check_signable_object_format(metadata) with open(metadata_path, 'wb') as file_object: - json.dump(metadata, file_object, indent=1, sort_keys=True) + json.dumps(metadata, file_object, indent=1, sort_keys=True).encode('utf-8') # Verify that the malicious 'targets.json' is not downloaded. Perform # a refresh of top-level metadata to demonstrate that the malicious @@ -269,7 +277,7 @@ def test_with_tuf_and_metadata_tampering(self): destination = os.path.join(self.client_directory) self.repository_updater.download_target(file1_fileinfo, destination) - except tuf.NoWorkingMirrorError, exception: + except tuf.NoWorkingMirrorError as exception: url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'targets', 'file1.txt') diff --git a/tests/test_download.py b/tests/test_download.py index bdef71f2..f8aca54e 100755 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -20,7 +20,13 @@ Otherwise, module that launches simple server would not be found. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals import hashlib import logging @@ -29,14 +35,13 @@ import subprocess import time import unittest -import urllib2 - import tuf import tuf.conf as conf import tuf.download as download import tuf.log import tuf.unittest_toolbox as unittest_toolbox +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_download') @@ -73,7 +78,7 @@ def setUp(self): # Computing hash of target file data. m = hashlib.md5() - m.update(self.target_data) + m.update(self.target_data.encode('utf-8')) digest = m.hexdigest() self.target_hash = {'md5':digest} @@ -93,8 +98,8 @@ def test_download_url_to_tempfileobj(self): download_file = download.safe_download temp_fileobj = download_file(self.url, self.target_data_length) - self.assertEquals(self.target_data, temp_fileobj.read()) - self.assertEquals(self.target_data_length, len(temp_fileobj.read())) + self.assertEqual(self.target_data, temp_fileobj.read().decode('utf-8')) + self.assertEqual(self.target_data_length, len(temp_fileobj.read())) temp_fileobj.close_temp_file() @@ -113,6 +118,7 @@ def test_download_url_to_tempfileobj_and_lengths(self): # STRICT_REQUIRED_LENGTH, which is True by default, mandates that we must # download exactly what is required. self.assertRaises(tuf.DownloadLengthMismatchError, download.safe_download, + #self.assertRaises(tuf.SlowRetrievalError, download.safe_download, self.url, self.target_data_length + 1) # NOTE: However, we do not catch a tuf.DownloadLengthMismatchError here for @@ -120,8 +126,8 @@ def test_download_url_to_tempfileobj_and_lengths(self): # STRICT_REQUIRED_LENGTH. temp_fileobj = download.unsafe_download(self.url, self.target_data_length + 1) - self.assertEquals(self.target_data, temp_fileobj.read()) - self.assertEquals(self.target_data_length, len(temp_fileobj.read())) + self.assertEqual(self.target_data, temp_fileobj.read().decode('utf-8')) + self.assertEqual(self.target_data_length, len(temp_fileobj.read())) temp_fileobj.close_temp_file() @@ -139,8 +145,8 @@ def test_download_url_to_tempfileobj_and_performance(self): end_cpu = time.clock() end_real = time.time() - self.assertEquals(self.target_data, temp_fileobj.read()) - self.assertEquals(self.target_data_length, len(temp_fileobj.read())) + self.assertEqual(self.target_data, temp_fileobj.read()) + self.assertEqual(self.target_data_length, len(temp_fileobj.read())) temp_fileobj.close_temp_file() print "Performance cpu time: "+str(end_cpu - star_cpu) @@ -162,15 +168,30 @@ def test_download_url_to_tempfileobj_and_urls(self): download_file, self.random_string(), self.target_data_length) - self.assertRaises(urllib2.HTTPError, + self.assertRaises(six.moves.urllib.error.HTTPError, download_file, - 'http://localhost:'+str(self.PORT)+'/'+self.random_string(), + 'http://localhost:' + str(self.PORT) + '/' + self.random_string(), self.target_data_length) - self.assertRaises(urllib2.URLError, + self.assertRaises(six.moves.urllib.error.URLError, download_file, - 'http://localhost:'+str(self.PORT+1)+'/'+self.random_string(), + 'http://localhost:' + str(self.PORT+1) + '/' + self.random_string(), self.target_data_length) + + + + def test__get_opener(self): + # Test normal case. + # A simple https server should be used to test the rest of the optional + # ssl-related functions of 'tuf.download.py'. + fake_cacert = self.make_temp_data_file() + + with open(fake_cacert, 'wt') as file_object: + file_object.write('fake cacert') + + tuf.conf.ssl_certificates = fake_cacert + tuf.download._get_opener('https') + # Run unit test. diff --git a/tests/test_ed25519_keys.py b/tests/test_ed25519_keys.py index 9a922b88..2cfc560f 100755 --- a/tests/test_ed25519_keys.py +++ b/tests/test_ed25519_keys.py @@ -1,9 +1,11 @@ +#!/usr/bin/env/ python + """ test_ed25519_keys.py - Vladimir Diaz + Vladimir Diaz October 11, 2013. @@ -15,7 +17,16 @@ Test cases for test_ed25519_keys.py. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import unittest +import os import logging import tuf @@ -46,7 +57,7 @@ def test_generate_public_and_private(self): def test_create_signature(self): global public global private - data = 'The quick brown fox jumps over the lazy dog' + data = b'The quick brown fox jumps over the lazy dog' signature, method = ed25519.create_signature(public, private, data) # Verify format of returned values. @@ -71,11 +82,24 @@ def test_create_signature(self): def test_verify_signature(self): global public global private - data = 'The quick brown fox jumps over the lazy dog' + data = b'The quick brown fox jumps over the lazy dog' signature, method = ed25519.create_signature(public, private, data) valid_signature = ed25519.verify_signature(public, method, signature, data) self.assertEqual(True, valid_signature) + + # Test with 'pynacl'. + valid_signature = ed25519.verify_signature(public, method, signature, data, + use_pynacl=True) + self.assertEqual(True, valid_signature) + + # Test with 'pynacl', but a bad signature is provided. + bad_signature = os.urandom(64) + valid_signature = ed25519.verify_signature(public, method, bad_signature, + data, use_pynacl=True) + self.assertEqual(False, valid_signature) + + # Check for improperly formatted arguments. self.assertRaises(tuf.FormatError, ed25519.verify_signature, 123, method, @@ -85,6 +109,10 @@ def test_verify_signature(self): self.assertRaises(tuf.FormatError, ed25519.verify_signature, public, 123, signature, data) + # Invalid signature method. + self.assertRaises(tuf.UnknownMethodError, ed25519.verify_signature, public, + 'unsupported_method', signature, data) + # Signature not a string. self.assertRaises(tuf.FormatError, ed25519.verify_signature, public, method, 123, data) @@ -99,13 +127,13 @@ def test_verify_signature(self): signature, '123')) # Mismatched signature. - bad_signature = 'a'*64 + bad_signature = b'a'*64 self.assertEqual(False, ed25519.verify_signature(public, method, bad_signature, data)) # Generated signature created with different data. new_signature, method = ed25519.create_signature(public, private, - 'mismatched data') + b'mismatched data') self.assertEqual(False, ed25519.verify_signature(public, method, new_signature, data)) diff --git a/tests/test_endless_data_attack.py b/tests/test_endless_data_attack.py index 6da1e496..b81a23f9 100755 --- a/tests/test_endless_data_attack.py +++ b/tests/test_endless_data_attack.py @@ -28,23 +28,30 @@ There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatability, where the print statement is a function, an +# Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals import os -import urllib import tempfile import random import time import shutil import json import subprocess -import unittest import logging +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf import tuf.formats @@ -52,6 +59,7 @@ import tuf.log import tuf.client.updater as updater import tuf.unittest_toolbox as unittest_toolbox +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_endless_data_attack') @@ -84,7 +92,7 @@ def setUpClass(cls): # NOTE: Following error is raised if a delay is not applied: # - time.sleep(.2) + time.sleep(.8) @@ -178,7 +186,7 @@ def test_without_tuf(self): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'targets', 'file1.txt') - urllib.urlretrieve(url_file, client_target_path) + six.moves.urllib.request.urlretrieve(url_file, client_target_path) self.assertTrue(os.path.exists(client_target_path)) length, hashes = tuf.util.get_file_details(client_target_path) @@ -187,7 +195,7 @@ def test_without_tuf(self): # Test: Download a target file that has been modified by an attacker with # extra data. - with open(target_path, 'r+b') as file_object: + with open(target_path, 'r+t') as file_object: original_content = file_object.read() file_object.write(original_content+('append large amount of data' * 100000)) large_length, hashes = tuf.util.get_file_details(target_path) @@ -196,7 +204,7 @@ def test_without_tuf(self): # Is the modified file actually larger? self.assertTrue(large_length > length) - urllib.urlretrieve(url_file, client_target_path) + six.moves.urllib.request.urlretrieve(url_file, client_target_path) length, hashes = tuf.util.get_file_details(client_target_path) download_fileinfo = tuf.formats.make_fileinfo(length, hashes) @@ -231,7 +239,7 @@ def test_with_tuf(self): # Modify 'file1.txt' and confirm that the TUF client only downloads up to # the expected file length. - with open(target_path, 'r+b') as file_object: + with open(target_path, 'r+t') as file_object: original_content = file_object.read() file_object.write(original_content+('append large amount of data' * 10000)) @@ -255,21 +263,24 @@ def test_with_tuf(self): original_length, hashes = tuf.util.get_file_details(timestamp_path) - with open(timestamp_path, 'r+b') as file_object: - original_content = file_object.read() - file_object.write(original_content+('append large amount of data' * 10000)) + with open(timestamp_path, 'r+') as file_object: + timestamp_content = tuf.util.load_json_file(timestamp_path) + large_data = 'LargeTimestamp' * 10000 + timestamp_content['signed']['_type'] = large_data + json.dump(timestamp_content, file_object, indent=1, sort_keys=True) + modified_length, hashes = tuf.util.get_file_details(timestamp_path) self.assertTrue(modified_length > original_length) - + # Does the TUF client download the upper limit of an unsafely fetched # 'timestamp.json'? 'timestamp.json' must not be greater than # 'tuf.conf.DEFAULT_TIMESTAMP_REQUIRED_LENGTH'. try: self.repository_updater.refresh() - except tuf.NoWorkingMirrorError, exception: - for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + except tuf.NoWorkingMirrorError as exception: + for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): self.assertTrue(isinstance(mirror_error, tuf.InvalidMetadataJSONError)) else: diff --git a/tests/test_extraneous_dependencies_attack.py b/tests/test_extraneous_dependencies_attack.py index bd0be5b9..0a9cc6cb 100755 --- a/tests/test_extraneous_dependencies_attack.py +++ b/tests/test_extraneous_dependencies_attack.py @@ -8,7 +8,7 @@ Zane Fisher. - August 19, 2013 + August 19, 2013. April 6, 2014. Refactored to use the 'unittest' module (test conditions in code, rather @@ -30,30 +30,37 @@ There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatability, where the print statement is a function, an +# Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals import os -import urllib import tempfile import random import time import shutil import json import subprocess -import unittest import logging +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf.formats import tuf.util import tuf.log import tuf.client.updater as updater import tuf.unittest_toolbox as unittest_toolbox - +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_extraneous_dependencies_attack') @@ -87,7 +94,7 @@ def setUpClass(cls): # NOTE: Following error is raised if a delay is not applied: # - time.sleep(.2) + time.sleep(.7) @@ -185,7 +192,7 @@ def test_with_tuf(self): tuf.formats.check_signable_object_format(role1_metadata) - with open(role1_filepath, 'wb') as file_object: + with open(role1_filepath, 'wt') as file_object: json.dump(role1_metadata, file_object, indent=1, sort_keys=True) # Un-install the metadata of the top-level roles so that the client can @@ -208,8 +215,8 @@ def test_with_tuf(self): # Verify that the specific 'tuf.BadHashError' exception is raised by each # mirror. - except tuf.NoWorkingMirrorError, exception: - for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + except tuf.NoWorkingMirrorError as exception: + for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'metadata', 'targets', 'role1.json') diff --git a/tests/test_formats.py b/tests/test_formats.py index 35fea51c..af3900bd 100755 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -17,13 +17,21 @@ Unit test for 'formats.py' """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import unittest import datetime import tuf import tuf.formats import tuf.schema - +import tuf._vendor.six as six class TestFormats(unittest.TestCase): @@ -252,13 +260,13 @@ def test_schemas(self): # Iterate 'valid_schemas', ensuring each 'valid_schema' correctly matches # its respective 'schema_type'. - for schema_name, (schema_type, valid_schema) in valid_schemas.items(): + for schema_name, (schema_type, valid_schema) in six.iteritems(valid_schemas): self.assertEqual(True, schema_type.matches(valid_schema)) # Test conditions for invalid schemas. # Set the 'valid_schema' of 'valid_schemas' to an invalid # value and test that it does not match 'schema_type'. - for schema_name, (schema_type, valid_schema) in valid_schemas.items(): + for schema_name, (schema_type, valid_schema) in six.iteritems(valid_schemas): invalid_schema = 0xBAD if isinstance(schema_type, tuf.schema.Integer): invalid_schema = 'BAD' @@ -485,7 +493,7 @@ def test_unix_timestamp_to_datetime(self): # Test conditions for valid arguments. UNIX_TIMESTAMP_SCHEMA = tuf.formats.UNIX_TIMESTAMP_SCHEMA self.assertTrue(datetime.datetime, tuf.formats.unix_timestamp_to_datetime(499137720)) - datetime_object = datetime.datetime(1985, 10, 26, 01, 22) + datetime_object = datetime.datetime(1985, 10, 26, 1, 22) self.assertEqual(datetime_object, tuf.formats.unix_timestamp_to_datetime(499137720)) # Test conditions for invalid arguments. @@ -510,9 +518,9 @@ def test_datetime_to_unix_timestamp(self): def test_format_base64(self): # Test conditions for valid arguments. - data = 'updateframework' + data = 'updateframework'.encode('utf-8') self.assertEqual('dXBkYXRlZnJhbWV3b3Jr', tuf.formats.format_base64(data)) - self.assertTrue(isinstance(tuf.formats.format_base64(data), basestring)) + self.assertTrue(isinstance(tuf.formats.format_base64(data), six.string_types)) # Test conditions for invalid arguments. self.assertRaises(tuf.FormatError, tuf.formats.format_base64, 123) @@ -523,8 +531,8 @@ def test_format_base64(self): def test_parse_base64(self): # Test conditions for valid arguments. base64 = 'dXBkYXRlZnJhbWV3b3Jr' - self.assertEqual('updateframework', tuf.formats.parse_base64(base64)) - self.assertTrue(isinstance(tuf.formats.parse_base64(base64), basestring)) + self.assertEqual(b'updateframework', tuf.formats.parse_base64(base64)) + self.assertTrue(isinstance(tuf.formats.parse_base64(base64), six.binary_type)) # Test conditions for invalid arguments. self.assertRaises(tuf.FormatError, tuf.formats.parse_base64, 123) diff --git a/tests/test_hash.py b/tests/test_hash.py index 9270c913..ecb82e98 100755 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_hash.py @@ -16,8 +18,15 @@ Unit test for 'hash.py'. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os -import StringIO import logging import tempfile import unittest @@ -25,6 +34,7 @@ import tuf import tuf.log import tuf.hash +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_hash') @@ -51,13 +61,13 @@ def _do_md5_update(self, library): digest_object = tuf.hash.digest('md5', library) self.assertEqual(digest_object.hexdigest(), 'd41d8cd98f00b204e9800998ecf8427e') - digest_object.update('a') + digest_object.update('a'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), '0cc175b9c0f1b6a831c399e269772661') - digest_object.update(u'bbb') + digest_object.update('bbb'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'f034e93091235adbb5d2781908e2b313') - digest_object.update('') + digest_object.update(''.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'f034e93091235adbb5d2781908e2b313') @@ -71,13 +81,13 @@ def _do_sha1_update(self, library): self.assertEqual(digest_object.hexdigest(), 'da39a3ee5e6b4b0d3255bfef95601890afd80709') - digest_object.update('a') + digest_object.update('a'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), '86f7e437faa5a7fce15d1ddcb9eaeaea377667b8') - digest_object.update(u'bbb') + digest_object.update('bbb'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'd7bfa42fc62b697bf6cf1cda9af1fb7f40a27817') - digest_object.update('') + digest_object.update(''.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'd7bfa42fc62b697bf6cf1cda9af1fb7f40a27817') @@ -91,13 +101,13 @@ def _do_sha224_update(self, library): self.assertEqual(digest_object.hexdigest(), 'd14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f') - digest_object.update('a') + digest_object.update('a'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'abd37534c7d9a2efb9465de931cd7055ffdb8879563ae98078d6d6d5') - digest_object.update(u'bbb') + digest_object.update('bbb'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'ab1342f31c2a6f242d9a3cefb503fb49465c95eb255c16ad791d688c') - digest_object.update('') + digest_object.update(''.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'ab1342f31c2a6f242d9a3cefb503fb49465c95eb255c16ad791d688c') @@ -110,13 +120,13 @@ def _do_sha256_update(self, library): digest_object = tuf.hash.digest('sha256', library) self.assertEqual(digest_object.hexdigest(), 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') - digest_object.update('a') + digest_object.update('a'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb') - digest_object.update(u'bbb') + digest_object.update('bbb'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), '01d162a5c95d4698c0a3e766ae80d85994b549b877ed275803725f43dadc83bd') - digest_object.update('') + digest_object.update(''.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), '01d162a5c95d4698c0a3e766ae80d85994b549b877ed275803725f43dadc83bd') @@ -130,15 +140,15 @@ def _do_sha384_update(self, library): self.assertEqual(digest_object.hexdigest(), '38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe' '76f65fbd51ad2f14898b95b') - digest_object.update('a') + digest_object.update('a'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), '54a59b9f22b0b80880d8427e548b7c23abd873486e1f035dce9cd697e85175033caa88e6d' '57bc35efae0b5afd3145f31') - digest_object.update(u'bbb') + digest_object.update('bbb'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'f2c1438e9cc1d24bebbf3b88e60adc169db0c5c459d02054ec131438bf20ebee5ca88c17c' 'b5f1a824fcccf8d2b20b0a9') - digest_object.update('') + digest_object.update(''.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), 'f2c1438e9cc1d24bebbf3b88e60adc169db0c5c459d02054ec131438bf20ebee5ca88c17c' 'b5f1a824fcccf8d2b20b0a9') @@ -154,15 +164,15 @@ def _do_sha512_update(self, library): self.assertEqual(digest_object.hexdigest(), 'cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5' 'd85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e') - digest_object.update('a') + digest_object.update('a'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652' 'bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75') - digest_object.update(u'bbb') + digest_object.update('bbb'.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), '09ade82ae3c5d54f8375f348563a372106488adef16a74b63b5591849f740bff55ceab22e' '117b4b09349b860f8a644adb32a9ea542abdecb80bf625160604251') - digest_object.update('') + digest_object.update(''.encode('utf-8')) self.assertEqual(digest_object.hexdigest(), '09ade82ae3c5d54f8375f348563a372106488adef16a74b63b5591849f740bff55ceab22e' '117b4b09349b860f8a644adb32a9ea542abdecb80bf625160604251') @@ -197,13 +207,14 @@ def _do_update_filename(self, library): data = 'abcdefgh' * 4096 fd, filename = tempfile.mkstemp() try: - os.write(fd, data) + os.write(fd, data.encode('utf-8')) os.close(fd) for algorithm in ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']: digest_object_truth = tuf.hash.digest(algorithm, library) - digest_object_truth.update(data) + digest_object_truth.update(data.encode('utf-8')) digest_object = tuf.hash.digest_filename(filename, algorithm, library) self.assertEqual(digest_object_truth.digest(), digest_object.digest()) + finally: os.remove(filename) @@ -214,22 +225,16 @@ def test_update_file_obj(self): def _do_update_file_obj(self, library): data = 'abcdefgh' * 4096 - file_obj = StringIO.StringIO() + file_obj = six.StringIO() file_obj.write(data) for algorithm in ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']: digest_object_truth = tuf.hash.digest(algorithm, library) - digest_object_truth.update(data) + digest_object_truth.update(data.encode('utf-8')) digest_object = tuf.hash.digest_fileobject(file_obj, algorithm, library) # Note: we don't seek because the update_file_obj call is supposed # to always seek to the beginning. self.assertEqual(digest_object_truth.digest(), digest_object.digest()) - - def test_data_to_string(self): - self.assertEqual('12', tuf.hash.data_to_string('12')) - self.assertEqual(u'hello', tuf.hash.data_to_string(unicode('hello'))) - self.assertEqual('12', tuf.hash.data_to_string(12)) - def test_unsupported_digest_algorithm_and_library(self): self.assertRaises(tuf.UnsupportedAlgorithmError, tuf.hash.digest, diff --git a/tests/test_indefinite_freeze_attack.py b/tests/test_indefinite_freeze_attack.py index 4aae5b5c..8d24a687 100755 --- a/tests/test_indefinite_freeze_attack.py +++ b/tests/test_indefinite_freeze_attack.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_indefinite_freeze_attack.py @@ -22,23 +24,30 @@ metadata without the client being aware. """ -# Help with Python 3 compatability, where the print statement is a function, an +# Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals import os -import urllib -import tempfile import random import time +import tempfile import shutil import json import subprocess -import unittest import logging +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf.formats import tuf.util @@ -46,6 +55,7 @@ import tuf.client.updater as updater import tuf.repository_tool as repo_tool import tuf.unittest_toolbox as unittest_toolbox +import tuf._vendor.six as six # The repository tool is imported and logs console messages by default. Disable # console log messages generated by this unit test. @@ -82,7 +92,7 @@ def setUpClass(cls): # NOTE: Following error is raised if a delay is not applied: # - time.sleep(.2) + time.sleep(.8) @@ -180,8 +190,12 @@ def test_without_tuf(self): tuf.formats.check_signable_object_format(timestamp_metadata) with open(timestamp_path, 'wb') as file_object: - json.dump(timestamp_metadata, file_object, indent=1, sort_keys=True) - + # Explicitly specify the JSON separators for Python 2 + 3 consistency. + timestamp_content = \ + json.dumps(timestamp_metadata, indent=1, separators=(',', ': '), + sort_keys=True).encode('utf-8') + file_object.write(timestamp_content) + client_timestamp_path = os.path.join(self.client_directory, 'timestamp.json') shutil.copy(timestamp_path, client_timestamp_path) @@ -192,7 +206,7 @@ def test_without_tuf(self): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'metadata', 'timestamp.json') - urllib.urlretrieve(url_file, client_timestamp_path) + six.moves.urllib.request.urlretrieve(url_file, client_timestamp_path) length, hashes = tuf.util.get_file_details(client_timestamp_path) download_fileinfo = tuf.formats.make_fileinfo(length, hashes) @@ -240,7 +254,7 @@ def test_with_tuf(self): self.repository_updater.refresh() except tuf.NoWorkingMirrorError as e: - for mirror_url, mirror_error in e.mirror_errors.iteritems(): + for mirror_url, mirror_error in six.iteritems(e.mirror_errors): self.assertTrue(isinstance(mirror_error, tuf.ExpiredMetadataError)) diff --git a/tests/test_keydb.py b/tests/test_keydb.py index 1f9b40f3..f7171233 100755 --- a/tests/test_keydb.py +++ b/tests/test_keydb.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_keydb.py @@ -15,6 +17,14 @@ Unit test for 'keydb.py'. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import unittest import logging @@ -150,6 +160,9 @@ def test_remove_key(self): self.assertRaises(tuf.UnknownKeyError, tuf.keydb.get_key, keyid) self.assertRaises(tuf.UnknownKeyError, tuf.keydb.get_key, keyid2) + # Test for 'keyid' not in keydb. + self.assertRaises(tuf.UnknownKeyError, tuf.keydb.remove_key, keyid) + # Test condition for unknown key argument. self.assertRaises(tuf.UnknownKeyError, tuf.keydb.remove_key, '1') @@ -169,7 +182,10 @@ def test_create_keydb_from_root_metadata(self): keyid = KEYS[0]['keyid'] rsakey2 = KEYS[1] keyid2 = KEYS[1]['keyid'] - keydict = {keyid: rsakey, keyid2: rsakey2} + keydict = {keyid: rsakey, keyid2: rsakey2, keyid: rsakey} + + # Add a duplicate 'keyid' to log/trigger a 'tuf.KeyAlreadyExistsError' + # block (loading continues). roledict = {'Root': {'keyids': [keyid], 'threshold': 1}, 'Targets': {'keyids': [keyid2], 'threshold': 1}} version = 8 @@ -182,6 +198,7 @@ def test_create_keydb_from_root_metadata(self): keydict, roledict, consistent_snapshot) self.assertEqual(None, tuf.keydb.create_keydb_from_root_metadata(root_metadata)) + tuf.keydb.create_keydb_from_root_metadata(root_metadata) # Ensure 'keyid' and 'keyid2' were added to the keydb database. diff --git a/tests/test_keys.py b/tests/test_keys.py index c1a2aad2..5e8b1c12 100755 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_keys.py @@ -16,6 +18,14 @@ TODO: test case for ed25519 key generation and refactor. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import unittest import logging @@ -183,7 +193,7 @@ def test_verify_signature(self): # in creating the 'rsa_signature'. Function should return 'False'. # Modifying 'DATA'. - _DATA = '1111'+DATA+'1111' + _DATA = '1111' + DATA + '1111' # Verifying the 'signature' of modified '_DATA'. verified = KEYS.verify_signature(self.rsakey_dict, rsa_signature, _DATA) @@ -199,6 +209,51 @@ def test_verify_signature(self): # Passing incorrect number of arguments. self.assertRaises(TypeError, KEYS.verify_signature) + + + + def test_create_rsa_encrypted_pem(self): + # Test valid arguments. + private = self.rsakey_dict['keyval']['private'] + passphrase = 'secret' + encrypted_pem = KEYS.create_rsa_encrypted_pem(private, passphrase) + self.assertTrue(tuf.formats.PEMRSA_SCHEMA.matches(encrypted_pem)) + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, KEYS.create_rsa_encrypted_pem, + 8, passphrase) + + self.assertRaises(tuf.FormatError, KEYS.create_rsa_encrypted_pem, + private, 8) + + # Test for missing required library. + KEYS._RSA_CRYPTO_LIBRARY = 'invalid' + self.assertRaises(tuf.UnsupportedLibraryError, KEYS.create_rsa_encrypted_pem, + private, passphrase) + KEYS._RSA_CRYPTO_LIBRARY = 'pycrypto' + + + + def test_decrypt_key(self): + # Test valid arguments. + passphrase = 'secret' + encrypted_key = KEYS.encrypt_key(self.rsakey_dict, passphrase).encode('utf-8') + decrypted_key = KEYS.decrypt_key(encrypted_key, passphrase) + + self.assertTrue(tuf.formats.ANYKEY_SCHEMA.matches(decrypted_key)) + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, KEYS.decrypt_key, + 8, passphrase) + + self.assertRaises(tuf.FormatError, KEYS.decrypt_key, + encrypted_key, 8) + + # Test for missing required library. + KEYS._GENERAL_CRYPTO_LIBRARY = 'invalid' + self.assertRaises(tuf.UnsupportedLibraryError, KEYS.decrypt_key, + encrypted_key, passphrase) + KEYS._GENERAL_CRYPTO_LIBRARY = 'pycrypto' diff --git a/tests/test_log.py b/tests/test_log.py index d2854440..006ba7af 100755 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """ @@ -29,13 +30,17 @@ class TestLog(unittest.TestCase): - - + + + def tearDown(self): + tuf.log.remove_console_handler() + def test_set_log_level(self): # Test normal case. global log_levels + global logger tuf.log.set_log_level() self.assertTrue(logger.isEnabledFor(logging.DEBUG)) @@ -53,22 +58,74 @@ def test_set_log_level(self): def test_set_filehandler_log_level(self): - pass + # Normal case. Default log level. + tuf.log.set_filehandler_log_level() + + # Expected log levels. + for level in log_levels: + tuf.log.set_log_level(level) + + # Test for improperly formatted argument. + self.assertRaises(tuf.FormatError, tuf.log.set_filehandler_log_level, '123') + + # Test for invalid argument. + self.assertRaises(tuf.FormatError, tuf.log.set_filehandler_log_level, 51) + + def test_set_console_log_level(self): - pass + # Test setting a console log level without first adding one. + self.assertRaises(tuf.Error, tuf.log.set_console_log_level) + + # Normal case. Default log level. Setting the console log level first + # requires adding a console logger. + tuf.log.add_console_handler() + tuf.log.set_console_log_level() + + # Expected log levels. + for level in log_levels: + tuf.log.set_console_log_level(level) + + # Test for improperly formatted argument. + self.assertRaises(tuf.FormatError, tuf.log.set_console_log_level, '123') + + # Test for invalid argument. + self.assertRaises(tuf.FormatError, tuf.log.set_console_log_level, 51) def test_add_console_handler(self): - pass + # Normal case. Default log level. + tuf.log.add_console_handler() + # Adding a console handler when one has already been added. + tuf.log.add_console_handler() + + # Expected log levels. + for level in log_levels: + tuf.log.set_console_log_level(level) + + # Test for improperly formatted argument. + self.assertRaises(tuf.FormatError, tuf.log.add_console_handler, '123') + + # Test for invalid argument. + self.assertRaises(tuf.FormatError, tuf.log.add_console_handler, 51) + + try: + raise TypeError('Test exception output in the console.') + + except TypeError as e: + logger.error(e) def test_remove_console_handler(self): - pass + # Normal case. + tuf.log.remove_console_handler() + # Removing a console handler that has not been added. Logs a warning. + tuf.log.remove_console_handler() + # Run unit test. diff --git a/tests/test_mirrors.py b/tests/test_mirrors.py index 67572d9f..4bf8e6a4 100755 --- a/tests/test_mirrors.py +++ b/tests/test_mirrors.py @@ -1,12 +1,14 @@ +#!/usr/bin/env python + """ test_mirrors.py - Konstantin Andrianov + Konstantin Andrianov. - March 26, 2012 + March 26, 2012. See LICENSE for licensing information. @@ -15,7 +17,13 @@ Unit test for 'mirrors.py'. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals import unittest @@ -23,7 +31,7 @@ import tuf.formats as formats import tuf.mirrors as mirrors import tuf.unittest_toolbox as unittest_toolbox - +import tuf._vendor.six as six class TestMirrors(unittest_toolbox.Modified_TestCase): @@ -53,18 +61,18 @@ def setUp(self): def test_get_list_of_mirrors(self): # Test: Normal case. mirror_list = mirrors.get_list_of_mirrors('meta', 'release.txt', self.mirrors) - self.assertEquals(len(mirror_list), 3) - for mirror, mirror_info in self.mirrors.items(): + self.assertEqual(len(mirror_list), 3) + for mirror, mirror_info in six.iteritems(self.mirrors): url = mirror_info['url_prefix']+'/metadata/release.txt' self.assertTrue(url in mirror_list) mirror_list = mirrors.get_list_of_mirrors('target', 'a.txt', self.mirrors) - self.assertEquals(len(mirror_list), 1) + self.assertEqual(len(mirror_list), 1) self.assertTrue(self.mirrors['mirror1']['url_prefix']+'/targets/a.txt' in \ mirror_list) mirror_list = mirrors.get_list_of_mirrors('target', 'a/b', self.mirrors) - self.assertEquals(len(mirror_list), 1) + self.assertEqual(len(mirror_list), 1) self.assertTrue(self.mirrors['mirror1']['url_prefix']+'/targets/a/b' in \ mirror_list) diff --git a/tests/test_mix_and_match_attack.py b/tests/test_mix_and_match_attack.py index 4b1191ed..e44f7927 100755 --- a/tests/test_mix_and_match_attack.py +++ b/tests/test_mix_and_match_attack.py @@ -27,23 +27,30 @@ Note: There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatability, where the print statement is a function, an +# Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals import os -import urllib import tempfile import random import time import shutil import json import subprocess -import unittest import logging +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf.formats import tuf.util @@ -51,6 +58,7 @@ import tuf.client.updater as updater import tuf.repository_tool as repo_tool import tuf.unittest_toolbox as unittest_toolbox +import tuf._vendor.six as six # The repository tool is imported and logs console messages by default. Disable # console log messages generated by this unit test. @@ -88,7 +96,7 @@ def setUpClass(cls): # NOTE: Following error is raised if a delay is not applied: # - time.sleep(.2) + time.sleep(.8) @@ -206,8 +214,8 @@ def test_with_tuf(self): # Modify a 'role1.json' target file, and add it to its metadata so that a # new version is generated. - with open(file3_path, 'wb') as file_object: - file_object.write('update file3') + with open(file3_path, 'wt') as file_object: + file_object.write('This is role2\'s target file.') repository.targets('role1').add_target(file3_path) repository.write() @@ -221,7 +229,7 @@ def test_with_tuf(self): shutil.move(backup_role1, role1_path) # Verify that the TUF client detects unexpected metadata (previously valid, - # but not up-to-date with the latest snashot of the repository) and refuses + # but not up-to-date with the latest snapshot of the repository) and refuses # to continue the update process. # Refresh top-level metadata so that the client is aware of the latest # snapshot of the repository. @@ -232,8 +240,8 @@ def test_with_tuf(self): # Verify that the specific 'tuf.BadHashError' exception is raised by each # mirror. - except tuf.NoWorkingMirrorError, exception: - for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + except tuf.NoWorkingMirrorError as exception: + for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'metadata', 'targets', 'role1.json') diff --git a/tests/test_pycrypto_keys.py b/tests/test_pycrypto_keys.py index f6adf7d0..4ef568ab 100755 --- a/tests/test_pycrypto_keys.py +++ b/tests/test_pycrypto_keys.py @@ -1,9 +1,11 @@ +#!/usr/bin/env python + """ test_pycrypto_keys.py - Vladimir Diaz + Vladimir Diaz October 10, 2013. @@ -15,6 +17,14 @@ Test cases for test_pycrypto_keys.py. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import unittest import logging @@ -56,7 +66,7 @@ def test_generate_rsa_public_and_private(self): def test_create_rsa_signature(self): global private_rsa - data = 'The quick brown fox jumps over the lazy dog' + data = 'The quick brown fox jumps over the lazy dog'.encode('utf-8') signature, method = pycrypto.create_rsa_signature(private_rsa, data) # Verify format of returned values. @@ -65,9 +75,12 @@ def test_create_rsa_signature(self): FORMAT_ERROR_MSG) self.assertEqual('RSASSA-PSS', method) - # Check for improperly formatted argument. + # Check for improperly formatted arguments. self.assertRaises(tuf.FormatError, pycrypto.create_rsa_signature, 123, data) + + self.assertRaises(TypeError, + pycrypto.create_rsa_signature, '', data) # Check for invalid 'data'. self.assertRaises(tuf.CryptoError, @@ -77,7 +90,7 @@ def test_create_rsa_signature(self): def test_verify_rsa_signature(self): global public_rsa global private_rsa - data = 'The quick brown fox jumps over the lazy dog' + data = 'The quick brown fox jumps over the lazy dog'.encode('utf-8') signature, method = pycrypto.create_rsa_signature(private_rsa, data) valid_signature = pycrypto.verify_rsa_signature(signature, method, public_rsa, @@ -94,21 +107,25 @@ def test_verify_rsa_signature(self): self.assertRaises(tuf.FormatError, pycrypto.verify_rsa_signature, 123, method, public_rsa, data) + self.assertRaises(tuf.UnknownMethodError, pycrypto.verify_rsa_signature, + signature, + 'invalid_method', + public_rsa, data) + # Check for invalid signature and data. self.assertRaises(tuf.CryptoError, pycrypto.verify_rsa_signature, signature, method, public_rsa, 123) self.assertEqual(False, pycrypto.verify_rsa_signature(signature, method, - public_rsa, 'mismatched data')) + public_rsa, b'mismatched data')) mismatched_signature, method = pycrypto.create_rsa_signature(private_rsa, - 'mismatched data') + b'mismatched data') self.assertEqual(False, pycrypto.verify_rsa_signature(mismatched_signature, method, public_rsa, data)) - def test_create_rsa_encrypted_pem(self): global public_rsa global private_rsa @@ -134,8 +151,13 @@ def test_create_rsa_encrypted_pem(self): pycrypto.create_rsa_encrypted_pem, 1, passphrase) self.assertRaises(tuf.FormatError, pycrypto.create_rsa_encrypted_pem, private_rsa, ['pw']) - - + + self.assertRaises(tuf.CryptoError, pycrypto.create_rsa_encrypted_pem, + 'abc', passphrase) + self.assertRaises(TypeError, pycrypto.create_rsa_encrypted_pem, '', passphrase) + + + def test_create_rsa_public_and_private_from_encrypted_pem(self): global private_rsa passphrase = 'pw' @@ -190,7 +212,83 @@ def test_create_rsa_public_and_private_from_encrypted_pem(self): self.assertRaises(tuf.CryptoError, pycrypto.create_rsa_public_and_private_from_encrypted_pem, 'invalid_pem', passphrase) - + + + + def test_encrypt_key(self): + # Test for valid arguments. + global public_rsa + global private_rsa + passphrase = 'pw' + + rsa_key = {'keytype': 'rsa', + 'keyid': 'd62247f817883f593cf6c66a5a55292488d457bcf638ae03207dbbba9dbe457d', + 'keyval': {'public': public_rsa, 'private': private_rsa}} + + encrypted_rsa_key = tuf.pycrypto_keys.encrypt_key(rsa_key, passphrase) + + # Test for invalid arguments. + rsa_key['keyval']['private'] = '' + self.assertRaises(tuf.FormatError, tuf.pycrypto_keys.encrypt_key, rsa_key, + 'passphrase') + + + def test_decrypt_key(self): + # Test for valid arguments. + global public_rsa + global private_rsa + passphrase = 'pw' + + rsa_key = {'keytype': 'rsa', + 'keyid': 'd62247f817883f593cf6c66a5a55292488d457bcf638ae03207dbbba9dbe457d', + 'keyval': {'public': public_rsa, 'private': private_rsa}} + + encrypted_rsa_key = tuf.pycrypto_keys.encrypt_key(rsa_key, passphrase).encode('utf-8') + + decrypted_rsa_key = tuf.pycrypto_keys.decrypt_key(encrypted_rsa_key, passphrase) + + + # Test for invalid arguments. + self.assertRaises(tuf.CryptoError, tuf.pycrypto_keys.decrypt_key, b'bad', + passphrase) + + # Test for invalid encrypted content (i.e., invalid hmac and ciphertext.) + encryption_delimiter = tuf.pycrypto_keys._ENCRYPTION_DELIMITER + salt, iterations, hmac, iv, ciphertext = \ + encrypted_rsa_key.decode('utf-8').split(encryption_delimiter) + + # Set an invalid hmac. The decryption routine sould raise a tuf.CryptoError + # exception because 'hmac' does not match the hmac calculated by the + # decryption routine. + bad_hmac = '12345abcd' + invalid_encrypted_rsa_key = \ + salt + encryption_delimiter + iterations + encryption_delimiter + \ + bad_hmac + encryption_delimiter + iv + encryption_delimiter + ciphertext + + self.assertRaises(tuf.CryptoError, tuf.pycrypto_keys.decrypt_key, + invalid_encrypted_rsa_key.encode('utf-8'), passphrase) + + # Test for invalid 'ciphertext' + bad_ciphertext = '12345abcde' + invalid_encrypted_rsa_key = \ + salt + encryption_delimiter + iterations + encryption_delimiter + \ + hmac + encryption_delimiter + iv + encryption_delimiter + bad_ciphertext + + self.assertRaises(tuf.CryptoError, tuf.pycrypto_keys.decrypt_key, + invalid_encrypted_rsa_key.encode('utf-8'), passphrase) + + + + def test__decrypt_key(self): + # Test for invalid arguments. + salt, iterations, derived_key = tuf.pycrypto_keys._generate_derived_key('pw') + derived_key_information = {'salt': salt, 'derived_key': derived_key, + 'iterations': iterations} + + self.assertRaises(tuf.CryptoError, tuf.pycrypto_keys._encrypt, + 8, derived_key_information) + + # Run the unit tests. diff --git a/tests/test_replay_attack.py b/tests/test_replay_attack.py index ddb34d28..aba4b227 100755 --- a/tests/test_replay_attack.py +++ b/tests/test_replay_attack.py @@ -27,15 +27,15 @@ Note: There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatability, where the print statement is a function, an +# Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals import os -import urllib import tempfile import random import time @@ -43,8 +43,15 @@ import shutil import json import subprocess -import unittest import logging +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf.formats import tuf.util @@ -52,6 +59,7 @@ import tuf.client.updater as updater import tuf.repository_tool as repo_tool import tuf.unittest_toolbox as unittest_toolbox +import tuf._vendor.six as six # The repository tool is imported and logs console messages by default. Disable # console log messages generated by this unit test. @@ -89,7 +97,7 @@ def setUpClass(cls): # NOTE: Following error is raised if a delay is not applied: # - time.sleep(.2) + time.sleep(.8) @@ -207,7 +215,7 @@ def test_without_tuf(self): # Set an arbitrary expiration so that the repository tool generates a new # version. - repository.timestamp.expiration = datetime.datetime(2030, 01, 01, 12, 12) + repository.timestamp.expiration = datetime.datetime(2030, 1, 1, 12, 12) repository.write() # Move the staged metadata to the "live" metadata. @@ -225,7 +233,7 @@ def test_without_tuf(self): client_timestamp_path = os.path.join(self.client_directory, 'metadata', 'current', 'timestamp.json') - urllib.urlretrieve(url_file, client_timestamp_path) + six.moves.urllib.request.urlretrieve(url_file, client_timestamp_path) length, hashes = tuf.util.get_file_details(client_timestamp_path) download_fileinfo = tuf.formats.make_fileinfo(length, hashes) @@ -237,7 +245,7 @@ def test_without_tuf(self): # and verify that the non-TUF client downloads it (expected, but not ideal). shutil.move(backup_timestamp, timestamp_path) - urllib.urlretrieve(url_file, client_timestamp_path) + six.moves.urllib.request.urlretrieve(url_file, client_timestamp_path) length, hashes = tuf.util.get_file_details(client_timestamp_path) download_fileinfo = tuf.formats.make_fileinfo(length, hashes) @@ -278,7 +286,7 @@ def test_with_tuf(self): # Set an arbitrary expiration so that the repository tool generates a new # version. - repository.timestamp.expiration = datetime.datetime(2030, 01, 01, 12, 12) + repository.timestamp.expiration = datetime.datetime(2030, 1, 1, 12, 12) repository.write() # Move the staged metadata to the "live" metadata. @@ -314,8 +322,8 @@ def test_with_tuf(self): # Verify that the specific 'tuf.ReplayedMetadataError' is raised by each # mirror. - except tuf.NoWorkingMirrorError, exception: - for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + except tuf.NoWorkingMirrorError as exception: + for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'metadata', 'timestamp.json') diff --git a/tests/test_repository_lib.py b/tests/test_repository_lib.py new file mode 100755 index 00000000..9885e476 --- /dev/null +++ b/tests/test_repository_lib.py @@ -0,0 +1,729 @@ +#!/usr/bin/env python + +""" + + test_repository_lib.py + + + Vladimir Diaz + + + June 1, 2014. + + + See LICENSE for licensing information. + + + Unit test for 'repository_lib.py'. +""" + +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +import os +import time +import datetime +import logging +import tempfile +import json +import shutil +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest + +import tuf +import tuf.log +import tuf.formats +import tuf.roledb +import tuf.keydb +import tuf.hash +import tuf.repository_lib as repo_lib +import tuf._vendor.six as six + +logger = logging.getLogger('tuf.test_repository_lib') + +repo_lib.disable_console_log_messages() + + + +class TestRepositoryToolFunctions(unittest.TestCase): + @classmethod + def setUpClass(cls): + + # setUpClass() is called before tests in an individual class are executed. + + # Create a temporary directory to store the repository, metadata, and target + # files. 'temporary_directory' must be deleted in TearDownClass() so that + # temporary files are always removed, even when exceptions occur. + cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) + + + + @classmethod + def tearDownClass(cls): + + # tearDownModule() is called after all the tests have run. + # http://docs.python.org/2/library/unittest.html#class-and-module-fixtures + + # Remove the temporary repository directory, which should contain all the + # metadata, targets, and key files generated for the test cases. + shutil.rmtree(cls.temporary_directory) + + + + def setUp(self): + pass + + + def tearDown(self): + pass + + + + def test_generate_and_write_rsa_keypair(self): + + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + test_keypath = os.path.join(temporary_directory, 'rsa_key') + + repo_lib.generate_and_write_rsa_keypair(test_keypath, password='pw') + self.assertTrue(os.path.exists(test_keypath)) + self.assertTrue(os.path.exists(test_keypath + '.pub')) + + # Ensure the generated key files are importable. + imported_pubkey = \ + repo_lib.import_rsa_publickey_from_file(test_keypath + '.pub') + self.assertTrue(tuf.formats.RSAKEY_SCHEMA.matches(imported_pubkey)) + + imported_privkey = \ + repo_lib.import_rsa_privatekey_from_file(test_keypath, 'pw') + self.assertTrue(tuf.formats.RSAKEY_SCHEMA.matches(imported_privkey)) + + # Custom 'bits' argument. + os.remove(test_keypath) + os.remove(test_keypath + '.pub') + repo_lib.generate_and_write_rsa_keypair(test_keypath, bits=2048, + password='pw') + self.assertTrue(os.path.exists(test_keypath)) + self.assertTrue(os.path.exists(test_keypath + '.pub')) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, repo_lib.generate_and_write_rsa_keypair, + 3, bits=2048, password='pw') + self.assertRaises(tuf.FormatError, repo_lib.generate_and_write_rsa_keypair, + test_keypath, bits='bad', password='pw') + self.assertRaises(tuf.FormatError, repo_lib.generate_and_write_rsa_keypair, + test_keypath, bits=2048, password=3) + + + # Test invalid 'bits' argument. + self.assertRaises(tuf.FormatError, repo_lib.generate_and_write_rsa_keypair, + test_keypath, bits=1024, password='pw') + + + + def test_import_rsa_privatekey_from_file(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + + # Load one of the pre-generated key files from 'tuf/tests/repository_data'. + # 'password' unlocks the pre-generated key files. + key_filepath = os.path.join('repository_data', 'keystore', + 'root_key') + self.assertTrue(os.path.exists(key_filepath)) + + imported_rsa_key = repo_lib.import_rsa_privatekey_from_file(key_filepath, + 'password') + self.assertTrue(tuf.formats.RSAKEY_SCHEMA.matches(imported_rsa_key)) + + + # Test improperly formatted argument. + self.assertRaises(tuf.FormatError, + repo_lib.import_rsa_privatekey_from_file, 3, 'pw') + + + # Test invalid argument. + # Non-existent key file. + nonexistent_keypath = os.path.join(temporary_directory, + 'nonexistent_keypath') + self.assertRaises(IOError, repo_lib.import_rsa_privatekey_from_file, + nonexistent_keypath, 'pw') + + # Invalid key file argument. + invalid_keyfile = os.path.join(temporary_directory, 'invalid_keyfile') + with open(invalid_keyfile, 'wb') as file_object: + file_object.write(b'bad keyfile') + self.assertRaises(tuf.CryptoError, repo_lib.import_rsa_privatekey_from_file, + invalid_keyfile, 'pw') + + + + def test_import_rsa_publickey_from_file(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + + # Load one of the pre-generated key files from 'tuf/tests/repository_data'. + key_filepath = os.path.join('repository_data', 'keystore', + 'root_key.pub') + self.assertTrue(os.path.exists(key_filepath)) + + imported_rsa_key = repo_lib.import_rsa_publickey_from_file(key_filepath) + self.assertTrue(tuf.formats.RSAKEY_SCHEMA.matches(imported_rsa_key)) + + + # Test improperly formatted argument. + self.assertRaises(tuf.FormatError, + repo_lib.import_rsa_privatekey_from_file, 3) + + + # Test invalid argument. + # Non-existent key file. + nonexistent_keypath = os.path.join(temporary_directory, + 'nonexistent_keypath') + self.assertRaises(IOError, repo_lib.import_rsa_publickey_from_file, + nonexistent_keypath) + + # Invalid key file argument. + invalid_keyfile = os.path.join(temporary_directory, 'invalid_keyfile') + with open(invalid_keyfile, 'wb') as file_object: + file_object.write(b'bad keyfile') + self.assertRaises(tuf.Error, repo_lib.import_rsa_publickey_from_file, + invalid_keyfile) + + + + def test_generate_and_write_ed25519_keypair(self): + + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + test_keypath = os.path.join(temporary_directory, 'ed25519_key') + + repo_lib.generate_and_write_ed25519_keypair(test_keypath, password='pw') + self.assertTrue(os.path.exists(test_keypath)) + self.assertTrue(os.path.exists(test_keypath + '.pub')) + + # Ensure the generated key files are importable. + imported_pubkey = \ + repo_lib.import_ed25519_publickey_from_file(test_keypath + '.pub') + self.assertTrue(tuf.formats.ED25519KEY_SCHEMA.matches(imported_pubkey)) + + imported_privkey = \ + repo_lib.import_ed25519_privatekey_from_file(test_keypath, 'pw') + self.assertTrue(tuf.formats.ED25519KEY_SCHEMA.matches(imported_privkey)) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, + repo_lib.generate_and_write_ed25519_keypair, + 3, password='pw') + self.assertRaises(tuf.FormatError, repo_lib.generate_and_write_rsa_keypair, + test_keypath, password=3) + + + + def test_import_ed25519_publickey_from_file(self): + # Test normal case. + # Generate ed25519 keys that can be imported. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + ed25519_keypath = os.path.join(temporary_directory, 'ed25519_key') + repo_lib.generate_and_write_ed25519_keypair(ed25519_keypath, password='pw') + + imported_ed25519_key = \ + repo_lib.import_ed25519_publickey_from_file(ed25519_keypath + '.pub') + self.assertTrue(tuf.formats.ED25519KEY_SCHEMA.matches(imported_ed25519_key)) + + + # Test improperly formatted argument. + self.assertRaises(tuf.FormatError, + repo_lib.import_ed25519_publickey_from_file, 3) + + + # Test invalid argument. + # Non-existent key file. + nonexistent_keypath = os.path.join(temporary_directory, + 'nonexistent_keypath') + self.assertRaises(IOError, repo_lib.import_ed25519_publickey_from_file, + nonexistent_keypath) + + # Invalid key file argument. + invalid_keyfile = os.path.join(temporary_directory, 'invalid_keyfile') + with open(invalid_keyfile, 'wb') as file_object: + file_object.write(b'bad keyfile') + + self.assertRaises(tuf.Error, repo_lib.import_ed25519_publickey_from_file, + invalid_keyfile) + + # Invalid public key imported (contains unexpected keytype.) + keytype = imported_ed25519_key['keytype'] + keyval = imported_ed25519_key['keyval'] + ed25519key_metadata_format = \ + tuf.keys.format_keyval_to_metadata(keytype, keyval, private=False) + + ed25519key_metadata_format['keytype'] = 'invalid_keytype' + with open(ed25519_keypath + '.pub', 'wb') as file_object: + file_object.write(json.dumps(ed25519key_metadata_format).encode('utf-8')) + + self.assertRaises(tuf.FormatError, + repo_lib.import_ed25519_publickey_from_file, + ed25519_keypath + '.pub') + + + + def test_import_ed25519_privatekey_from_file(self): + # Test normal case. + # Generate ed25519 keys that can be imported. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + ed25519_keypath = os.path.join(temporary_directory, 'ed25519_key') + repo_lib.generate_and_write_ed25519_keypair(ed25519_keypath, password='pw') + + imported_ed25519_key = \ + repo_lib.import_ed25519_privatekey_from_file(ed25519_keypath, 'pw') + self.assertTrue(tuf.formats.ED25519KEY_SCHEMA.matches(imported_ed25519_key)) + + + # Test improperly formatted argument. + self.assertRaises(tuf.FormatError, + repo_lib.import_ed25519_privatekey_from_file, 3, 'pw') + + + # Test invalid argument. + # Non-existent key file. + nonexistent_keypath = os.path.join(temporary_directory, + 'nonexistent_keypath') + self.assertRaises(IOError, repo_lib.import_ed25519_privatekey_from_file, + nonexistent_keypath, 'pw') + + # Invalid key file argument. + invalid_keyfile = os.path.join(temporary_directory, 'invalid_keyfile') + with open(invalid_keyfile, 'wb') as file_object: + file_object.write(b'bad keyfile') + + self.assertRaises(tuf.Error, repo_lib.import_ed25519_privatekey_from_file, + invalid_keyfile, 'pw') + + # Invalid private key imported (contains unexpected keytype.) + imported_ed25519_key['keytype'] = 'invalid_keytype' + + # Use 'pycrypto_keys.py' to bypass the key format validation performed by + # 'keys.py'. + salt, iterations, derived_key = \ + tuf.pycrypto_keys._generate_derived_key('pw') + + # Store the derived key info in a dictionary, the object expected + # by the non-public _encrypt() routine. + derived_key_information = {'salt': salt, 'iterations': iterations, + 'derived_key': derived_key} + + # Convert the key object to json string format and encrypt it with the + # derived key. + encrypted_key = \ + tuf.pycrypto_keys._encrypt(json.dumps(imported_ed25519_key), + derived_key_information) + + with open(ed25519_keypath, 'wb') as file_object: + file_object.write(encrypted_key.encode('utf-8')) + + self.assertRaises(tuf.FormatError, + repo_lib.import_ed25519_privatekey_from_file, + ed25519_keypath, 'pw') + + + + def test_get_metadata_filenames(self): + + # Test normal case. + metadata_directory = os.path.join('metadata/') + filenames = {'root.json': metadata_directory + 'root.json', + 'targets.json': metadata_directory + 'targets.json', + 'snapshot.json': metadata_directory + 'snapshot.json', + 'timestamp.json': metadata_directory + 'timestamp.json'} + + self.assertEqual(filenames, repo_lib.get_metadata_filenames('metadata/')) + + # If a directory argument is not specified, the current working directory + # is used. + metadata_directory = os.getcwd() + filenames = {'root.json': os.path.join(metadata_directory, 'root.json'), + 'targets.json': os.path.join(metadata_directory, 'targets.json'), + 'snapshot.json': os.path.join(metadata_directory, 'snapshot.json'), + 'timestamp.json': os.path.join(metadata_directory, 'timestamp.json')} + self.assertEqual(filenames, repo_lib.get_metadata_filenames()) + + + # Test improperly formatted argument. + self.assertRaises(tuf.FormatError, repo_lib.get_metadata_filenames, 3) + + + + def test_get_metadata_fileinfo(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + test_filepath = os.path.join(temporary_directory, 'file.txt') + + with open(test_filepath, 'wt') as file_object: + file_object.write('test file') + + # Generate test fileinfo object. It is assumed SHA256 hashes are computed + # by get_metadata_fileinfo(). + file_length = os.path.getsize(test_filepath) + digest_object = tuf.hash.digest_filename(test_filepath) + file_hashes = {'sha256': digest_object.hexdigest()} + fileinfo = {'length': file_length, 'hashes': file_hashes} + self.assertTrue(tuf.formats.FILEINFO_SCHEMA.matches(fileinfo)) + + self.assertEqual(fileinfo, repo_lib.get_metadata_fileinfo(test_filepath)) + + + # Test improperly formatted argument. + self.assertRaises(tuf.FormatError, repo_lib.get_metadata_fileinfo, 3) + + + # Test non-existent file. + nonexistent_filepath = os.path.join(temporary_directory, 'oops.txt') + self.assertRaises(tuf.Error, repo_lib.get_metadata_fileinfo, + nonexistent_filepath) + + + + def test_get_target_hash(self): + # Test normal case. + expected_target_hashes = { + '/file1.txt': 'e3a3d89eb3b70ce3fbce6017d7b8c12d4abd5635427a0e8a238f53157df85b3d', + '/README.txt': '8faee106f1bb69f34aaf1df1e3c2e87d763c4d878cb96b91db13495e32ceb0b0', + '/packages/file2.txt': 'c9c4a5cdd84858dd6a23d98d7e6e6b2aec45034946c16b2200bc317c75415e92' + } + for filepath, target_hash in six.iteritems(expected_target_hashes): + self.assertTrue(tuf.formats.RELPATH_SCHEMA.matches(filepath)) + self.assertTrue(tuf.formats.HASH_SCHEMA.matches(target_hash)) + self.assertEqual(repo_lib.get_target_hash(filepath), target_hash) + + # Test for improperly formatted argument. + self.assertRaises(tuf.FormatError, repo_lib.get_target_hash, 8) + + + + def test_generate_root_metadata(self): + # Test normal case. + # Load the root metadata provided in 'tuf/tests/repository_data/'. + root_filepath = os.path.join('repository_data', 'repository', + 'metadata', 'root.json') + root_signable = tuf.util.load_json_file(root_filepath) + + # generate_root_metadata() expects the top-level roles and keys to be + # available in 'tuf.keydb' and 'tuf.roledb'. + tuf.roledb.create_roledb_from_root_metadata(root_signable['signed']) + tuf.keydb.create_keydb_from_root_metadata(root_signable['signed']) + expires = '1985-10-21T01:22:00Z' + + root_metadata = repo_lib.generate_root_metadata(1, expires, + consistent_snapshot=False) + self.assertTrue(tuf.formats.ROOT_SCHEMA.matches(root_metadata)) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, repo_lib.generate_root_metadata, + '3', expires, False) + self.assertRaises(tuf.FormatError, repo_lib.generate_root_metadata, + 1, '3', False) + self.assertRaises(tuf.FormatError, repo_lib.generate_root_metadata, + 1, expires, 3) + + # Test for missing required roles and keys. + tuf.roledb.clear_roledb() + tuf.keydb.clear_keydb() + self.assertRaises(tuf.Error, repo_lib.generate_root_metadata, + 1, expires, False) + + + + def test_generate_targets_metadata(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + targets_directory = os.path.join(temporary_directory, 'targets') + file1_path = os.path.join(targets_directory, 'file.txt') + tuf.util.ensure_parent_dir(file1_path) + + with open(file1_path, 'wt') as file_object: + file_object.write('test file.') + + # Set valid generate_targets_metadata() arguments. + version = 1 + datetime_object = datetime.datetime(2030, 1, 1, 12, 0) + expiration_date = datetime_object.isoformat() + 'Z' + target_files = ['file.txt'] + + delegations = {"keys": { + "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf": { + "keytype": "ed25519", + "keyval": { + "public": "3eb81026ded5af2c61fb3d4b272ac53cd1049a810ee88f4df1fc35cdaf918157" + } + } + }, + "roles": [ + { + "keyids": [ + "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf" + ], + "name": "targets/warehouse", + "paths": [ + "/file1.txt", "/README.txt", '/warehouse/' + ], + "threshold": 1 + } + ] + } + + targets_metadata = \ + repo_lib.generate_targets_metadata(targets_directory, target_files, + version, expiration_date, delegations, + False) + self.assertTrue(tuf.formats.TARGETS_SCHEMA.matches(targets_metadata)) + + # Verify that 'digest.filename' file is saved to 'targets_directory' if + # the 'write_consistent_targets' argument is True. + list_targets_directory = os.listdir(targets_directory) + targets_metadata = \ + repo_lib.generate_targets_metadata(targets_directory, target_files, + version, expiration_date, delegations, + write_consistent_targets=True) + new_list_targets_directory = os.listdir(targets_directory) + + # Verify that 'targets_directory' contains only one extra item. + self.assertTrue(len(list_targets_directory) + 1, + len(new_list_targets_directory)) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, repo_lib.generate_targets_metadata, + 3, target_files, version, expiration_date) + self.assertRaises(tuf.FormatError, repo_lib.generate_targets_metadata, + targets_directory, 3, version, expiration_date) + self.assertRaises(tuf.FormatError, repo_lib.generate_targets_metadata, + targets_directory, target_files, '3', expiration_date) + self.assertRaises(tuf.FormatError, repo_lib.generate_targets_metadata, + targets_directory, target_files, version, '3') + + # Improperly formatted 'delegations' and 'write_consistent_targets' + self.assertRaises(tuf.FormatError, repo_lib.generate_targets_metadata, + targets_directory, target_files, version, expiration_date, + 3, False) + self.assertRaises(tuf.FormatError, repo_lib.generate_targets_metadata, + targets_directory, target_files, version, expiration_date, + delegations, 3) + + + # Test invalid 'target_files' argument. + self.assertRaises(tuf.Error, repo_lib.generate_targets_metadata, + targets_directory, ['nonexistent_file.txt'], version, + expiration_date) + + + + + def test_generate_snapshot_metadata(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + original_repository_path = os.path.join('repository_data', + 'repository') + repository_directory = os.path.join(temporary_directory, 'repository') + shutil.copytree(original_repository_path, repository_directory) + metadata_directory = os.path.join(repository_directory, + repo_lib.METADATA_STAGED_DIRECTORY_NAME) + root_filename = os.path.join(metadata_directory, repo_lib.ROOT_FILENAME) + targets_filename = os.path.join(metadata_directory, + repo_lib.TARGETS_FILENAME) + version = 1 + expiration_date = '1985-10-21T13:20:00Z' + + snapshot_metadata = \ + repo_lib.generate_snapshot_metadata(metadata_directory, version, + expiration_date, root_filename, + targets_filename, + consistent_snapshot=False) + self.assertTrue(tuf.formats.SNAPSHOT_SCHEMA.matches(snapshot_metadata)) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, repo_lib.generate_snapshot_metadata, + 3, version, expiration_date, + root_filename, targets_filename, consistent_snapshot=False) + self.assertRaises(tuf.FormatError, repo_lib.generate_snapshot_metadata, + metadata_directory, '3', expiration_date, + root_filename, targets_filename, consistent_snapshot=False) + self.assertRaises(tuf.FormatError, repo_lib.generate_snapshot_metadata, + metadata_directory, version, '3', + root_filename, targets_filename, consistent_snapshot=False) + self.assertRaises(tuf.FormatError, repo_lib.generate_snapshot_metadata, + metadata_directory, version, expiration_date, + 3, targets_filename, consistent_snapshot=False) + self.assertRaises(tuf.FormatError, repo_lib.generate_snapshot_metadata, + metadata_directory, version, expiration_date, + root_filename, 3, consistent_snapshot=False) + self.assertRaises(tuf.FormatError, repo_lib.generate_snapshot_metadata, + metadata_directory, version, expiration_date, + root_filename, targets_filename, 3) + + + + def test_generate_timestamp_metadata(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + original_repository_path = os.path.join('repository_data', + 'repository') + repository_directory = os.path.join(temporary_directory, 'repository') + shutil.copytree(original_repository_path, repository_directory) + metadata_directory = os.path.join(repository_directory, + repo_lib.METADATA_STAGED_DIRECTORY_NAME) + snapshot_filename = os.path.join(metadata_directory, + repo_lib.SNAPSHOT_FILENAME) + + # Set valid generate_timestamp_metadata() arguments. + version = 1 + expiration_date = '1985-10-21T13:20:00Z' + + compressions = ['gz'] + + snapshot_metadata = \ + repo_lib.generate_timestamp_metadata(snapshot_filename, version, + expiration_date, compressions) + self.assertTrue(tuf.formats.TIMESTAMP_SCHEMA.matches(snapshot_metadata)) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, repo_lib.generate_timestamp_metadata, + 3, version, expiration_date, compressions) + self.assertRaises(tuf.FormatError, repo_lib.generate_timestamp_metadata, + snapshot_filename, '3', expiration_date, compressions) + self.assertRaises(tuf.FormatError, repo_lib.generate_timestamp_metadata, + snapshot_filename, version, '3', compressions) + self.assertRaises(tuf.FormatError, repo_lib.generate_timestamp_metadata, + snapshot_filename, version, expiration_date, 3) + self.assertRaises(tuf.FormatError, repo_lib.generate_timestamp_metadata, + snapshot_filename, version, expiration_date, ['compress']) + + + + + def test_sign_metadata(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + metadata_path = os.path.join('repository_data', + 'repository', 'metadata') + keystore_path = os.path.join('repository_data', + 'keystore') + root_filename = os.path.join(metadata_path, 'root.json') + root_metadata = tuf.util.load_json_file(root_filename)['signed'] + + tuf.keydb.create_keydb_from_root_metadata(root_metadata) + tuf.roledb.create_roledb_from_root_metadata(root_metadata) + root_keyids = tuf.roledb.get_role_keyids('root') + + root_private_keypath = os.path.join(keystore_path, 'root_key') + root_private_key = \ + repo_lib.import_rsa_privatekey_from_file(root_private_keypath, + 'password') + + # sign_metadata() expects the private key 'root_metadata' to be in + # 'tuf.keydb'. Remove any public keys that may be loaded before + # adding private key, otherwise a 'tuf.KeyAlreadyExists' exception is + # raised. + tuf.keydb.remove_key(root_private_key['keyid']) + tuf.keydb.add_key(root_private_key) + + root_signable = repo_lib.sign_metadata(root_metadata, root_keyids, + root_filename) + self.assertTrue(tuf.formats.SIGNABLE_SCHEMA.matches(root_signable)) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, repo_lib.sign_metadata, 3, root_keyids, + 'root.json') + self.assertRaises(tuf.FormatError, repo_lib.sign_metadata, root_metadata, + 3, 'root.json') + self.assertRaises(tuf.FormatError, repo_lib.sign_metadata, root_metadata, + root_keyids, 3) + + + + def test_write_metadata_file(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + metadata_directory = os.path.join('repository_data', + 'repository', 'metadata') + root_filename = os.path.join(metadata_directory, 'root.json') + root_signable = tuf.util.load_json_file(root_filename) + + output_filename = os.path.join(temporary_directory, 'root.json') + compressions = ['gz'] + + self.assertFalse(os.path.exists(output_filename)) + repo_lib.write_metadata_file(root_signable, output_filename, compressions, + consistent_snapshot=False) + self.assertTrue(os.path.exists(output_filename)) + self.assertTrue(os.path.exists(output_filename + '.gz')) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, repo_lib.write_metadata_file, + 3, output_filename, compressions, False) + self.assertRaises(tuf.FormatError, repo_lib.write_metadata_file, + root_signable, 3, compressions, False) + self.assertRaises(tuf.FormatError, repo_lib.write_metadata_file, + root_signable, output_filename, 3, False) + self.assertRaises(tuf.FormatError, repo_lib.write_metadata_file, + root_signable, output_filename, compressions, 3) + + + + def test_create_tuf_client_directory(self): + # Test normal case. + temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) + repository_directory = os.path.join('repository_data', + 'repository') + client_directory = os.path.join(temporary_directory, 'client') + + repo_lib.create_tuf_client_directory(repository_directory, client_directory) + + self.assertTrue(os.path.exists(client_directory)) + metadata_directory = os.path.join(client_directory, 'metadata') + current_directory = os.path.join(metadata_directory, 'current') + previous_directory = os.path.join(metadata_directory, 'previous') + self.assertTrue(os.path.exists(client_directory)) + self.assertTrue(os.path.exists(metadata_directory)) + self.assertTrue(os.path.exists(current_directory)) + self.assertTrue(os.path.exists(previous_directory)) + + + # Test improperly formatted arguments. + self.assertRaises(tuf.FormatError, repo_lib.create_tuf_client_directory, + 3, client_directory) + self.assertRaises(tuf.FormatError, repo_lib.create_tuf_client_directory, + repository_directory, 3) + + + # Test invalid argument (i.e., client directory already exists.) + self.assertRaises(tuf.RepositoryError, repo_lib.create_tuf_client_directory, + repository_directory, client_directory) + + + def test__check_directory(self): + # Test for non-existent directory. + self.assertRaises(tuf.Error, repo_lib._check_directory, 'non-existent') + + + +# Run the test cases. +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_repository_tool.py b/tests/test_repository_tool.py index c104640e..17db845e 100755 --- a/tests/test_repository_tool.py +++ b/tests/test_repository_tool.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_repository_tool.py @@ -15,6 +17,14 @@ Unit test for 'repository_tool.py'. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os import time import datetime @@ -22,6 +32,14 @@ import logging import tempfile import shutil +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf import tuf.log @@ -30,6 +48,7 @@ import tuf.keydb import tuf.hash import tuf.repository_tool as repo_tool +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_repository_tool') @@ -222,11 +241,14 @@ def test_write_and_write_partial(self): # Verify that an exception is *not* raised for multiple repository.write(). repository.write() + # Verify the status() does not raise an exception. + repository.status() + # Verify that a write() fails if a repository is loaded and a change # is made to a role. repo_tool.load_repository(repository_directory) - repository.timestamp.expiration = datetime.datetime(2030, 01, 01, 12, 00) + repository.timestamp.expiration = datetime.datetime(2030, 1, 1, 12, 0) self.assertRaises(tuf.UnsignedMetadataError, repository.write) # Verify that a write_partial() is allowed. @@ -370,7 +392,7 @@ def test_expiration(self): self.assertTrue(isinstance(expiration, datetime.datetime)) # Test expiration setter. - self.metadata.expiration = datetime.datetime(2030, 01, 01, 12, 00) + self.metadata.expiration = datetime.datetime(2030, 1, 1, 12, 0) expiration = self.metadata.expiration self.assertTrue(isinstance(expiration, datetime.datetime)) @@ -856,7 +878,8 @@ def test_delegations(self): threshold = 1 self.targets_object.delegate(rolename, public_keys, list_of_targets, - threshold, restricted_paths=None, + threshold, backtrack=True, + restricted_paths=None, path_hash_prefixes=None) # Test that a valid Targets() object is returned by delegations(). @@ -990,8 +1013,9 @@ def test_delegate(self): path_hash_prefixes = ['e3a3', '8fae', 'd543'] self.targets_object.delegate(rolename, public_keys, list_of_targets, - threshold, restricted_paths, - path_hash_prefixes) + threshold, backtrack=True, + restricted_paths=restricted_paths, + path_hash_prefixes=path_hash_prefixes) self.assertEqual(self.targets_object.get_delegated_rolenames(), ['targets/tuf']) @@ -1356,594 +1380,6 @@ def test_load_repository(self): - def test_generate_and_write_rsa_keypair(self): - - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - test_keypath = os.path.join(temporary_directory, 'rsa_key') - - repo_tool.generate_and_write_rsa_keypair(test_keypath, password='pw') - self.assertTrue(os.path.exists(test_keypath)) - self.assertTrue(os.path.exists(test_keypath + '.pub')) - - # Ensure the generated key files are importable. - imported_pubkey = \ - repo_tool.import_rsa_publickey_from_file(test_keypath + '.pub') - self.assertTrue(tuf.formats.RSAKEY_SCHEMA.matches(imported_pubkey)) - - imported_privkey = \ - repo_tool.import_rsa_privatekey_from_file(test_keypath, 'pw') - self.assertTrue(tuf.formats.RSAKEY_SCHEMA.matches(imported_privkey)) - - # Custom 'bits' argument. - os.remove(test_keypath) - os.remove(test_keypath + '.pub') - repo_tool.generate_and_write_rsa_keypair(test_keypath, bits=2048, - password='pw') - self.assertTrue(os.path.exists(test_keypath)) - self.assertTrue(os.path.exists(test_keypath + '.pub')) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, repo_tool.generate_and_write_rsa_keypair, - 3, bits=2048, password='pw') - self.assertRaises(tuf.FormatError, repo_tool.generate_and_write_rsa_keypair, - test_keypath, bits='bad', password='pw') - self.assertRaises(tuf.FormatError, repo_tool.generate_and_write_rsa_keypair, - test_keypath, bits=2048, password=3) - - - # Test invalid 'bits' argument. - self.assertRaises(tuf.FormatError, repo_tool.generate_and_write_rsa_keypair, - test_keypath, bits=1024, password='pw') - - - - def test_import_rsa_privatekey_from_file(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - - # Load one of the pre-generated key files from 'tuf/tests/repository_data'. - # 'password' unlocks the pre-generated key files. - key_filepath = os.path.join('repository_data', 'keystore', - 'root_key') - self.assertTrue(os.path.exists(key_filepath)) - - imported_rsa_key = repo_tool.import_rsa_privatekey_from_file(key_filepath, - 'password') - self.assertTrue(tuf.formats.RSAKEY_SCHEMA.matches(imported_rsa_key)) - - - # Test improperly formatted argument. - self.assertRaises(tuf.FormatError, - repo_tool.import_rsa_privatekey_from_file, 3, 'pw') - - - # Test invalid argument. - # Non-existent key file. - nonexistent_keypath = os.path.join(temporary_directory, - 'nonexistent_keypath') - self.assertRaises(IOError, repo_tool.import_rsa_privatekey_from_file, - nonexistent_keypath, 'pw') - - # Invalid key file argument. - invalid_keyfile = os.path.join(temporary_directory, 'invalid_keyfile') - with open(invalid_keyfile, 'wb') as file_object: - file_object.write('bad keyfile') - self.assertRaises(tuf.CryptoError, repo_tool.import_rsa_privatekey_from_file, - invalid_keyfile, 'pw') - - - - def test_import_rsa_publickey_from_file(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - - # Load one of the pre-generated key files from 'tuf/tests/repository_data'. - key_filepath = os.path.join('repository_data', 'keystore', - 'root_key.pub') - self.assertTrue(os.path.exists(key_filepath)) - - imported_rsa_key = repo_tool.import_rsa_publickey_from_file(key_filepath) - self.assertTrue(tuf.formats.RSAKEY_SCHEMA.matches(imported_rsa_key)) - - - # Test improperly formatted argument. - self.assertRaises(tuf.FormatError, - repo_tool.import_rsa_privatekey_from_file, 3) - - - # Test invalid argument. - # Non-existent key file. - nonexistent_keypath = os.path.join(temporary_directory, - 'nonexistent_keypath') - self.assertRaises(IOError, repo_tool.import_rsa_publickey_from_file, - nonexistent_keypath) - - # Invalid key file argument. - invalid_keyfile = os.path.join(temporary_directory, 'invalid_keyfile') - with open(invalid_keyfile, 'wb') as file_object: - file_object.write('bad keyfile') - self.assertRaises(tuf.Error, repo_tool.import_rsa_publickey_from_file, - invalid_keyfile) - - - - def test_generate_and_write_ed25519_keypair(self): - - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - test_keypath = os.path.join(temporary_directory, 'ed25519_key') - - repo_tool.generate_and_write_ed25519_keypair(test_keypath, password='pw') - self.assertTrue(os.path.exists(test_keypath)) - self.assertTrue(os.path.exists(test_keypath + '.pub')) - - # Ensure the generated key files are importable. - imported_pubkey = \ - repo_tool.import_ed25519_publickey_from_file(test_keypath + '.pub') - self.assertTrue(tuf.formats.ED25519KEY_SCHEMA.matches(imported_pubkey)) - - imported_privkey = \ - repo_tool.import_ed25519_privatekey_from_file(test_keypath, 'pw') - self.assertTrue(tuf.formats.ED25519KEY_SCHEMA.matches(imported_privkey)) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, - repo_tool.generate_and_write_ed25519_keypair, - 3, password='pw') - self.assertRaises(tuf.FormatError, repo_tool.generate_and_write_rsa_keypair, - test_keypath, password=3) - - - - def test_import_ed25519_publickey_from_file(self): - # Test normal case. - # Generate ed25519 keys that can be imported. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - ed25519_keypath = os.path.join(temporary_directory, 'ed25519_key') - repo_tool.generate_and_write_ed25519_keypair(ed25519_keypath, password='pw') - - imported_ed25519_key = \ - repo_tool.import_ed25519_publickey_from_file(ed25519_keypath + '.pub') - self.assertTrue(tuf.formats.ED25519KEY_SCHEMA.matches(imported_ed25519_key)) - - - # Test improperly formatted argument. - self.assertRaises(tuf.FormatError, - repo_tool.import_ed25519_publickey_from_file, 3) - - - # Test invalid argument. - # Non-existent key file. - nonexistent_keypath = os.path.join(temporary_directory, - 'nonexistent_keypath') - self.assertRaises(IOError, repo_tool.import_ed25519_publickey_from_file, - nonexistent_keypath) - - # Invalid key file argument. - invalid_keyfile = os.path.join(temporary_directory, 'invalid_keyfile') - with open(invalid_keyfile, 'wb') as file_object: - file_object.write('bad keyfile') - - self.assertRaises(tuf.Error, repo_tool.import_ed25519_publickey_from_file, - invalid_keyfile) - - - - def test_import_ed25519_privatekey_from_file(self): - # Test normal case. - # Generate ed25519 keys that can be imported. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - ed25519_keypath = os.path.join(temporary_directory, 'ed25519_key') - repo_tool.generate_and_write_ed25519_keypair(ed25519_keypath, password='pw') - - imported_ed25519_key = \ - repo_tool.import_ed25519_privatekey_from_file(ed25519_keypath, 'pw') - self.assertTrue(tuf.formats.ED25519KEY_SCHEMA.matches(imported_ed25519_key)) - - - # Test improperly formatted argument. - self.assertRaises(tuf.FormatError, - repo_tool.import_ed25519_privatekey_from_file, 3, 'pw') - - - # Test invalid argument. - # Non-existent key file. - nonexistent_keypath = os.path.join(temporary_directory, - 'nonexistent_keypath') - self.assertRaises(IOError, repo_tool.import_ed25519_privatekey_from_file, - nonexistent_keypath, 'pw') - - # Invalid key file argument. - invalid_keyfile = os.path.join(temporary_directory, 'invalid_keyfile') - with open(invalid_keyfile, 'wb') as file_object: - file_object.write('bad keyfile') - - self.assertRaises(tuf.Error, repo_tool.import_ed25519_privatekey_from_file, - invalid_keyfile, 'pw') - - - - def test_get_metadata_filenames(self): - - # Test normal case. - metadata_directory = os.path.join('metadata/') - filenames = {'root.json': metadata_directory + 'root.json', - 'targets.json': metadata_directory + 'targets.json', - 'snapshot.json': metadata_directory + 'snapshot.json', - 'timestamp.json': metadata_directory + 'timestamp.json'} - - self.assertEqual(filenames, repo_tool.get_metadata_filenames('metadata/')) - - # If a directory argument is not specified, the current working directory - # is used. - metadata_directory = os.getcwd() - filenames = {'root.json': os.path.join(metadata_directory, 'root.json'), - 'targets.json': os.path.join(metadata_directory, 'targets.json'), - 'snapshot.json': os.path.join(metadata_directory, 'snapshot.json'), - 'timestamp.json': os.path.join(metadata_directory, 'timestamp.json')} - self.assertEqual(filenames, repo_tool.get_metadata_filenames()) - - - # Test improperly formatted argument. - self.assertRaises(tuf.FormatError, repo_tool.get_metadata_filenames, 3) - - - - def test_get_metadata_fileinfo(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - test_filepath = os.path.join(temporary_directory, 'file.txt') - - with open(test_filepath, 'wb') as file_object: - file_object.write('test file') - - # Generate test fileinfo object. It is assumed SHA256 hashes are computed - # by get_metadata_fileinfo(). - file_length = os.path.getsize(test_filepath) - digest_object = tuf.hash.digest_filename(test_filepath) - file_hashes = {'sha256': digest_object.hexdigest()} - fileinfo = {'length': file_length, 'hashes': file_hashes} - self.assertTrue(tuf.formats.FILEINFO_SCHEMA.matches(fileinfo)) - - self.assertEqual(fileinfo, repo_tool.get_metadata_fileinfo(test_filepath)) - - - # Test improperly formatted argument. - self.assertRaises(tuf.FormatError, repo_tool.get_metadata_fileinfo, 3) - - - # Test non-existent file. - nonexistent_filepath = os.path.join(temporary_directory, 'oops.txt') - self.assertRaises(tuf.Error, repo_tool.get_metadata_fileinfo, - nonexistent_filepath) - - - - def test_get_target_hash(self): - # Test normal case. - expected_target_hashes = { - '/file1.txt': 'e3a3d89eb3b70ce3fbce6017d7b8c12d4abd5635427a0e8a238f53157df85b3d', - '/README.txt': '8faee106f1bb69f34aaf1df1e3c2e87d763c4d878cb96b91db13495e32ceb0b0', - '/packages/file2.txt': 'c9c4a5cdd84858dd6a23d98d7e6e6b2aec45034946c16b2200bc317c75415e92' - } - for filepath, target_hash in expected_target_hashes.items(): - self.assertTrue(tuf.formats.RELPATH_SCHEMA.matches(filepath)) - self.assertTrue(tuf.formats.HASH_SCHEMA.matches(target_hash)) - self.assertEqual(repo_tool.get_target_hash(filepath), target_hash) - - # Test for improperly formatted argument. - self.assertRaises(tuf.FormatError, repo_tool.get_target_hash, 8) - - - - def test_generate_root_metadata(self): - # Test normal case. - # Load the root metadata provided in 'tuf/tests/repository_data/'. - root_filepath = os.path.join('repository_data', 'repository', - 'metadata', 'root.json') - root_signable = tuf.util.load_json_file(root_filepath) - - # generate_root_metadata() expects the top-level roles and keys to be - # available in 'tuf.keydb' and 'tuf.roledb'. - tuf.roledb.create_roledb_from_root_metadata(root_signable['signed']) - tuf.keydb.create_keydb_from_root_metadata(root_signable['signed']) - expires = '1985-10-21T01:22:00Z' - - root_metadata = repo_tool.generate_root_metadata(1, expires, - consistent_snapshot=False) - self.assertTrue(tuf.formats.ROOT_SCHEMA.matches(root_metadata)) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, repo_tool.generate_root_metadata, - '3', expires, False) - self.assertRaises(tuf.FormatError, repo_tool.generate_root_metadata, - 1, '3', False) - self.assertRaises(tuf.FormatError, repo_tool.generate_root_metadata, - 1, expires, 3) - - # Test for missing required roles and keys. - tuf.roledb.clear_roledb() - tuf.keydb.clear_keydb() - self.assertRaises(tuf.Error, repo_tool.generate_root_metadata, - 1, expires, False) - - - - def test_generate_targets_metadata(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - targets_directory = os.path.join(temporary_directory, 'targets') - file1_path = os.path.join(targets_directory, 'file.txt') - tuf.util.ensure_parent_dir(file1_path) - - with open(file1_path, 'wb') as file_object: - file_object.write('test file.') - - # Set valid generate_targets_metadata() arguments. - version = 1 - datetime_object = datetime.datetime(2030, 01, 01, 12, 00) - expiration_date = datetime_object.isoformat() + 'Z' - target_files = ['file.txt'] - - delegations = {"keys": { - "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf": { - "keytype": "ed25519", - "keyval": { - "public": "3eb81026ded5af2c61fb3d4b272ac53cd1049a810ee88f4df1fc35cdaf918157" - } - } - }, - "roles": [ - { - "keyids": [ - "a394c28384648328b16731f81440d72243c77bb44c07c040be99347f0df7d7bf" - ], - "name": "targets/warehouse", - "paths": [ - "/file1.txt", "/README.txt", '/warehouse/' - ], - "threshold": 1 - } - ] - } - - targets_metadata = \ - repo_tool.generate_targets_metadata(targets_directory, target_files, - version, expiration_date, delegations, - False) - self.assertTrue(tuf.formats.TARGETS_SCHEMA.matches(targets_metadata)) - - # Verify that 'digest.filename' file is saved to 'targets_directory' if - # the 'write_consistent_targets' argument is True. - list_targets_directory = os.listdir(targets_directory) - targets_metadata = \ - repo_tool.generate_targets_metadata(targets_directory, target_files, - version, expiration_date, delegations, - write_consistent_targets=True) - new_list_targets_directory = os.listdir(targets_directory) - - # Verify that 'targets_directory' contains only one extra item. - self.assertTrue(len(list_targets_directory) + 1, - len(new_list_targets_directory)) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, repo_tool.generate_targets_metadata, - 3, target_files, version, expiration_date) - self.assertRaises(tuf.FormatError, repo_tool.generate_targets_metadata, - targets_directory, 3, version, expiration_date) - self.assertRaises(tuf.FormatError, repo_tool.generate_targets_metadata, - targets_directory, target_files, '3', expiration_date) - self.assertRaises(tuf.FormatError, repo_tool.generate_targets_metadata, - targets_directory, target_files, version, '3') - - # Improperly formatted 'delegations' and 'write_consistent_targets' - self.assertRaises(tuf.FormatError, repo_tool.generate_targets_metadata, - targets_directory, target_files, version, expiration_date, - 3, False) - self.assertRaises(tuf.FormatError, repo_tool.generate_targets_metadata, - targets_directory, target_files, version, expiration_date, - delegations, 3) - - - # Test invalid 'target_files' argument. - self.assertRaises(tuf.Error, repo_tool.generate_targets_metadata, - targets_directory, ['nonexistent_file.txt'], version, - expiration_date) - - - - - def test_generate_snapshot_metadata(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - original_repository_path = os.path.join('repository_data', - 'repository') - repository_directory = os.path.join(temporary_directory, 'repository') - shutil.copytree(original_repository_path, repository_directory) - metadata_directory = os.path.join(repository_directory, - repo_tool.METADATA_STAGED_DIRECTORY_NAME) - root_filename = os.path.join(metadata_directory, repo_tool.ROOT_FILENAME) - targets_filename = os.path.join(metadata_directory, - repo_tool.TARGETS_FILENAME) - version = 1 - expiration_date = '1985-10-21T13:20:00Z' - - snapshot_metadata = \ - repo_tool.generate_snapshot_metadata(metadata_directory, version, - expiration_date, root_filename, - targets_filename, - consistent_snapshot=False) - self.assertTrue(tuf.formats.SNAPSHOT_SCHEMA.matches(snapshot_metadata)) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, repo_tool.generate_snapshot_metadata, - 3, version, expiration_date, - root_filename, targets_filename, consistent_snapshot=False) - self.assertRaises(tuf.FormatError, repo_tool.generate_snapshot_metadata, - metadata_directory, '3', expiration_date, - root_filename, targets_filename, consistent_snapshot=False) - self.assertRaises(tuf.FormatError, repo_tool.generate_snapshot_metadata, - metadata_directory, version, '3', - root_filename, targets_filename, consistent_snapshot=False) - self.assertRaises(tuf.FormatError, repo_tool.generate_snapshot_metadata, - metadata_directory, version, expiration_date, - 3, targets_filename, consistent_snapshot=False) - self.assertRaises(tuf.FormatError, repo_tool.generate_snapshot_metadata, - metadata_directory, version, expiration_date, - root_filename, 3, consistent_snapshot=False) - self.assertRaises(tuf.FormatError, repo_tool.generate_snapshot_metadata, - metadata_directory, version, expiration_date, - root_filename, targets_filename, 3) - - - - def test_generate_timestamp_metadata(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - original_repository_path = os.path.join('repository_data', - 'repository') - repository_directory = os.path.join(temporary_directory, 'repository') - shutil.copytree(original_repository_path, repository_directory) - metadata_directory = os.path.join(repository_directory, - repo_tool.METADATA_STAGED_DIRECTORY_NAME) - snapshot_filename = os.path.join(metadata_directory, - repo_tool.SNAPSHOT_FILENAME) - - # Set valid generate_timestamp_metadata() arguments. - version = 1 - expiration_date = '1985-10-21T13:20:00Z' - - compressions = ['gz'] - - snapshot_metadata = \ - repo_tool.generate_timestamp_metadata(snapshot_filename, version, - expiration_date, compressions) - self.assertTrue(tuf.formats.TIMESTAMP_SCHEMA.matches(snapshot_metadata)) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, repo_tool.generate_timestamp_metadata, - 3, version, expiration_date, compressions) - self.assertRaises(tuf.FormatError, repo_tool.generate_timestamp_metadata, - snapshot_filename, '3', expiration_date, compressions) - self.assertRaises(tuf.FormatError, repo_tool.generate_timestamp_metadata, - snapshot_filename, version, '3', compressions) - self.assertRaises(tuf.FormatError, repo_tool.generate_timestamp_metadata, - snapshot_filename, version, expiration_date, 3) - self.assertRaises(tuf.FormatError, repo_tool.generate_timestamp_metadata, - snapshot_filename, version, expiration_date, ['compress']) - - - - - def test_sign_metadata(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - metadata_path = os.path.join('repository_data', - 'repository', 'metadata') - keystore_path = os.path.join('repository_data', - 'keystore') - root_filename = os.path.join(metadata_path, 'root.json') - root_metadata = tuf.util.load_json_file(root_filename)['signed'] - - tuf.keydb.create_keydb_from_root_metadata(root_metadata) - tuf.roledb.create_roledb_from_root_metadata(root_metadata) - root_keyids = tuf.roledb.get_role_keyids('root') - - root_private_keypath = os.path.join(keystore_path, 'root_key') - root_private_key = \ - repo_tool.import_rsa_privatekey_from_file(root_private_keypath, - 'password') - - # sign_metadata() expects the private key 'root_metadata' to be in - # 'tuf.keydb'. Remove any public keys that may be loaded before - # adding private key, otherwise a 'tuf.KeyAlreadyExists' exception is - # raised. - tuf.keydb.remove_key(root_private_key['keyid']) - tuf.keydb.add_key(root_private_key) - - root_signable = repo_tool.sign_metadata(root_metadata, root_keyids, - root_filename) - self.assertTrue(tuf.formats.SIGNABLE_SCHEMA.matches(root_signable)) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, repo_tool.sign_metadata, 3, root_keyids, - 'root.json') - self.assertRaises(tuf.FormatError, repo_tool.sign_metadata, root_metadata, - 3, 'root.json') - self.assertRaises(tuf.FormatError, repo_tool.sign_metadata, root_metadata, - root_keyids, 3) - - - - def test_write_metadata_file(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - metadata_directory = os.path.join('repository_data', - 'repository', 'metadata') - root_filename = os.path.join(metadata_directory, 'root.json') - root_signable = tuf.util.load_json_file(root_filename) - - output_filename = os.path.join(temporary_directory, 'root.json') - compressions = ['gz'] - - self.assertFalse(os.path.exists(output_filename)) - repo_tool.write_metadata_file(root_signable, output_filename, compressions, - consistent_snapshot=False) - self.assertTrue(os.path.exists(output_filename)) - self.assertTrue(os.path.exists(output_filename + '.gz')) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, repo_tool.write_metadata_file, - 3, output_filename, compressions, False) - self.assertRaises(tuf.FormatError, repo_tool.write_metadata_file, - root_signable, 3, compressions, False) - self.assertRaises(tuf.FormatError, repo_tool.write_metadata_file, - root_signable, output_filename, 3, False) - self.assertRaises(tuf.FormatError, repo_tool.write_metadata_file, - root_signable, output_filename, compressions, 3) - - - - def test_create_tuf_client_directory(self): - # Test normal case. - temporary_directory = tempfile.mkdtemp(dir=self.temporary_directory) - repository_directory = os.path.join('repository_data', - 'repository') - client_directory = os.path.join(temporary_directory, 'client') - - repo_tool.create_tuf_client_directory(repository_directory, client_directory) - - self.assertTrue(os.path.exists(client_directory)) - metadata_directory = os.path.join(client_directory, 'metadata') - current_directory = os.path.join(metadata_directory, 'current') - previous_directory = os.path.join(metadata_directory, 'previous') - self.assertTrue(os.path.exists(client_directory)) - self.assertTrue(os.path.exists(metadata_directory)) - self.assertTrue(os.path.exists(current_directory)) - self.assertTrue(os.path.exists(previous_directory)) - - - # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, repo_tool.create_tuf_client_directory, - 3, client_directory) - self.assertRaises(tuf.FormatError, repo_tool.create_tuf_client_directory, - repository_directory, 3) - - - # Test invalid argument (i.e., client directory already exists.) - self.assertRaises(tuf.RepositoryError, repo_tool.create_tuf_client_directory, - repository_directory, client_directory) - - # Run the test cases. if __name__ == '__main__': unittest.main() diff --git a/tests/test_roledb.py b/tests/test_roledb.py index eb34e43f..61c0b091 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_roledb.py @@ -15,6 +17,13 @@ Unit test for 'roledb.py'. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals import unittest import logging diff --git a/tests/test_schema.py b/tests/test_schema.py index 66320a01..e8137fdc 100755 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_schema.py @@ -13,10 +15,18 @@ Unit test for 'schema.py' - """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import unittest +import re import logging import tuf @@ -109,7 +119,6 @@ def test_AnyString(self): self.assertTrue(anystring_schema.matches('')) self.assertTrue(anystring_schema.matches('a string')) - self.assertTrue(anystring_schema.matches(u'a unicode string')) # Test conditions for invalid arguments. self.assertFalse(anystring_schema.matches(['a'])) @@ -203,7 +212,6 @@ def test_Integer(self): integer_schema = tuf.schema.Integer() self.assertTrue(integer_schema.matches(99)) - self.assertTrue(integer_schema.matches(0L)) self.assertTrue(tuf.schema.Integer(lo=10, hi=30).matches(25)) # Test conditions for invalid arguments. @@ -273,6 +281,11 @@ def test_Object(self): self.assertRaises(tuf.FormatError, tuf.schema.Object, a=tuf.schema.AnyString(), b=1) + # Test condition for invalid non-dict arguments. + self.assertFalse(object_schema.matches([{'a':'XYZ'}])) + self.assertFalse(object_schema.matches(8)) + + def test_Struct(self): # Test conditions for valid arguments. @@ -318,16 +331,95 @@ def test_Struct(self): def test_RegularExpression(self): - # Test conditions for valid arguments. + # Test conditions for valid arguments. + # RegularExpression(pattern, modifiers, re_object, re_name). re_schema = tuf.schema.RegularExpression('h.*d') self.assertTrue(re_schema.matches('hello world')) + # Provide a pattern that contains the trailing '$' + re_schema_2 = tuf.schema.RegularExpression(pattern='abc$', + modifiers=0, + re_object=None, + re_name='my_re') + + self.assertTrue(re_schema_2.matches('abc')) + + # Test for valid optional arguments. + compiled_re = re.compile('^[a-z].*') + re_schema_optional = tuf.schema.RegularExpression(pattern='abc', + modifiers=0, + re_object=compiled_re, + re_name='my_re') + self.assertTrue(re_schema_optional.matches('abc')) + + # Valid arguments, but the 'pattern' argument is unset (required if the + # 're_object' is 'None'.) + self.assertRaises(tuf.FormatError, tuf.schema.RegularExpression, None, 0, + None, None) + + # Valid arguments, 're_name' is unset, and 'pattern' is None. An exception + # is not raised, but 're_name' is set to 'pattern'. + re_schema_optional = tuf.schema.RegularExpression(pattern=None, + modifiers=0, + re_object=compiled_re, + re_name=None) + + self.assertTrue(re_schema_optional.matches('abc')) + self.assertTrue(re_schema_optional._re_name == 'pattern') + # Test conditions for invalid arguments. self.assertFalse(re_schema.matches('Hello World')) self.assertFalse(re_schema.matches('hello world!')) self.assertFalse(re_schema.matches([33, 'Hello'])) + self.assertRaises(tuf.FormatError, tuf.schema.RegularExpression, 8) + + + + def test_LengthString(self): + # Test conditions for valid arguments. + length_string = tuf.schema.LengthString(11) + + self.assertTrue(length_string.matches('Hello World')) + self.assertTrue(length_string.matches('Hello Marty')) + + # Test conditions for invalid arguments. + self.assertRaises(tuf.FormatError, tuf.schema.LengthString, 'hello') + + self.assertFalse(length_string.matches('hello')) + self.assertFalse(length_string.matches(8)) + + + + def test_LengthBytes(self): + # Test conditions for valid arguments. + length_bytes = tuf.schema.LengthBytes(11) + + self.assertTrue(length_bytes.matches(b'Hello World')) + self.assertTrue(length_bytes.matches(b'Hello Marty')) + + # Test conditions for invalid arguments. + self.assertRaises(tuf.FormatError, tuf.schema.LengthBytes, 'hello') + self.assertRaises(tuf.FormatError, tuf.schema.LengthBytes, True) + + self.assertFalse(length_bytes.matches(b'hello')) + self.assertFalse(length_bytes.matches(8)) + + + + def test_AnyBytes(self): + # Test conditions for valid arguments. + anybytes_schema = tuf.schema.AnyBytes() + + self.assertTrue(anybytes_schema.matches(b'')) + self.assertTrue(anybytes_schema.matches(b'a string')) + + # Test conditions for invalid arguments. + self.assertFalse(anybytes_schema.matches('a string')) + self.assertFalse(anybytes_schema.matches(['a'])) + self.assertFalse(anybytes_schema.matches(3)) + self.assertFalse(anybytes_schema.matches({'a': 'string'})) # Run the unit tests. diff --git a/tests/test_sig.py b/tests/test_sig.py index b4461b82..a9ca535a 100755 --- a/tests/test_sig.py +++ b/tests/test_sig.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ test_sig.py @@ -16,6 +18,14 @@ Test cases for for sig.py. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import unittest import logging diff --git a/tests/test_slow_retrieval_attack.py b/tests/test_slow_retrieval_attack.py index 364fb8ef..635b9058 100755 --- a/tests/test_slow_retrieval_attack.py +++ b/tests/test_slow_retrieval_attack.py @@ -30,24 +30,31 @@ Note: There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatability, where the print statement is a function, an +# Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals import os import sys -import urllib import tempfile import random import time import shutil import json import subprocess -import unittest import logging +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf.formats import tuf.util @@ -55,6 +62,7 @@ import tuf.client.updater as updater import tuf.unittest_toolbox as unittest_toolbox import tuf.repository_tool as repo_tool +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_slow_retrieval_attack') repo_tool.disable_console_log_messages() @@ -102,7 +110,7 @@ def _start_slow_server(self, mode): # NOTE: Following error is raised if a delay is not applied: # - time.sleep(.2) + time.sleep(.5) return server_process @@ -157,9 +165,9 @@ def setUp(self): repository = repo_tool.load_repository(self.repository_directory) file1_filepath = os.path.join(self.repository_directory, 'targets', 'file1.txt') - with open(file1_filepath, 'wb') as file_object: - file_object.write('a' * total_bytes) + data = 'a' * total_bytes + file_object.write(data.encode('utf-8')) key_file = os.path.join(self.keystore_directory, 'timestamp_key') timestamp_private = repo_tool.import_rsa_privatekey_from_file(key_file, @@ -210,69 +218,6 @@ def tearDown(self): - def test_without_tuf_mode_1(self): - # Simulate a slow retrieval attack. - # 'mode_1': When download begins,the server blocks the download - # for a long time by doing nothing before it sends the first byte of data. - - # Retrieve 'file1.txt' provided by the pre-generated repository. - url_prefix = self.repository_mirrors['mirror1']['url_prefix'] - url_file = os.path.join(url_prefix, 'targets', 'file1.txt') - client_filepath = os.path.join(self.client_directory, 'file1.txt') - - # Generate the fileinfo of 'file.txt' to compare it to what is downloaded. - # The download should complete, albeit slowly (the slow retrieval server - # sets a limit on the delay.) - filepath = os.path.join(self.repository_directory, 'targets', 'file1.txt') - length, hashes = tuf.util.get_file_details(filepath) - fileinfo = tuf.formats.make_fileinfo(length, hashes) - - try: - server_process = self._start_slow_server('mode_1') - urllib.urlretrieve(url_file, client_filepath) - - # Verify the expected file size and hash of the downloaded file. - length, hashes = tuf.util.get_file_details(client_filepath) - download_fileinfo = tuf.formats.make_fileinfo(length, hashes) - self.assertEqual(fileinfo, download_fileinfo) - - finally: - # Terminate the slow retrieval (mode 1) server. - self._stop_slow_server(server_process) - - - - def test_without_tuf_mode_2(self): - # Simulate a slow retrieval attack. - # 'mode_1': When download begins, the server blocks the download for a long - # time by doing nothing before it sends the first byte of data. - - url_prefix = self.repository_mirrors['mirror1']['url_prefix'] - url_file = os.path.join(url_prefix, 'targets', 'file1.txt') - client_filepath = os.path.join(self.client_directory, 'file1.txt') - - # Generate the fileinfo of 'file.txt' to compare it to what is downloaded. - # The download should complete, albeit slowly (the slow retrieval server - # sets a limit on the delay.) - filepath = os.path.join(self.repository_directory, 'targets', 'file1.txt') - length, hashes = tuf.util.get_file_details(filepath) - fileinfo = tuf.formats.make_fileinfo(length, hashes) - - try: - server_process = self._start_slow_server('mode_2') - urllib.urlretrieve(url_file, client_filepath) - - # Verify the expected file size and hash of the downloaded file. - length, hashes = tuf.util.get_file_details(client_filepath) - download_fileinfo = tuf.formats.make_fileinfo(length, hashes) - self.assertEqual(fileinfo, download_fileinfo) - - finally: - # Terminate the slow retrieval (mode 2) server. - self._stop_slow_server(server_process) - - - def test_with_tuf_mode_1(self): # Simulate a slow retrieval attack. # 'mode_1': When download begins,the server blocks the download for a long @@ -285,18 +230,18 @@ def test_with_tuf_mode_1(self): client_filepath = os.path.join(self.client_directory, 'file1.txt') try: file1_target = self.repository_updater.target('file1.txt') - self.repository_updater.download_target(file1_target, client_filepath) + self.repository_updater.download_target(file1_target, self.client_directory) # Verify that the specific 'tuf.SlowRetrievalError' exception is raised by # each mirror. - except tuf.NoWorkingMirrorError, exception: - for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + except tuf.NoWorkingMirrorError as exception: + for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'targets', 'file1.txt') # Verify that 'file1.txt' is the culprit. self.assertEqual(url_file, mirror_url) - self.assertTrue(isinstance(mirror_error, tuf.SlowRetrievalError)) + self.assertTrue(isinstance(mirror_error, tuf.DownloadLengthMismatchError)) else: self.fail('TUF did not prevent a slow retrieval attack.') @@ -316,20 +261,20 @@ def test_with_tuf_mode_2(self): client_filepath = os.path.join(self.client_directory, 'file1.txt') try: file1_target = self.repository_updater.target('file1.txt') - self.repository_updater.download_target(file1_target, client_filepath) + self.repository_updater.download_target(file1_target, self.client_directory) # Verify that the specific 'tuf.SlowRetrievalError' exception is raised by # each mirror. 'file1.txt' should be large enough to trigger a slow # retrieval attack, otherwise the expected exception may not be consistently # raised. - except tuf.NoWorkingMirrorError, exception: - for mirror_url, mirror_error in exception.mirror_errors.iteritems(): + except tuf.NoWorkingMirrorError as exception: + for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'targets', 'file1.txt') # Verify that 'file1.txt' is the culprit. self.assertEqual(url_file, mirror_url) - self.assertTrue(isinstance(mirror_error, tuf.SlowRetrievalError)) + self.assertTrue(isinstance(mirror_error, tuf.DownloadLengthMismatchError)) else: # Another possibility is to check for a successfully downloaded diff --git a/tests/test_updater.py b/tests/test_updater.py index ec2414b6..515d74b5 100755 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -38,7 +38,13 @@ less dependent than 2. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals import os import time @@ -46,9 +52,16 @@ import copy import tempfile import logging -import unittest import random import subprocess +import sys + +# 'unittest2' required for testing under Python < 2.7. +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest import tuf import tuf.util @@ -60,6 +73,7 @@ import tuf.repository_tool as repo_tool import tuf.unittest_toolbox as unittest_toolbox import tuf.client.updater as updater +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_updater') repo_tool.disable_console_log_messages() @@ -337,7 +351,7 @@ def test_1__update_fileinfo(self): root_filepath = os.path.join(self.client_metadata_current, 'root.json') length, hashes = tuf.util.get_file_details(root_filepath) root_fileinfo = tuf.formats.make_fileinfo(length, hashes) - self.assertTrue('root.json' in fileinfo_dict.keys()) + self.assertTrue('root.json' in fileinfo_dict) self.assertEqual(fileinfo_dict['root.json'], root_fileinfo) # Verify that 'self.fileinfo' is incremented if another role is updated. @@ -554,8 +568,8 @@ def test_3__update_metadata(self): self.repository_updater._update_metadata('targets', targets_compressed_fileinfo) - except tuf.NoWorkingMirrorError, e: - for mirror_error in e.mirror_errors.values(): + except tuf.NoWorkingMirrorError as e: + for mirror_error in six.itervalues(e.mirror_errors): assert isinstance(mirror_error, tuf.BadHashError) # Invalid fileinfo for the compressed version of 'targets.json' @@ -572,8 +586,8 @@ def test_3__update_metadata(self): targets_compressed_fileinfo, 'gzip', targets_fileinfo) - except tuf.NoWorkingMirrorError, e: - for mirror_error in e.mirror_errors.values(): + except tuf.NoWorkingMirrorError as e: + for mirror_error in six.itervalues(e.mirror_errors): assert isinstance(mirror_error, tuf.DownloadLengthMismatchError) @@ -648,7 +662,7 @@ def test_3__targets_of_role(self): # target files. self.assertTrue(tuf.formats.TARGETFILES_SCHEMA.matches(targets_list)) for target in targets_list: - self.assertTrue((target['filepath'], target['fileinfo']) in targets_in_metadata.items()) + self.assertTrue((target['filepath'], target['fileinfo']) in six.iteritems(targets_in_metadata)) @@ -681,7 +695,7 @@ def test_4_refresh(self): # Reference 'self.Repository.metadata['current']['targets']'. Ensure # 'target3' is not already specified. targets_metadata = self.repository_updater.metadata['current']['targets'] - self.assertFalse(target3 in targets_metadata['targets'].keys()) + self.assertFalse(target3 in targets_metadata['targets']) # Verify the expected version numbers of the roles to be modified. self.assertTrue(self.repository_updater.metadata['current']['targets']\ @@ -700,7 +714,7 @@ def test_4_refresh(self): targets_metadata = self.repository_updater.metadata['current']['targets'] targets_directory = os.path.join(self.repository_directory, 'targets') target3 = target3[len(targets_directory):] - self.assertTrue(target3 in targets_metadata['targets'].keys()) + self.assertTrue(target3 in targets_metadata['targets']) # Verify the expected version numbers of the updated roles. self.assertTrue(self.repository_updater.metadata['current']['targets']\ @@ -795,7 +809,7 @@ def test_5_targets_of_role(self): # target files. self.assertTrue(tuf.formats.TARGETFILES_SCHEMA.matches(targets_list)) for target in targets_list: - self.assertTrue((target['filepath'], target['fileinfo']) in expected_targets.items()) + self.assertTrue((target['filepath'], target['fileinfo']) in six.iteritems(expected_targets)) # Test: Invalid arguments. @@ -829,7 +843,73 @@ def test_6_target(self): # Test: invalid target path. self.assertRaises(tuf.UnknownTargetError, self.repository_updater.target, self.random_path()) + + # Test updater.target() backtracking behavior (enabled by default.) + targets_directory = os.path.join(self.repository_directory, 'targets') + foo_directory = os.path.join(targets_directory, 'foo') + os.makedirs(foo_directory) + foo_package = os.path.join(foo_directory, 'foo1.1.tar.gz') + with open(foo_package, 'wb') as file_object: + file_object.write(b'new release') + + # Modify delegations on the remote repository to test backtracking behavior. + repository = repo_tool.load_repository(self.repository_directory) + + + repository.targets.delegate('role2', [self.role_keys['targets']['public']], + [], restricted_paths=[foo_directory]) + + repository.targets.delegate('role3', [self.role_keys['targets']['public']], + [foo_package], restricted_paths=[foo_directory]) + repository.targets.load_signing_key(self.role_keys['targets']['private']) + repository.targets('role2').load_signing_key(self.role_keys['targets']['private']) + repository.targets('role3').load_signing_key(self.role_keys['targets']['private']) + repository.snapshot.load_signing_key(self.role_keys['snapshot']['private']) + repository.timestamp.load_signing_key(self.role_keys['timestamp']['private']) + repository.write() + + # Move the staged metadata to the "live" metadata. + shutil.rmtree(os.path.join(self.repository_directory, 'metadata')) + shutil.copytree(os.path.join(self.repository_directory, 'metadata.staged'), + os.path.join(self.repository_directory, 'metadata')) + + # updater.target() should find 'foo1.1.tar.gz' by backtracking to + # 'targets/role3'. 'targets/role2' allows backtracking. + self.repository_updater.refresh() + self.repository_updater.target('foo/foo1.1.tar.gz') + + + # Test when 'targets/role2' does *not* allow backtracking. If + # 'foo/foo1.1.tar.gz' is not provided by the authoritative 'target/role2', + # updater.target() should return a 'tuf.UnknownTargetError' exception. + repository = repo_tool.load_repository(self.repository_directory) + + repository.targets.revoke('role2') + repository.targets.revoke('role3') + + # Ensure we delegate in trusted order (i.e., 'role2' has higher priority.) + repository.targets.delegate('role2', [self.role_keys['targets']['public']], + [], backtrack=False, restricted_paths=[foo_directory]) + repository.targets.delegate('role3', [self.role_keys['targets']['public']], + [foo_package], restricted_paths=[foo_directory]) + + repository.targets('role2').load_signing_key(self.role_keys['targets']['private']) + repository.targets('role3').load_signing_key(self.role_keys['targets']['private']) + repository.targets.load_signing_key(self.role_keys['targets']['private']) + repository.snapshot.load_signing_key(self.role_keys['snapshot']['private']) + repository.timestamp.load_signing_key(self.role_keys['timestamp']['private']) + repository.write() + + # Move the staged metadata to the "live" metadata. + shutil.rmtree(os.path.join(self.repository_directory, 'metadata')) + shutil.copytree(os.path.join(self.repository_directory, 'metadata.staged'), + os.path.join(self.repository_directory, 'metadata')) + + # Verify that 'tuf.UnknownTargetError' is raised by updater.target(). + self.repository_updater.refresh() + self.assertRaises(tuf.UnknownTargetError, self.repository_updater.target, + 'foo/foo1.1.tar.gz') @@ -840,7 +920,7 @@ def test_6_download_target(self): # that will be passed as an argument to 'download_target()'. destination_directory = self.make_temp_directory() target_filepaths = \ - self.repository_updater.metadata['current']['targets']['targets'].keys() + list(self.repository_updater.metadata['current']['targets']['targets'].keys()) # Test: normal case. @@ -872,14 +952,14 @@ def test_6_download_target(self): # field contains at least one confined target and excludes needed target # file. mirrors = self.repository_updater.mirrors - for mirror_name, mirror_info in mirrors.items(): + for mirror_name, mirror_info in six.iteritems(mirrors): mirrors[mirror_name]['confined_target_dirs'] = [self.random_path()] try: self.repository_updater.download_target(target_fileinfo, destination_directory) - except tuf.NoWorkingMirrorError, exception: + except tuf.NoWorkingMirrorError as exception: # Ensure that no mirrors were found due to mismatch in confined target # directories. get_list_of_mirrors() returns an empty list in this case, # which does not generate specific exception errors. @@ -1023,6 +1103,25 @@ def test_8_remove_obsolete_targets(self): # in 'destination_directory' remains the same. self.repository_updater.remove_obsolete_targets(destination_directory) self.assertTrue(os.listdir(destination_directory), 1) + + + + + + def test_9__get_target_hash(self): + # Test normal case. + # Test target filepaths with ascii and non-ascii characters. + expected_target_hashes = { + '/file1.txt': 'e3a3d89eb3b70ce3fbce6017d7b8c12d4abd5635427a0e8a238f53157df85b3d', + '/Jalape\xc3\xb1o': '78bfd5c314680545eb48ecad508aceb861f8d6e680f4fe1b791da45c298cda88' + } + for filepath, target_hash in six.iteritems(expected_target_hashes): + self.assertTrue(tuf.formats.RELPATH_SCHEMA.matches(filepath)) + self.assertTrue(tuf.formats.HASH_SCHEMA.matches(target_hash)) + self.assertEqual(self.repository_updater._get_target_hash(filepath), target_hash) + + # Test for improperly formatted argument. + self.assertRaises(tuf.FormatError, tuf.util.get_target_hash, 8) diff --git a/tests/test_util.py b/tests/test_util.py index 17a26262..9c1c20ef 100755 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -8,7 +8,7 @@ Konstantin Andrianov. - February 1, 2013 + February 1, 2013. See LICENSE for licensing information. @@ -16,7 +16,14 @@ Unit test for 'util.py' """ + +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals import os import sys @@ -29,8 +36,9 @@ import tuf import tuf.log import tuf.hash -import tuf.util as util +import tuf.util import tuf.unittest_toolbox as unittest_toolbox +import tuf._vendor.six as six logger = logging.getLogger('tuf.test_util') @@ -39,7 +47,7 @@ class TestUtil(unittest_toolbox.Modified_TestCase): def setUp(self): unittest_toolbox.Modified_TestCase.setUp(self) - self.temp_fileobj = util.TempFile() + self.temp_fileobj = tuf.util.TempFile() @@ -57,13 +65,16 @@ def test_A1_tempfile_close_temp_file(self): def _extract_tempfile_directory(self, config_temp_dir=None): - """[Helper] Takes a directory (essentially specified in the config.py as - 'temporary_directory') and substitutes tempfile.TemporaryFile() with - tempfile.mkstemp() in order to extract actual directory of the stored - tempfile. Returns the config's temporary directory (or default temp - directory) and actual directory.""" + """ + Takes a directory (essentially specified in the conf.py as + 'temporary_directory') and substitutes tempfile.TemporaryFile() with + tempfile.mkstemp() in order to extract actual directory of the stored + tempfile. Returns the config's temporary directory (or default temp + directory) and actual directory. + """ + # Patching 'tuf.conf.temporary_directory'. - util.tuf.conf.temporary_directory = config_temp_dir + tuf.conf.temporary_directory = config_temp_dir if config_temp_dir is None: # 'config_temp_dir' needs to be set to default. @@ -72,10 +83,10 @@ def _extract_tempfile_directory(self, config_temp_dir=None): # Patching 'tempfile.TemporaryFile()' (by substituting # temfile.TemporaryFile() with tempfile.mkstemp()) in order to get the # directory of the stored tempfile object. - saved_tempfile_TemporaryFile = util.tempfile.NamedTemporaryFile - util.tempfile.NamedTemporaryFile = tempfile.mkstemp - _temp_fileobj = util.TempFile() - util.tempfile.NamedTemporaryFile = saved_tempfile_TemporaryFile + saved_tempfile_TemporaryFile = tuf.util.tempfile.NamedTemporaryFile + tuf.util.tempfile.NamedTemporaryFile = tempfile.mkstemp + _temp_fileobj = tuf.util.TempFile() + tuf.util.tempfile.NamedTemporaryFile = saved_tempfile_TemporaryFile junk, _tempfilepath = _temp_fileobj.temporary_file _tempfile_dir = os.path.dirname(_tempfilepath) @@ -90,21 +101,30 @@ def _extract_tempfile_directory(self, config_temp_dir=None): def test_A2_tempfile_init(self): - # Goal: Verify that tempfile is stored in an appropriate temp directory. + # Goal: Verify that temporary files are stored in the appropriate temp + # directory. The location of the temporary files is set in 'tuf.conf.py'. # Test: Expected input verification. - config_temp_dirs = [None, self.make_temp_directory()] - for config_temp_dir in config_temp_dirs: - config_temp_dir, actual_dir = \ - self._extract_tempfile_directory(config_temp_dir) - self.assertEquals(config_temp_dir, actual_dir) - + # Assumed 'tuf.conf.temporary_directory' is 'None' initially. + temp_file = tuf.util.TempFile() + temp_file_directory = os.path.dirname(temp_file.temporary_file.name) + self.assertEqual(tempfile.gettempdir(), temp_file_directory) + + saved_temporary_directory = tuf.conf.temporary_directory + temp_directory = self.make_temp_directory() + tuf.conf.temporary_directory = temp_directory + temp_file = tuf.util.TempFile() + temp_file_directory = os.path.dirname(temp_file.temporary_file.name) + self.assertEqual(temp_directory, temp_file_directory) + + tuf.conf.temporary_directory = saved_temporary_directory + # Test: Unexpected input handling. config_temp_dirs = [self.random_string(), 123, ['a'], {'a':1}] for config_temp_dir in config_temp_dirs: config_temp_dir, actual_dir = \ self._extract_tempfile_directory(config_temp_dir) - self.assertEquals(tempfile.gettempdir(), actual_dir) + self.assertEqual(tempfile.gettempdir(), actual_dir) @@ -116,8 +136,8 @@ def test_A3_tempfile_read(self): self.temp_fileobj.temporary_file = fileobj # Test: Expected input. - self.assertEquals(self.temp_fileobj.read(), '1234567890') - self.assertEquals(self.temp_fileobj.read(4), '1234') + self.assertEqual(self.temp_fileobj.read().decode('utf-8'), '1234567890') + self.assertEqual(self.temp_fileobj.read(4).decode('utf-8'), '1234') # Test: Unexpected input. for bogus_arg in ['abcd', ['abcd'], {'a':'a'}, -100]: @@ -127,8 +147,11 @@ def test_A3_tempfile_read(self): def test_A4_tempfile_write(self): data = self.random_string() - self.temp_fileobj.write(data) - self.assertEquals(data, self.temp_fileobj.read()) + self.temp_fileobj.write(data.encode('utf-8')) + self.assertEqual(data, self.temp_fileobj.read().decode('utf-8')) + + self.temp_fileobj.write(data.encode('utf-8'), auto_flush=False) + self.assertEqual(data, self.temp_fileobj.read().decode('utf-8')) @@ -136,15 +159,18 @@ def test_A5_tempfile_move(self): # Destination directory to save the temporary file in. dest_temp_dir = self.make_temp_directory() dest_path = os.path.join(dest_temp_dir, self.random_string()) - self.temp_fileobj.write(self.random_string()) + self.temp_fileobj.write(self.random_string().encode('utf-8')) self.temp_fileobj.move(dest_path) self.assertTrue(dest_path) def _compress_existing_file(self, filepath): - """[Helper]Compresses file 'filepath' and returns file path of - the compresses file.""" + """ + [Helper]Compresses file 'filepath' and returns file path of + the compresses file. + """ + # NOTE: DO NOT forget to remove the newly created compressed file! if os.path.exists(filepath): compressed_filepath = filepath+'.gz' @@ -153,9 +179,11 @@ def _compress_existing_file(self, filepath): f_out.writelines(f_in) f_out.close() f_in.close() + return compressed_filepath + else: - print 'Compression of '+repr(filepath)+' failed. Path does not exist.' + logger.error('Compression of '+repr(filepath)+' failed. Path does not exist.') sys.exit(1) @@ -167,9 +195,10 @@ def _decompress_file(self, compressed_filepath): file_content = f.read() f.close() return file_content + else: - print 'Decompression of '+repr(compressed_filepath)+' failed. '+\ - 'Path does not exist.' + logger.error('Decompression of '+repr(compressed_filepath)+' failed. '+\ + 'Path does not exist.') sys.exit(1) @@ -191,19 +220,26 @@ def test_A6_tempfile_decompress_temp_file_object(self): self.assertRaises(tuf.Error, self.temp_fileobj.decompress_temp_file_object, arg) self.temp_fileobj.decompress_temp_file_object('gzip') - self.assertEquals(self.temp_fileobj.read(), fileobj.read()) + self.assertEqual(self.temp_fileobj.read(), fileobj.read()) # Checking the content of the TempFile's '_orig_file' instance. - _orig_data_file = \ - self.make_temp_data_file(data=self.temp_fileobj._orig_file.read()) - data_in_orig_file = self._decompress_file(_orig_data_file) + check_compressed_original = self.make_temp_file() + with open(check_compressed_original, 'wb') as file_object: + file_object.write(self.temp_fileobj._orig_file.read()) + data_in_orig_file = self._decompress_file(check_compressed_original) fileobj.seek(0) - self.assertEquals(data_in_orig_file, fileobj.read()) - + self.assertEqual(data_in_orig_file, fileobj.read()) + # Try decompressing once more. self.assertRaises(tuf.Error, - self.temp_fileobj.decompress_temp_file_object,'gzip') + self.temp_fileobj.decompress_temp_file_object, 'gzip') + # Test decompression of invalid gzip file. + temp_file = tuf.util.TempFile() + fileobj.seek(0) + temp_file.write(fileobj.read()) + temp_file.decompress_temp_file_object('gzip') + def test_B1_get_file_details(self): @@ -218,16 +254,17 @@ def test_B1_get_file_details(self): file_length = os.path.getsize(filepath) # Test: Expected input. - self.assertEquals(util.get_file_details(filepath), (file_length, file_hash)) + self.assertEqual(tuf.util.get_file_details(filepath), (file_length, file_hash)) # Test: Incorrect input. bogus_inputs = [self.random_string(), 1234, [self.random_string()], {'a': 'b'}, None] + for bogus_input in bogus_inputs: - if isinstance(bogus_input, basestring): - self.assertRaises(tuf.Error, util.get_file_details, bogus_input) + if isinstance(bogus_input, six.string_types): + self.assertRaises(tuf.Error, tuf.util.get_file_details, bogus_input) else: - self.assertRaises(tuf.FormatError, util.get_file_details, bogus_input) + self.assertRaises(tuf.FormatError, tuf.util.get_file_details, bogus_input) @@ -236,11 +273,11 @@ def test_B2_ensure_parent_dir(self): non_existing_parent_dir = os.path.join(existing_parent_dir, 'a', 'b') for parent_dir in [existing_parent_dir, non_existing_parent_dir, 12, [3]]: - if isinstance(parent_dir, basestring): - util.ensure_parent_dir(os.path.join(parent_dir, 'a.txt')) + if isinstance(parent_dir, six.string_types): + tuf.util.ensure_parent_dir(os.path.join(parent_dir, 'a.txt')) self.assertTrue(os.path.isdir(parent_dir)) else: - self.assertRaises(tuf.FormatError, util.ensure_parent_dir, parent_dir) + self.assertRaises(tuf.FormatError, tuf.util.ensure_parent_dir, parent_dir) @@ -283,26 +320,31 @@ def test_B4_import_json(self): def test_B5_load_json_string(self): # Test normal case. data = ['a', {'b': ['c', None, 30.3, 29]}] - json_string = util.json.dumps(data) - self.assertEquals(data, util.load_json_string(json_string)) + json_string = tuf.util.json.dumps(data) + self.assertEqual(data, tuf.util.load_json_string(json_string)) # Test invalid arguments. - self.assertRaises(tuf.Error, util.load_json_string, 8) + self.assertRaises(tuf.Error, tuf.util.load_json_string, 8) invalid_json_string = {'a': tuf.FormatError} - self.assertRaises(tuf.Error, util.load_json_string, invalid_json_string) + self.assertRaises(tuf.Error, tuf.util.load_json_string, invalid_json_string) def test_B6_load_json_file(self): data = ['a', {'b': ['c', None, 30.3, 29]}] filepath = self.make_temp_file() - fileobj = open(filepath, 'wb') - util.json.dump(data, fileobj) + fileobj = open(filepath, 'wt') + tuf.util.json.dump(data, fileobj) fileobj.close() - self.assertEquals(data, util.load_json_file(filepath)) + self.assertEqual(data, tuf.util.load_json_file(filepath)) + + # Test a gzipped file. + compressed_filepath = self._compress_existing_file(filepath) + self.assertEqual(data, tuf.util.load_json_file(compressed_filepath)) + Errors = (tuf.FormatError, IOError) - for bogus_arg in ['a', 1, ['a'], {'a':'b'}]: - self.assertRaises(Errors, util.load_json_file, bogus_arg) + for bogus_arg in [b'a', 1, [b'a'], {'a':b'b'}]: + self.assertRaises(Errors, tuf.util.load_json_file, bogus_arg) @@ -313,13 +355,13 @@ def test_C1_get_target_hash(self): '/README.txt': '8faee106f1bb69f34aaf1df1e3c2e87d763c4d878cb96b91db13495e32ceb0b0', '/warehouse/file2.txt': 'd543a573a2cec67026eff06e75702303559e64e705eba06f65799baaf0424417' } - for filepath, target_hash in expected_target_hashes.items(): + for filepath, target_hash in six.iteritems(expected_target_hashes): self.assertTrue(tuf.formats.RELPATH_SCHEMA.matches(filepath)) self.assertTrue(tuf.formats.HASH_SCHEMA.matches(target_hash)) - self.assertEqual(util.get_target_hash(filepath), target_hash) + self.assertEqual(tuf.util.get_target_hash(filepath), target_hash) # Test for improperly formatted argument. - self.assertRaises(tuf.FormatError, util.get_target_hash, 8) + self.assertRaises(tuf.FormatError, tuf.util.get_target_hash, 8) @@ -350,20 +392,20 @@ def test_C2_find_delegated_role(self): ] self.assertTrue(tuf.formats.ROLELIST_SCHEMA.matches(role_list)) - self.assertEqual(util.find_delegated_role(role_list, 'targets/tuf'), 1) - self.assertEqual(util.find_delegated_role(role_list, 'targets/warehouse'), 0) + self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/tuf'), 1) + self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/warehouse'), 0) # Test for non-existent role. 'find_delegated_role()' returns 'None' # if the role is not found. - self.assertEqual(util.find_delegated_role(role_list, 'targets/non-existent'), + self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/non-existent'), None) # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, util.find_delegated_role, 8, role_list) - self.assertRaises(tuf.FormatError, util.find_delegated_role, 8, 'targets/tuf') + self.assertRaises(tuf.FormatError, tuf.util.find_delegated_role, 8, role_list) + self.assertRaises(tuf.FormatError, tuf.util.find_delegated_role, 8, 'targets/tuf') # Test duplicate roles. role_list.append(role_list[1]) - self.assertRaises(tuf.RepositoryError, util.find_delegated_role, role_list, + self.assertRaises(tuf.RepositoryError, tuf.util.find_delegated_role, role_list, 'targets/tuf') # Test missing 'name' attribute (optional, but required by @@ -371,7 +413,7 @@ def test_C2_find_delegated_role(self): # Delete the duplicate role, and the remaining role's 'name' attribute. del role_list[2] del role_list[0]['name'] - self.assertRaises(tuf.RepositoryError, util.find_delegated_role, role_list, + self.assertRaises(tuf.RepositoryError, tuf.util.find_delegated_role, role_list, 'targets/warehouse') @@ -384,38 +426,38 @@ def test_C3_paths_are_consistent_with_hash_prefixes(self): # Ensure the paths of 'list_of_targets' each have the epected path hash # prefix listed in 'path_hash_prefixes'. for filepath in list_of_targets: - self.assertTrue(util.get_target_hash(filepath)[0:4] in path_hash_prefixes) + self.assertTrue(tuf.util.get_target_hash(filepath)[0:4] in path_hash_prefixes) - self.assertTrue(util.paths_are_consistent_with_hash_prefixes(list_of_targets, + self.assertTrue(tuf.util.paths_are_consistent_with_hash_prefixes(list_of_targets, path_hash_prefixes)) extra_invalid_prefix = ['e3a3', '8fae', 'd543', '0000'] - self.assertTrue(util.paths_are_consistent_with_hash_prefixes(list_of_targets, + self.assertTrue(tuf.util.paths_are_consistent_with_hash_prefixes(list_of_targets, extra_invalid_prefix)) # Test improperly formatted arguments. self.assertRaises(tuf.FormatError, - util.paths_are_consistent_with_hash_prefixes, 8, + tuf.util.paths_are_consistent_with_hash_prefixes, 8, path_hash_prefixes) self.assertRaises(tuf.FormatError, - util.paths_are_consistent_with_hash_prefixes, + tuf.util.paths_are_consistent_with_hash_prefixes, list_of_targets, 8) self.assertRaises(tuf.FormatError, - util.paths_are_consistent_with_hash_prefixes, + tuf.util.paths_are_consistent_with_hash_prefixes, list_of_targets, ['zza1']) # Test invalid list of targets. bad_target_path = '/file5.txt' - self.assertTrue(util.get_target_hash(bad_target_path)[0:4] not in + self.assertTrue(tuf.util.get_target_hash(bad_target_path)[0:4] not in path_hash_prefixes) - self.assertFalse(util.paths_are_consistent_with_hash_prefixes([bad_target_path], + self.assertFalse(tuf.util.paths_are_consistent_with_hash_prefixes([bad_target_path], path_hash_prefixes)) # Add invalid target path to 'list_of_targets'. list_of_targets.append(bad_target_path) - self.assertFalse(util.paths_are_consistent_with_hash_prefixes(list_of_targets, + self.assertFalse(tuf.util.paths_are_consistent_with_hash_prefixes(list_of_targets, path_hash_prefixes)) @@ -449,42 +491,42 @@ def test_C4_ensure_all_targets_allowed(self): } self.assertTrue(tuf.formats.DELEGATIONS_SCHEMA.matches(parent_delegations)) - util.ensure_all_targets_allowed(rolename, list_of_targets, + tuf.util.ensure_all_targets_allowed(rolename, list_of_targets, parent_delegations) # The target files of 'targets' are always allowed. 'list_of_targets' and # 'parent_delegations' are not checked in this case. - util.ensure_all_targets_allowed('targets', list_of_targets, + tuf.util.ensure_all_targets_allowed('targets', list_of_targets, parent_delegations) # Test improperly formatted arguments. - self.assertRaises(tuf.FormatError, util.ensure_all_targets_allowed, + self.assertRaises(tuf.FormatError, tuf.util.ensure_all_targets_allowed, 8, list_of_targets, parent_delegations) - self.assertRaises(tuf.FormatError, util.ensure_all_targets_allowed, + self.assertRaises(tuf.FormatError, tuf.util.ensure_all_targets_allowed, rolename, 8, parent_delegations) - self.assertRaises(tuf.FormatError, util.ensure_all_targets_allowed, + self.assertRaises(tuf.FormatError, tuf.util.ensure_all_targets_allowed, rolename, list_of_targets, 8) # Test for invalid 'rolename', which has not been delegated by its parent, # 'targets'. - self.assertRaises(tuf.RepositoryError, util.ensure_all_targets_allowed, + self.assertRaises(tuf.RepositoryError, tuf.util.ensure_all_targets_allowed, 'targets/non-delegated_rolename', list_of_targets, parent_delegations) # Test for target file that is not allowed by the parent role. - self.assertRaises(tuf.ForbiddenTargetError, util.ensure_all_targets_allowed, + self.assertRaises(tuf.ForbiddenTargetError, tuf.util.ensure_all_targets_allowed, 'targets/warehouse', ['file8.txt'], parent_delegations) - self.assertRaises(tuf.ForbiddenTargetError, util.ensure_all_targets_allowed, + self.assertRaises(tuf.ForbiddenTargetError, tuf.util.ensure_all_targets_allowed, 'targets/warehouse', ['file1.txt', 'bad-README.txt'], parent_delegations) # Test for required attributes. # Missing 'paths' attribute. del parent_delegations['roles'][0]['paths'] - self.assertRaises(tuf.FormatError, util.ensure_all_targets_allowed, + self.assertRaises(tuf.FormatError, tuf.util.ensure_all_targets_allowed, 'targets/warehouse', list_of_targets, parent_delegations) # Test 'path_hash_prefixes' attribute. @@ -492,16 +534,33 @@ def test_C4_ensure_all_targets_allowed(self): parent_delegations['roles'][0]['path_hash_prefixes'] = path_hash_prefixes # Test normal case for 'path_hash_prefixes'. - util.ensure_all_targets_allowed('targets/warehouse', list_of_targets, + tuf.util.ensure_all_targets_allowed('targets/warehouse', list_of_targets, parent_delegations) # Test target file with a path_hash_prefix that is not allowed in its # parent role. - path_hash_prefix = util.get_target_hash('file5.txt')[0:4] + path_hash_prefix = tuf.util.get_target_hash('file5.txt')[0:4] self.assertTrue(path_hash_prefix not in parent_delegations['roles'][0] ['path_hash_prefixes']) - self.assertRaises(tuf.ForbiddenTargetError, util.ensure_all_targets_allowed, + self.assertRaises(tuf.ForbiddenTargetError, tuf.util.ensure_all_targets_allowed, 'targets/warehouse', ['file5.txt'], parent_delegations) + + + + def test_C5_unittest_toolbox_make_temp_directory(self): + # Verify that the tearDown function does not fail when + # unittest_toolbox.make_temp_directory deletes the generated temp directory + # here. + temp_directory = self.make_temp_directory() + os.rmdir(temp_directory) + + + + def test_c6_get_compressed_length(self): + self.temp_fileobj.write(b'hello world') + self.assertTrue(self.temp_fileobj.get_compressed_length() == 11) + + temp_file = tuf.util.TempFile() diff --git a/tox.ini b/tox.ini index 9a3935f8..0c586e04 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py27 +envlist = py26, py27, py32, py33, py34 [testenv] @@ -18,3 +18,9 @@ deps = coverage pynacl pycrypto + + +[testenv:py26] +deps = + {[testenv]deps} + unittest2 diff --git a/tuf/__init__.py b/tuf/__init__.py index cd6de9e7..4b9d3b89 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -20,10 +20,18 @@ provide that reason in those cases. """ -import urlparse +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import logging import tuf.log +import tuf._vendor.six as six logging = logging.getLogger('tuf.__init__') @@ -68,7 +76,7 @@ def __init__(self, exception): def __str__(self): # Show the original exception. - return str(self.exception) + return repr(self.exception) @@ -90,8 +98,8 @@ def __init__(self, expected_hash, observed_hash): self.observed_hash = observed_hash def __str__(self): - return 'Observed hash ('+str(self.observed_hash)+\ - ') != expected hash ('+str(self.expected_hash)+')' + return 'Observed hash (' + repr(self.observed_hash)+\ + ') != expected hash (' + repr(self.expected_hash)+')' @@ -155,9 +163,9 @@ def __init__(self, metadata_role, previous_version, current_version): def __str__(self): - return 'Downloaded '+str(self.metadata_role)+' is older ('+\ - str(self.previous_version)+') than the version currently '+\ - 'installed ('+repr(self.current_version)+').' + return 'Downloaded ' + repr(self.metadata_role)+' is older ('+\ + repr(self.previous_version) + ') than the version currently '+\ + 'installed (' + repr(self.current_version) + ').' @@ -178,7 +186,7 @@ def __init__(self, metadata_role_name): self.metadata_role_name = metadata_role_name def __str__(self): - return str(self.metadata_role_name)+' metadata has bad signature!' + return repr(self.metadata_role_name) + ' metadata has bad signature.' @@ -209,7 +217,7 @@ def __init__(self, exception): def __str__(self): # Show the original exception. - return str(self.exception) + return repr(self.exception) @@ -231,8 +239,8 @@ def __init__(self, expected_length, observed_length): self.observed_length = observed_length #bytes def __str__(self): - return 'Observed length ('+str(self.observed_length)+\ - ') <= expected length ('+str(self.expected_length)+')' + return 'Observed length (' + repr(self.observed_length)+\ + ') <= expected length (' + repr(self.expected_length) + ').' @@ -245,8 +253,8 @@ def __init__(self, average_download_speed): self.__average_download_speed = average_download_speed #bytes/second def __str__(self): - return "Download was too slow. Average speed: "+\ - str(self.__average_download_speed)+" bytes/second" + return 'Download was too slow. Average speed: ' +\ + repr(self.__average_download_speed) + ' bytes per second.' @@ -294,17 +302,26 @@ class InvalidNameError(Error): class UnsignedMetadataError(Error): """Indicate metadata object with insufficient threshold of signatures.""" + + def __init__(self, message, signable): + self.exception_message = message + self.signable = signable + + def __str__(self): + return self.exception_message class NoWorkingMirrorError(Error): - """An updater will throw this exception in case it could not download a - metadata or target file. + """ + An updater will throw this exception in case it could not download a + metadata or target file. - A dictionary of Exception instances indexed by every mirror URL will also be - provided.""" + A dictionary of Exception instances indexed by every mirror URL will also be + provided. + """ def __init__(self, mirror_errors): # Dictionary of URL strings to Exception instances @@ -316,15 +333,15 @@ def __str__(self): for mirror_url, mirror_error in self.mirror_errors.iteritems(): try: # http://docs.python.org/2/library/urlparse.html#urlparse.urlparse - mirror_url_tokens = urlparse.urlparse(mirror_url) + mirror_url_tokens = six.moves.urllib.parse.urlparse(mirror_url) except: - logging.exception('Failed to parse mirror URL: '+str(mirror_url)) + logging.exception('Failed to parse mirror URL: ' + repr(mirror_url)) mirror_netloc = mirror_url else: mirror_netloc = mirror_url_tokens.netloc - all_errors += '\n '+str(mirror_netloc)+': '+str(mirror_error) + all_errors += '\n ' + repr(mirror_netloc) + ': ' + repr(mirror_error) return all_errors diff --git a/tuf/_vendor/six.py b/tuf/_vendor/six.py new file mode 100644 index 00000000..019130f7 --- /dev/null +++ b/tuf/_vendor/six.py @@ -0,0 +1,646 @@ +"""Utilities for writing code that runs on Python 2 and 3""" + +# Copyright (c) 2010-2014 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.6.1" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + try: + result = self._resolve() + except ImportError: + # See the nice big comment in MovedModule.__getattr__. + raise AttributeError("%s could not be imported " % self.name) + setattr(obj, self.name, result) # Invokes __set__. + # This is a bit ugly, but it avoids running this again. + delattr(obj.__class__, self.name) + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + # It turns out many Python frameworks like to traverse sys.modules and + # try to load various attributes. This causes problems if this is a + # platform-specific module on the wrong platform, like _winreg on + # Unixes. Therefore, we silently pretend unimportable modules do not + # have any attributes. See issues #51, #53, #56, and #63 for the full + # tales of woe. + # + # First, if possible, avoid loading the module just to look at __file__, + # __name__, or __path__. + if (attr in ("__file__", "__name__", "__path__") and + self.mod not in sys.modules): + raise AttributeError(attr) + try: + _module = self._resolve() + except ImportError: + raise AttributeError(attr) + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + + +class _MovedItems(_LazyModule): + """Lazy loading of moved objects""" + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "xmlrpclib", "xmlrpc.server"), + MovedModule("winreg", "_winreg"), +] +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + sys.modules[__name__ + ".moves." + attr.name] = attr +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = sys.modules[__name__ + ".moves"] = _MovedItems(__name__ + ".moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +sys.modules[__name__ + ".moves.urllib_parse"] = sys.modules[__name__ + ".moves.urllib.parse"] = Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse") + + +class Module_six_moves_urllib_error(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +sys.modules[__name__ + ".moves.urllib_error"] = sys.modules[__name__ + ".moves.urllib.error"] = Module_six_moves_urllib_error(__name__ + ".moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +sys.modules[__name__ + ".moves.urllib_request"] = sys.modules[__name__ + ".moves.urllib.request"] = Module_six_moves_urllib_request(__name__ + ".moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +sys.modules[__name__ + ".moves.urllib_response"] = sys.modules[__name__ + ".moves.urllib.response"] = Module_six_moves_urllib_response(__name__ + ".moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +sys.modules[__name__ + ".moves.urllib_robotparser"] = sys.modules[__name__ + ".moves.urllib.robotparser"] = Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + parse = sys.modules[__name__ + ".moves.urllib_parse"] + error = sys.modules[__name__ + ".moves.urllib_error"] + request = sys.modules[__name__ + ".moves.urllib_request"] + response = sys.modules[__name__ + ".moves.urllib_response"] + robotparser = sys.modules[__name__ + ".moves.urllib_robotparser"] + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + + +sys.modules[__name__ + ".moves.urllib"] = Module_six_moves_urllib(__name__ + ".moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" + + _iterkeys = "keys" + _itervalues = "values" + _iteritems = "items" + _iterlists = "lists" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + _iterkeys = "iterkeys" + _itervalues = "itervalues" + _iteritems = "iteritems" + _iterlists = "iterlists" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +def iterkeys(d, **kw): + """Return an iterator over the keys of a dictionary.""" + return iter(getattr(d, _iterkeys)(**kw)) + +def itervalues(d, **kw): + """Return an iterator over the values of a dictionary.""" + return iter(getattr(d, _itervalues)(**kw)) + +def iteritems(d, **kw): + """Return an iterator over the (key, value) pairs of a dictionary.""" + return iter(getattr(d, _iteritems)(**kw)) + +def iterlists(d, **kw): + """Return an iterator over the (key, [values]) pairs of a dictionary.""" + return iter(getattr(d, _iterlists)(**kw)) + + +if PY3: + def b(s): + return s.encode("latin-1") + def u(s): + return s + unichr = chr + if sys.version_info[1] <= 1: + def int2byte(i): + return bytes((i,)) + else: + # This is about 2x faster than the implementation above on 3.2+ + int2byte = operator.methodcaller("to_bytes", 1, "big") + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO +else: + def b(s): + return s + # Workaround for standalone backslash + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + def byte2int(bs): + return ord(bs[0]) + def indexbytes(buf, i): + return ord(buf[i]) + def iterbytes(buf): + return (ord(byte) for byte in buf) + import StringIO + StringIO = BytesIO = StringIO.StringIO +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) + +_add_doc(reraise, """Reraise an exception.""") + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + return meta("NewBase", bases, {}) + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper diff --git a/tuf/compatibility/ssl_match_hostname.py b/tuf/_vendor/ssl_match_hostname.py similarity index 100% rename from tuf/compatibility/ssl_match_hostname.py rename to tuf/_vendor/ssl_match_hostname.py diff --git a/tuf/client/basic_client.py b/tuf/client/basic_client.py index c57852ca..0fa51d25 100755 --- a/tuf/client/basic_client.py +++ b/tuf/client/basic_client.py @@ -51,6 +51,14 @@ """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import sys import optparse import logging @@ -94,7 +102,7 @@ def update_client(repository_mirror): # Does 'repository_mirror' have the correct format? try: tuf.formats.URL_SCHEMA.check_match(repository_mirror) - except tuf.FormatError, e: + except tuf.FormatError as e: message = 'The repository mirror supplied is invalid.' raise tuf.RepositoryError(message) @@ -126,7 +134,7 @@ def update_client(repository_mirror): for target in updated_targets: try: updater.download_target(target, destination_directory) - except tuf.DownloadError, e: + except tuf.DownloadError as e: pass # Remove any files from the destination directory that are no longer being @@ -211,7 +219,8 @@ def parse_options(): # the current directory. try: update_client(repository_mirror) - except (tuf.NoWorkingMirrorError, tuf.RepositoryError), e: + + except (tuf.NoWorkingMirrorError, tuf.RepositoryError) as e: sys.stderr.write('Error: '+str(e)+'\n') sys.exit(1) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 726ce482..5f7d4802 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -99,12 +99,19 @@ updater.download_target(target, destination_directory) """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import errno import logging import os import shutil import time -import urllib import random import tuf @@ -120,6 +127,7 @@ import tuf.sig import tuf.util import tuf._vendor.iso8601 as iso8601 +import tuf._vendor.six as six logger = logging.getLogger('tuf.client.updater') @@ -505,7 +513,7 @@ def _import_delegations(self, parent_role): # Iterate through the keys of the delegated roles of 'parent_role' # and load them. - for keyid, keyinfo in keys_info.items(): + for keyid, keyinfo in six.iteritems(keys_info): if keyinfo['keytype'] in ['rsa', 'ed25519']: key = tuf.keys.format_metadata_to_key(keyinfo) @@ -517,13 +525,13 @@ def _import_delegations(self, parent_role): except tuf.KeyAlreadyExistsError: pass - except (tuf.FormatError, tuf.Error), e: + except (tuf.FormatError, tuf.Error) as e: logger.exception('Failed to add keyid: '+repr(keyid)+'.') logger.error('Aborting role delegation for parent role '+parent_role+'.') raise else: - logger.warn('Invalid key type for '+repr(keyid)+'.') + logger.warning('Invalid key type for '+repr(keyid)+'.') continue # Add the roles to the role database. @@ -535,8 +543,8 @@ def _import_delegations(self, parent_role): logger.debug('Adding delegated role: '+str(rolename)+'.') tuf.roledb.add_role(rolename, roleinfo) - except tuf.RoleAlreadyExistsError, e: - logger.warn('Role already exists: '+rolename) + except tuf.RoleAlreadyExistsError as e: + logger.warning('Role already exists: '+rolename) except: logger.exception('Failed to add delegated role: '+rolename+'.') @@ -642,7 +650,7 @@ def refresh(self, unsafely_update_root_if_necessary=True): self._update_metadata_if_changed('root') self._update_metadata_if_changed('targets') - except tuf.NoWorkingMirrorError, e: + except tuf.NoWorkingMirrorError as e: if unsafely_update_root_if_necessary: message = 'Valid top-level metadata cannot be downloaded. Unsafely '+\ 'update the Root metadata.' @@ -689,7 +697,7 @@ def _check_hashes(self, file_object, trusted_hashes): # Verify each trusted hash of 'trusted_hashes'. If all are valid, simply # return. - for algorithm, trusted_hash in trusted_hashes.items(): + for algorithm, trusted_hash in six.iteritems(trusted_hashes): digest_object = tuf.hash.digest(algorithm) digest_object.update(file_object.read()) computed_hash = digest_object.hexdigest() @@ -842,7 +850,7 @@ def verify_target_file(target_file_object): # 'compression' argument to _get_file() is needed only for decompression of # metadata. Target files may be compressed or uncompressed. if self.consistent_snapshot: - target_digest = random.choice(file_hashes.values()) + target_digest = random.choice(list(file_hashes.values())) dirname, basename = os.path.split(target_filepath) target_filepath = os.path.join(dirname, target_digest+'.'+basename) @@ -897,12 +905,12 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, None. """ - metadata = metadata_file_object.read() + metadata = metadata_file_object.read().decode('utf-8') try: metadata_signable = tuf.util.load_json_string(metadata) - except Exception, exception: + except Exception as exception: raise tuf.InvalidMetadataJSONError(exception) else: @@ -931,7 +939,7 @@ def _verify_uncompressed_metadata_file(self, metadata_file_object, # are not allowed. if metadata_signable['signed']['_type'] == 'Targets': if metadata_role != 'targets': - metadata_targets = metadata_signable['signed']['targets'].keys() + metadata_targets = list(metadata_signable['signed']['targets'].keys()) parent_rolename = tuf.roledb.get_parent_rolename(metadata_role) parent_role_metadata = self.metadata['current'][parent_rolename] parent_delegations = parent_role_metadata['delegations'] @@ -1198,7 +1206,7 @@ def _get_file(self, filepath, verify_file_function, file_type, # uncompressed version). verify_file_function(file_object) - except Exception, exception: + except Exception as exception: # Remember the error from this mirror, and "reset" the target file. logger.exception('Update failed from '+file_mirror+'.') file_mirror_errors[file_mirror] = exception @@ -1319,11 +1327,11 @@ def _update_metadata(self, metadata_role, uncompressed_fileinfo, if self.consistent_snapshot: if compression: filename_digest = \ - random.choice(compressed_fileinfo['hashes'].values()) + random.choice(list(compressed_fileinfo['hashes'].values())) else: filename_digest = \ - random.choice(uncompressed_fileinfo['hashes'].values()) + random.choice(list(uncompressed_fileinfo['hashes'].values())) dirname, basename = os.path.split(remote_filename) remote_filename = os.path.join(dirname, filename_digest+'.'+basename) @@ -1352,7 +1360,7 @@ def _update_metadata(self, metadata_role, uncompressed_fileinfo, # Next, move the verified updated metadata file to the 'current' directory. # Note that the 'move' method comes from tuf.util's TempFile class. # 'metadata_file_object' is an instance of tuf.util.TempFile. - metadata_signable = tuf.util.load_json_string(metadata_file_object.read()) + metadata_signable = tuf.util.load_json_string(metadata_file_object.read().decode('utf-8')) if compression == 'gzip': current_uncompressed_filepath = \ os.path.join(self.metadata_directory['current'], @@ -1605,7 +1613,7 @@ def _fileinfo_has_changed(self, metadata_filename, new_fileinfo): # without having that result in considering all files as needing to be # updated, or not all hash algorithms listed can be calculated on the # specific client. - for algorithm, hash_value in new_fileinfo['hashes'].items(): + for algorithm, hash_value in six.iteritems(new_fileinfo['hashes']): # We're only looking for a single match. This isn't a security # check, we just want to prevent unnecessary downloads. if algorithm in current_fileinfo['hashes']: @@ -1892,7 +1900,7 @@ def _refresh_targets_metadata(self, rolename='targets', include_delegations=Fals # See if this role provides metadata and, if we're including delegations, # look for metadata from delegated roles. role_prefix = rolename + '/' - for metadata_path in self.metadata['current']['snapshot']['meta'].keys(): + for metadata_path in six.iterkeys(self.metadata['current']['snapshot']['meta']): if metadata_path == rolename + '.json': roles_to_update.append(metadata_path[:-len('.json')]) elif include_delegations and metadata_path.startswith(role_prefix): @@ -2000,8 +2008,8 @@ def refresh_targets_metadata_chain(self, rolename): # This only goes to -1 because we only want to store the parents (so we # ignore the last element). for next_role in parts[1:-1]: - parent_roles.append(roles_added+'/'+next_role) - roles_added = roles_added+'/'+next_role + parent_roles.append(roles_added + '/' + next_role) + roles_added = roles_added + '/' + next_role message = 'Minimum metadata to download and set the chain of trust: '+\ repr(parent_roles)+'.' @@ -2010,13 +2018,13 @@ def refresh_targets_metadata_chain(self, rolename): # Check if 'snapshot.json' provides metadata for each of the roles in # 'parent_roles'. All the available roles on the repository are specified # in the 'snapshot.json' metadata. - targets_metadata_allowed = self.metadata['current']['snapshot']['meta'].keys() + targets_metadata_allowed = list(self.metadata['current']['snapshot']['meta'].keys()) for parent_role in parent_roles: parent_role = parent_role + '.json' if parent_role not in targets_metadata_allowed: message = '"snapshot.json" does not provide all the parent roles '+\ - 'of '+repr(rolename)+'.' + 'of ' + repr(rolename) + '.' raise tuf.RepositoryError(message) # Remove the 'targets' role because it gets updated when the targets.json @@ -2034,7 +2042,7 @@ def refresh_targets_metadata_chain(self, rolename): # Sort the roles so that parent roles always come first. parent_roles.sort() - logger.debug('Roles to update: '+repr(parent_roles)+'.') + logger.debug('Roles to update: ' + repr(parent_roles) + '.') # Iterate 'parent_roles', load each role's metadata file from disk, and # update it if it has changed. @@ -2108,7 +2116,7 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): return targets # Get the targets specified by the role itself. - for filepath, fileinfo in self.metadata['current'][rolename]['targets'].items(): + for filepath, fileinfo in six.iteritems(self.metadata['current'][rolename]['targets']): new_target = {} new_target['filepath'] = filepath new_target['fileinfo'] = fileinfo @@ -2209,7 +2217,7 @@ def target(self, target_filepath): # 'target_filepath' might contain URL encoding escapes. # http://docs.python.org/2/library/urllib.html#urllib.unquote - target_filepath = urllib.unquote(target_filepath) + target_filepath = six.moves.urllib.parse.unquote(target_filepath) if not target_filepath.startswith('/'): target_filepath = '/' + target_filepath @@ -2292,15 +2300,28 @@ def _preorder_depth_first_walk(self, target_filepath): if target is None: - # Push children in reverse order of appearance onto the stack. + child_roles_to_visit = [] # NOTE: This may be a slow operation if there are many delegated roles. - for child_role in reversed(child_roles): + for child_role in child_roles: child_role_name = self._visit_child_role(child_role, target_filepath) - if child_role_name is None: + if not child_role['backtrack'] and child_role_name is not None: + logger.debug('Adding child role '+repr(child_role_name)) + logger.debug('Not backtracking to other roles.') + role_names = [] + child_roles_to_visit.append(child_role_name) + break + + elif child_role_name is None: logger.debug('Skipping child role '+repr(child_role_name)) + else: logger.debug('Adding child role '+repr(child_role_name)) - role_names.append(child_role_name) + child_roles_to_visit.append(child_role_name) + + # Push 'child_roles_to_visit' in reverse order of appearance onto + # 'role_names'. Roles are popped from the end of the 'role_names' list. + child_roles_to_visit.reverse() + role_names.extend(child_roles_to_visit) else: logger.debug('Found target in current role '+repr(role_name)) @@ -2342,13 +2363,15 @@ def _get_target_from_targets_role(self, role_name, targets, target_filepath): target = None # Does the current role name have our target? - logger.debug('Asking role '+repr(role_name)+' about target '+\ + logger.debug('Asking role ' + repr(role_name) + ' about target '+\ repr(target_filepath)) - for filepath, fileinfo in targets.iteritems(): + + for filepath, fileinfo in six.iteritems(targets): if filepath == target_filepath: - logger.debug('Found target '+target_filepath+' in role '+role_name) + logger.debug('Found target ' + target_filepath + ' in role ' + role_name) target = {'filepath': filepath, 'fileinfo': fileinfo} break + else: logger.debug('No target '+target_filepath+' in role '+role_name) @@ -2425,16 +2448,17 @@ def _visit_child_role(self, child_role, target_filepath): # 'role_name' should have been validated when it was downloaded. # The 'paths' or 'path_hash_prefixes' fields should not be missing, # so we raise a format error here in case they are both missing. - raise tuf.FormatError(repr(child_role_name)+' has neither ' \ - '"paths" nor "path_hash_prefixes"!') + raise tuf.FormatError(repr(child_role_name) + ' has neither ' \ + '"paths" nor "path_hash_prefixes".') if child_role_is_relevant: - logger.debug('Child role '+repr(child_role_name)+' has target '+ + logger.debug('Child role ' + repr(child_role_name) + ' has target ' + \ repr(target_filepath)) return child_role_name + else: - logger.debug('Child role '+repr(child_role_name)+ - ' does not have target '+repr(target_filepath)) + logger.debug('Child role ' + repr(child_role_name) + \ + ' does not have target ' + repr(target_filepath)) return None @@ -2472,20 +2496,11 @@ def _get_target_hash(self, target_filepath, hash_function='sha256'): """ # Calculate the hash of the filepath to determine which bin to find the - # target. The client currently assumes the repository uses - # 'hash_function' to generate hashes. - + # target. The client currently assumes the repository (i.e., repository + # tool) uses 'hash_function' to generate hashes and UTF-8. digest_object = tuf.hash.digest(hash_function) - - try: - digest_object.update(target_filepath) - except UnicodeEncodeError: - # Sometimes, there are Unicode characters in target paths. We assume a - # UTF-8 encoding and try to hash that. - digest_object = tuf.hash.digest(hash_function) - encoded_target_filepath = target_filepath.encode('utf-8') - digest_object.update(encoded_target_filepath) - + encoded_target_filepath = target_filepath.encode('utf-8') + digest_object.update(encoded_target_filepath) target_filepath_hash = digest_object.hexdigest() return target_filepath_hash @@ -2528,24 +2543,24 @@ def remove_obsolete_targets(self, destination_directory): for role in tuf.roledb.get_rolenames(): if role.startswith('targets'): if role in self.metadata['previous'] and self.metadata['previous'][role] != None: - for target in self.metadata['previous'][role]['targets'].keys(): - if target not in self.metadata['current'][role]['targets'].keys(): + for target in self.metadata['previous'][role]['targets']: + if target not in self.metadata['current'][role]['targets']: # 'target' is only in 'previous', so remove it. - logger.warn('Removing obsolete file: '+repr(target)+'.') + logger.warning('Removing obsolete file: ' + repr(target) + '.') # Remove the file if it hasn't been removed already. destination = os.path.join(destination_directory, target) try: os.remove(destination) - except OSError, e: + except OSError as e: # If 'filename' already removed, just log it. if e.errno == errno.ENOENT: - logger.info('File '+repr(destination)+' was already removed.') + logger.info('File ' + repr(destination) + ' was already removed.') else: logger.error(str(e)) - except Exception, e: + except Exception as e: logger.error(str(e)) @@ -2607,7 +2622,7 @@ def updated_targets(self, targets, destination_directory): # Try one of the algorithm/digest combos for a mismatch. We break # as soon as we find a mismatch. - for algorithm, digest in target['fileinfo']['hashes'].items(): + for algorithm, digest in six.iteritems(target['fileinfo']['hashes']): digest_object = None try: digest_object = tuf.hash.digest_filename(target_filepath, @@ -2691,7 +2706,7 @@ def download_target(self, target, destination_directory): try: os.makedirs(target_dirpath) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass @@ -2699,6 +2714,6 @@ def download_target(self, target, destination_directory): raise else: - logger.warn(str(target_dirpath)+' does not exist.') + logger.warning(repr(target_dirpath) + ' does not exist.') target_file_object.move(destination) diff --git a/tuf/compatibility/__init__.py b/tuf/compatibility/__init__.py deleted file mode 100644 index 5ae1822b..00000000 --- a/tuf/compatibility/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -We copy some backwards compatibility from pip. - -https://github.com/pypa/pip/tree/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/backwardcompat -""" - - -import sys - - -if sys.version_info >= (3,): - import http.client as httplib - import urllib.parse as urlparse - import urllib.request as urllib2 -else: - import httplib - import urllib2 - import urlparse - - -## py25 has no builtin ssl module -## only >=py32 has ssl.match_hostname and ssl.CertificateError -try: - import ssl - try: - from ssl import match_hostname, CertificateError - except ImportError: - from tuf.compatibility.ssl_match_hostname import match_hostname, CertificateError -except ImportError: - ssl = None - - -# patch for py25 socket to work with http://pypi.python.org/pypi/ssl/ -import socket -if not hasattr(socket, 'create_connection'): # for Python 2.5 - # monkey-patch socket module - from tuf.compatibility.socket_create_connection import create_connection - socket.create_connection = create_connection - diff --git a/tuf/compatibility/socket_create_connection.py b/tuf/compatibility/socket_create_connection.py deleted file mode 100644 index 1a11f4bd..00000000 --- a/tuf/compatibility/socket_create_connection.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -We copy some functions from the Python 2.7.3 socket module. - -http://hg.python.org/releasing/2.7.3/file/7bb96963d067/Lib/socket.py -""" - - -_GLOBAL_DEFAULT_TIMEOUT = object() - - -def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, - source_address=None): - """Connect to *address* and return the socket object. - - Convenience function. Connect to *address* (a 2-tuple ``(host, - port)``) and return the socket object. Passing the optional - *timeout* parameter will set the timeout on the socket instance - before attempting to connect. If no *timeout* is supplied, the - global default timeout setting returned by :func:`getdefaulttimeout` - is used. If *source_address* is set it must be a tuple of (host, port) - for the socket to bind as a source address before making the connection. - An host of '' or port 0 tells the OS to use the default. - """ - - host, port = address - err = None - for res in getaddrinfo(host, port, 0, SOCK_STREAM): - af, socktype, proto, canonname, sa = res - sock = None - try: - sock = socket(af, socktype, proto) - if timeout is not _GLOBAL_DEFAULT_TIMEOUT: - sock.settimeout(timeout) - if source_address: - sock.bind(source_address) - sock.connect(sa) - return sock - - except error as _: - err = _ - if sock is not None: - sock.close() - - if err is not None: - raise err - else: - raise error("getaddrinfo returns an empty list") - diff --git a/tuf/conf.py b/tuf/conf.py index eec68c14..0d6e2a02 100755 --- a/tuf/conf.py +++ b/tuf/conf.py @@ -18,6 +18,14 @@ and cryptography libraries clients wish to use. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + # Set a directory that should be used for all temporary files. If this # is None, then the system default will be used. The system default # will also be used if a directory path set here is invalid or @@ -49,7 +57,7 @@ DEFAULT_ROOT_REQUIRED_LENGTH = 512000 #bytes # Set a timeout value in seconds (float) for non-blocking socket operations. -SOCKET_TIMEOUT = 1 #seconds +SOCKET_TIMEOUT = 2 #seconds # The maximum chunk of data, in bytes, we would download in every round. CHUNK_SIZE = 8192 #bytes @@ -59,7 +67,7 @@ MIN_AVERAGE_DOWNLOAD_SPEED = CHUNK_SIZE #bytes/second # The time (in seconds) we ignore a server with a slow initial retrieval speed. -SLOW_START_GRACE_PERIOD = 30 #seconds +SLOW_START_GRACE_PERIOD = 3 #seconds # The current "good enough" number of PBKDF2 passphrase iterations. # We recommend that important keys, such as root, be kept offline. diff --git a/tuf/download.py b/tuf/download.py index d1fe6539..0eab6606 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -13,21 +13,24 @@ See LICENSE for licensing information. - Perform any file downloads and check their validity. This means that the - hash and length of a downloaded file has to match the hash and length - supplied by the metadata of that file. The downloaded file is technically a - file-like object that will automatically destroys itself once closed. Note - that the file-like object, 'tuf.util.TempFile', is returned by the - '_download_file()' function. + Download metadata and target files and check their validity. The hash and + length of a downloaded file has to match the hash and length supplied by the + metadata of that file. The downloaded file is technically a file-like object + that will automatically destroys itself once closed. Note that the file-like + object, 'tuf.util.TempFile', is returned by the '_download_file()' function. """ -# Induce "true division" (http://www.python.org/dev/peps/pep-0238/). +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals -import httplib -import logging -import os.path +import os import socket +import logging import timeit import tuf @@ -35,324 +38,304 @@ import tuf.hash import tuf.util import tuf.formats - -from tuf.compatibility import httplib, ssl, urllib2, urlparse - -if ssl: - from tuf.compatibility import match_hostname -else: - raise tuf.Error("No SSL support!") # TODO: degrade gracefully - -# We will be overriding socket._fileobject to perform non-blocking socket -# reads. Therefore, we will need these global variables. -# http://hg.python.org/cpython/file/5be3fa83d436/Lib/socket.py#l84 +import tuf._vendor.six as six try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO + from ssl import match_hostname, CertificateError -try: - import errno except ImportError: - errno = None -EINTR = getattr(errno, 'EINTR', 4) + from tuf._vendor.ssl_match_hostname import match_hostname, CertificateError # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.download') +def safe_download(url, required_length): + """ + + Given the 'url' and 'required_length' of the desired file, open a connection + to 'url', download it, and return the contents of the file. Also ensure + the length of the downloaded file matches 'required_length' exactly. + tuf.download.unsafe_download() may be called if an upper download limit is + preferred. + + 'tuf.util.TempFile', the file-like object returned, is used instead of + regular tempfile object because of additional functionality provided, such + as handling compressed metadata and automatically closing files after + moving to final destination. + + + url: + A URL string that represents the location of the file. + + required_length: + An integer value representing the length of the file. This is an exact + limit. + + A 'tuf.util.TempFile' object is created on disk to store the contents of + 'url'. + + + tuf.DownloadLengthMismatchError, if there was a mismatch of observed vs + expected lengths while downloading the file. + + tuf.FormatError, if any of the arguments are improperly formatted. -class SaferSocketFileObject(socket._fileobject): - """We override socket._fileobject to produce a file-like object which reads - from a socket more safely than its ancestor. One the safety properties is - that reading from a socket must be a non-blocking operation.""" - - def __init__(self, sock, mode='rb', bufsize=-1, close=False): - super(SaferSocketFileObject, self).__init__(sock, mode=mode, - bufsize=bufsize, close=close) - - # Count the number of bytes received with this socket. - self.__number_of_bytes_received = 0 - # Count the seconds spent receiving with this socket. Tolerate servers with - # a slow start by ignoring their delivery speed for - # tuf.conf.SLOW_START_GRACE_PERIOD seconds. - assert tuf.conf.SLOW_START_GRACE_PERIOD > 0 - self.__seconds_spent_receiving = -tuf.conf.SLOW_START_GRACE_PERIOD - # Remember the time a clock was started. - self.__start_time = None + Any other unforeseen runtime exception. + + + A 'tuf.util.TempFile' file-like object that points to the contents of 'url'. + """ + + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True) - def __start_clock(self): - """ - - Start the clock to measure time difference later. +def unsafe_download(url, required_length): + """ + + Given the 'url' and 'required_length' of the desired file, open a connection + to 'url', download it, and return the contents of the file. Also ensure + the length of the downloaded file is up to 'required_length', and no larger. + tuf.download.safe_download() may be called if an exact download limit is + preferred. + + 'tuf.util.TempFile', the file-like object returned, is used instead of + regular tempfile object because of additional functionality provided, such + as handling compressed metadata and automatically closing files after + moving to final destination. + + + url: + A URL string that represents the location of the file. + + required_length: + An integer value representing the length of the file. This is an upper + limit. - - None. + + A 'tuf.util.TempFile' object is created on disk to store the contents of + 'url'. + + + tuf.DownloadLengthMismatchError, if there was a mismatch of observed vs + expected lengths while downloading the file. + + tuf.FormatError, if any of the arguments are improperly formatted. - - AssertionError: - When any internal condition is not true. - - - Start time is kept inside this object. - - - None. - """ - - # We must have reset the clock before this. - assert self.__start_time is None - # We use (platform-specific) wall time, so it will be imprecise sometimes. - self.__start_time = timeit.default_timer() + Any other unforeseen runtime exception. + + + A 'tuf.util.TempFile' file-like object that points to the contents of 'url'. + """ + + return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=False) - def __stop_clock_and_check_speed(self, data_length): - """ - - Stop the clock and try to detect slow retrieval. +def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): + """ + + Given the url, hashes and length of the desired file, this function + opens a connection to 'url' and downloads the file while ensuring its + length and hashes match 'required_hashes' and 'required_length'. + + tuf.util.TempFile is used instead of regular tempfile object because of + additional functionality provided by 'tuf.util.TempFile'. + + + url: + A URL string that represents the location of the file. + + required_length: + An integer value representing the length of the file. - - data_length: - A non-negative integer indicating the size of data retrieved in bytes. + STRICT_REQUIRED_LENGTH: + A Boolean indicator used to signal whether we should perform strict + checking of required_length. True by default. We explicitly set this to + False when we know that we want to turn this off for downloading the + timestamp metadata, which has no signed required_length. - - tuf.SlowRetrievalError: - If the average download speed falls below - 'tuf.conf.MIN_AVERAGE_DOWNLOAD_SPEED'. + + A 'tuf.util.TempFile' object is created on disk to store the contents of + 'url'. + + + tuf.DownloadLengthMismatchError, if there was a mismatch of observed vs + expected lengths while downloading the file. + + tuf.FormatError, if any of the arguments are improperly formatted. - AssertionError: - When any internal condition is not true. + Any other unforeseen runtime exception. + + + A 'tuf.util.TempFile' file-like object that points to the contents of 'url'. + """ - - Start time is cleared inside this object. + # Do all of the arguments have the appropriate format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.URL_SCHEMA.check_match(url) + tuf.formats.LENGTH_SCHEMA.check_match(required_length) - - None. - """ + # 'url.replace()' is for compatibility with Windows-based systems because + # they might put back-slashes in place of forward-slashes. This converts it + # to the common format. + url = url.replace('\\', '/') + logger.info('Downloading: '+str(url)) - # We use (platform-specific) wall time, so it will be imprecise sometimes. - stop_time = timeit.default_timer() - # We must have already started the clock. - assert self.__start_time > 0 - time_delta = stop_time-self.__start_time - # Reset the clock. - self.__start_time = None + # This is the temporary file that we will return to contain the contents of + # the downloaded file. + temp_file = tuf.util.TempFile() - # Measure the average download speed. - self.__number_of_bytes_received += data_length - self.__seconds_spent_receiving += time_delta + try: + # Open the connection to the remote file. + connection = _open_connection(url) - # self.__seconds_spent_receiving begins at negative - # 'tuf.conf.SLOW_START_GRACE_PERIOD'. - if self.__seconds_spent_receiving > 0: - average_download_speed = \ - self.__number_of_bytes_received/self.__seconds_spent_receiving + # We ask the server about how big it thinks this file should be. + reported_length = _get_content_length(connection) - # If the average download speed is below a certain threshold, we flag this - # as a possible slow-retrieval attack. This threshold will determine our - # bias: if it is too low, we will have more false positives; if it is too - # high, we will have more false negatives. + # Then, we check whether the required length matches the reported length. + _check_content_length(reported_length, required_length, + STRICT_REQUIRED_LENGTH) + + # Download the contents of the URL, up to the required length, to a + # temporary file, and get the total number of downloaded bytes. + total_downloaded = _download_fixed_amount_of_data(connection, temp_file, + required_length) + + # Does the total number of downloaded bytes match the required length? + _check_downloaded_length(total_downloaded, required_length, + STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH) + + except: + # Close 'temp_file'; any written data is lost. + temp_file.close_temp_file() + logger.exception('Could not download URL: '+str(url)) + raise + + else: + return temp_file + + + + + +def _download_fixed_amount_of_data(connection, temp_file, required_length): + """ + + This is a helper function, where the download really happens. While-block + reads data from connection a fixed chunk of data at a time, or less, until + 'required_length' is reached. + + + connection: + The object that the _open_connection returns for communicating with the + server about the contents of a URL. + + temp_file: + A temporary file where the contents at the URL specified by the + 'connection' object will be stored. + + required_length: + The number of bytes that we must download for the file. This is almost + always specified by the TUF metadata for the data file in question + (except in the case of timestamp metadata, in which case we would fix a + reasonable upper bound). + + + Data from the server will be written to 'temp_file'. + + + Runtime or network exceptions will be raised without question. + + + total_downloaded: + The total number of bytes downloaded for the desired file. + """ + + # Tolerate servers with a slow start by ignoring their delivery speed for + # 'tuf.conf.SLOW_START_GRACE_PERIOD' seconds. Set 'seconds_spent_receiving' + # to negative SLOW_START_GRACE_PERIOD seconds, and begin checking the average + # download speed once it is positive. + grace_period = -tuf.conf.SLOW_START_GRACE_PERIOD + + # Keep track of total bytes downloaded. + number_of_bytes_received = 0 + + start_time = timeit.default_timer() + + try: + while True: + # We download a fixed chunk of data in every round. This is so that we + # can defend against slow retrieval attacks. Furthermore, we do not wish + # to download an extremely large file in one shot. + data = b'' + read_amount = min(tuf.conf.CHUNK_SIZE, + required_length - number_of_bytes_received) + #logger.debug('Reading next chunk...') + + try: + data = connection.read(read_amount) + + # Python 3.2 returns 'IOError' if the remote file object has timed out. + except (socket.error, IOError): + pass + + number_of_bytes_received = number_of_bytes_received + len(data) + + # Data successfully read from the connection. Store it. + temp_file.write(data) + + if number_of_bytes_received == required_length: + break + + stop_time = timeit.default_timer() + seconds_spent_receiving = stop_time - start_time + + if (seconds_spent_receiving + grace_period) < 0: + #logger.debug('Ignoring average download speed for another: '+\ + #str(-seconds_spent_receiving) + ' seconds') + continue + + # Measure the average download speed. + average_download_speed = number_of_bytes_received / seconds_spent_receiving + + # If the average download speed is below a certain threshold, we flag + # this as a possible slow-retrieval attack. if average_download_speed < tuf.conf.MIN_AVERAGE_DOWNLOAD_SPEED: - raise tuf.SlowRetrievalError(average_download_speed) + break + else: logger.debug('Good average download speed: '+\ - str(average_download_speed)+' bytes/second') - else: - logger.debug('Ignoring average download speed for another: '+\ - str(-self.__seconds_spent_receiving)+' seconds') - - - - - - - def read(self, size): - """ - - We override the ancestor read (socket._fileobject.read) operation to be a - non-blocking operation. - - Original code is at: - http://hg.python.org/cpython/file/5be3fa83d436/Lib/socket.py#l336 - - - size: - The length of the data chunk that we would like to download. We assume - that the size of the expected data chunk is accurate; otherwise, we are - liable to miscount the number of truly slowly-retrieved chunks. - - - tuf.SlowRetrievalError, in case we detect a slow-retrieval attack. - - Any other exception thrown by socket._fileobject.read. - - - None. - - - Received data up to 'size' bytes. - """ - - # We should never try to specify a negative size. - assert size >= 0 - - # Use max, disallow tiny reads in a loop as they are very inefficient. - # We never leave read() with any leftover data from a new recv() call - # in our internal buffer. - rbufsize = max(self._rbufsize, self.default_bufsize) - # Our use of StringIO rather than lists of string objects returned by - # recv() minimizes memory usage and fragmentation that occurs when - # rbufsize is large compared to the typical return value of recv(). - buf = self._rbuf - buf.seek(0, 2) # seek end - - # Read until size bytes or EOF seen, whichever comes first - buf_len = buf.tell() - if buf_len >= size: - # Already have size bytes in our buffer? Extract and return. - buf.seek(0) - rv = buf.read(size) - self._rbuf = StringIO() - self._rbuf.write(buf.read()) - return rv - - self._rbuf = StringIO() # reset _rbuf. we consume it via buf. - # Since we try to detect slow retrieval, this should not be an infinite loop. - while True: - left = size - buf_len - # recv() will malloc the amount of memory given as its - # parameter even though it often returns much less data - # than that. The returned data string is short lived - # as we copy it into a StringIO and free it. This avoids - # fragmentation issues on many platforms. - try: - self.__start_clock() - data = self._sock.recv(left) - except socket.timeout: - self.__stop_clock_and_check_speed(0) - continue - except socket.error, e: - if e.args[0] == EINTR: - self.__stop_clock_and_check_speed(0) - continue - raise - else: - self.__stop_clock_and_check_speed(len(data)) + str(average_download_speed) + ' bytes per second') + + # We might have no more data to read. Check number of bytes downloaded. if not data: + message = 'Downloaded '+str(number_of_bytes_received)+'/'+ \ + str(required_length)+' bytes.' + logger.debug(message) + + # Finally, we signal that the download is complete. break - n = len(data) - if n == size and not buf_len: - # Shortcut. Avoid buffer data copies when: - # - We have no data in our buffer. - # AND - # - Our call to recv returned exactly the - # number of bytes we were asked to read. - return data - if n == left: - buf.write(data) - del data # explicit free - break - assert n <= left, "recv(%d) returned %d bytes" % (left, n) - buf.write(data) - buf_len += n - del data # explicit free - #assert buf_len == buf.tell() - return buf.getvalue() - - - - - -class SaferHTTPResponse(httplib.HTTPResponse): - """A safer version of httplib.HTTPResponse, in which we only use safe socket - file-like objects.""" - - def __init__(self, sock, debuglevel=0, strict=0, method=None, - buffering=False): - httplib.HTTPResponse.__init__(self, sock, debuglevel, - strict, method) - - # Delete the previous socket file-like object... - del self.fp - # ...and replace it with our safer version. - if buffering: - self.fp = SaferSocketFileObject(sock._sock, 'rb') - else: - self.fp = SaferSocketFileObject(sock._sock, 'rb', 0) - - - - - -class VerifiedHTTPSConnection(httplib.HTTPSConnection): - """ - A connection that wraps connections with ssl certificate verification. - - https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L72 - """ - - def connect(self): - - self.connection_kwargs = {} - - #TODO: refactor compatibility logic into tuf.compatibility? - - # for > py2.5 - if hasattr(self, 'timeout'): - self.connection_kwargs.update(timeout = self.timeout) - - # for >= py2.7 - if hasattr(self, 'source_address'): - self.connection_kwargs.update(source_address = self.source_address) - - sock = socket.create_connection((self.host, self.port), **self.connection_kwargs) - - # for >= py2.7 - if getattr(self, '_tunnel_host', None): - self.sock = sock - self._tunnel() - - # set location of certificate authorities - assert os.path.isfile( tuf.conf.ssl_certificates ) - cert_path = tuf.conf.ssl_certificates - - # TODO: Disallow SSLv2. - # http://docs.python.org/dev/library/ssl.html#protocol-versions - # TODO: Select the right ciphers. - # http://docs.python.org/dev/library/ssl.html#cipher-selection - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=cert_path) - - match_hostname(self.sock.getpeercert(), self.host) - - - - - -class VerifiedHTTPSHandler(urllib2.HTTPSHandler): - """ - A HTTPSHandler that uses our own VerifiedHTTPSConnection. - - https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L109 - """ - - def __init__(self, connection_class = VerifiedHTTPSConnection): - self.specialized_conn_class = connection_class - urllib2.HTTPSHandler.__init__(self) - - def https_open(self, req): - return self.do_open(self.specialized_conn_class, req) + + except: + raise + + else: + # This else block returns and skips closing the connection in the finally + # block, so close the connection here. + connection.close() + return number_of_bytes_received + + finally: + # Whatever happens, make sure that we always close the connection. + connection.close() @@ -366,7 +349,7 @@ def _get_request(url): https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L147 """ - return urllib2.Request(url, headers={'Accept-encoding': 'identity'}) + return six.moves.urllib.request.Request(url, headers={'Accept-encoding': 'identity'}) @@ -385,15 +368,16 @@ def _get_opener(scheme=None): # If we are going over https, use an opener which will provide SSL # certificate verification. https_handler = VerifiedHTTPSHandler() - opener = urllib2.build_opener(https_handler) + opener = six.moves.urllib.request.build_opener(https_handler) # strip out HTTPHandler to prevent MITM spoof for handler in opener.handlers: - if isinstance(handler, urllib2.HTTPHandler): + if isinstance(handler, six.moves.urllib.request.HTTPHandler): opener.handlers.remove(handler) + else: # Otherwise, use the default opener. - opener = urllib2.build_opener() + opener = six.moves.urllib.request.build_opener() return opener @@ -434,84 +418,11 @@ def _open_connection(url): # servers do not recognize connections that originates from # Python-urllib/x.y. - parsed_url = urlparse.urlparse(url) + parsed_url = six.moves.urllib.parse.urlparse(url) opener = _get_opener(scheme=parsed_url.scheme) request = _get_request(url) - return opener.open(request) - - - - - -def _download_fixed_amount_of_data(connection, temp_file, required_length): - """ - - This is a helper function, where the download really happens. While-block - reads data from connection a fixed chunk of data at a time, or less, until - 'required_length' is reached. - - connection: - The object that the _open_connection returns for communicating with the - server about the contents of a URL. - - temp_file: - A temporary file where the contents at the URL specified by the - 'connection' object will be stored. - - required_length: - The number of bytes that we must download for the file. This is almost - always specified by the TUF metadata for the data file in question - (except in the case of timestamp metadata, in which case we would fix a - reasonable upper bound). - - - Data from the server will be written to 'temp_file'. - - - Runtime or network exceptions will be raised without question. - - - total_downloaded: - The total number of bytes we have downloaded for the desired file and - which should be equal to 'required_length'. - """ - - # Keep track of total bytes downloaded. - total_downloaded = 0 - - try: - while True: - # We download a fixed chunk of data in every round. This is so that we - # can defend against slow retrieval attacks. Furthermore, we do not wish - # to download an extremely large file in one shot. - amount_to_read = min(tuf.conf.CHUNK_SIZE, - required_length-total_downloaded) - logger.debug('Reading next chunk...') - data = connection.read(amount_to_read) - - # We might have no more data to read. Check number of bytes downloaded. - if not data: - message = 'Downloaded '+str(total_downloaded)+'/'+ \ - str(required_length)+' bytes.' - logger.debug(message) - - # Finally, we signal that the download is complete. - break - - # Data successfully read from the connection. Store it. - temp_file.write(data) - total_downloaded = total_downloaded + len(data) - - except: - raise - - else: - return total_downloaded - - finally: - # Whatever happens, make sure that we always close the connection. - connection.close() + return opener.open(request, timeout = tuf.conf.SOCKET_TIMEOUT) @@ -542,14 +453,19 @@ def _get_content_length(connection): try: # What is the length of this document according to the HTTP spec? reported_length = connection.info().get('Content-Length') + # Try casting it as a decimal number. reported_length = int(reported_length, 10) + # Make sure that it is a nonnegative integer. assert reported_length > -1 + except: - logger.exception('Could not get content length about '+str(connection)+ - ' from server!') + message = \ + 'Could not get content length about ' + str(connection) + ' from server.' + logger.exception(message) reported_length = None + finally: return reported_length @@ -653,7 +569,7 @@ def _check_downloaded_length(total_downloaded, required_length, logger.info('Downloaded '+str(total_downloaded)+' bytes out of the '+\ 'expected '+str(required_length)+ ' bytes.') else: - difference_in_bytes = abs(total_downloaded-required_length) + difference_in_bytes = abs(total_downloaded - required_length) # What we downloaded is not equal to the required length, but did we ask # for strict checking of required length? @@ -679,117 +595,61 @@ def _check_downloaded_length(total_downloaded, required_length, -def safe_download(url, required_length): - return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True) - - - - -def unsafe_download(url, required_length): - return _download_file(url, required_length, STRICT_REQUIRED_LENGTH=False) - - - - - -def _download_file(url, required_length, STRICT_REQUIRED_LENGTH=True): +class VerifiedHTTPSConnection(six.moves.http_client.HTTPSConnection): """ - - Given the url, hashes and length of the desired file, this function - opens a connection to 'url' and downloads the file while ensuring its - length and hashes match 'required_hashes' and 'required_length'. - - tuf.util.TempFile is used instead of regular tempfile object because of - additional functionality provided by 'tuf.util.TempFile'. - - - url: - A URL string that represents the location of the file. - - required_length: - An integer value representing the length of the file. + A connection that wraps connections with ssl certificate verification. - STRICT_REQUIRED_LENGTH: - A Boolean indicator used to signal whether we should perform strict - checking of required_length. True by default. We explicitly set this to - False when we know that we want to turn this off for downloading the - timestamp metadata, which has no signed required_length. - - - A 'tuf.util.TempFile' object is created on disk to store the contents of - 'url'. - - - tuf.DownloadLengthMismatchError, if there was a mismatch of observed vs - expected lengths while downloading the file. - - tuf.FormatError, if any of the arguments are improperly formatted. - - Any other unforeseen runtime exception. - - - A 'tuf.util.TempFile' file-like object which points to the contents of - 'url'. + https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L72 """ - # Do all of the arguments have the appropriate format? - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.URL_SCHEMA.check_match(url) - tuf.formats.LENGTH_SCHEMA.check_match(required_length) + def connect(self): - # 'url.replace()' is for compatibility with Windows-based systems because - # they might put back-slashes in place of forward-slashes. This converts it - # to the common format. - url = url.replace('\\', '/') - logger.info('Downloading: '+str(url)) + self.connection_kwargs = {} - # NOTE: Not thread-safe. - # Save current values or functions for restoration later. - previous_socket_timeout = socket.getdefaulttimeout() - previous_http_response_class = httplib.HTTPConnection.response_class + # for > py2.5 + if hasattr(self, 'timeout'): + self.connection_kwargs.update(timeout = self.timeout) - # This is the temporary file that we will return to contain the contents of - # the downloaded file. - temp_file = tuf.util.TempFile() + # for >= py2.7 + if hasattr(self, 'source_address'): + self.connection_kwargs.update(source_address = self.source_address) - try: - # NOTE: Not thread-safe. - # Set timeout to induce non-blocking socket operations. - socket.setdefaulttimeout(tuf.conf.SOCKET_TIMEOUT) - # Replace the socket file-like object class with our safer version. - httplib.HTTPConnection.response_class = SaferHTTPResponse + sock = socket.create_connection((self.host, self.port), **self.connection_kwargs) - # Open the connection to the remote file. - connection = _open_connection(url) + # for >= py2.7 + if getattr(self, '_tunnel_host', None): + self.sock = sock + self._tunnel() - # We ask the server about how big it thinks this file should be. - reported_length = _get_content_length(connection) + # set location of certificate authorities + assert os.path.isfile( tuf.conf.ssl_certificates ) + cert_path = tuf.conf.ssl_certificates - # Then, we check whether the required length matches the reported length. - _check_content_length(reported_length, required_length, - STRICT_REQUIRED_LENGTH) + # TODO: Disallow SSLv2. + # http://docs.python.org/dev/library/ssl.html#protocol-versions + # TODO: Select the right ciphers. + # http://docs.python.org/dev/library/ssl.html#cipher-selection + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=cert_path) - # Download the contents of the URL, up to the required length, to a - # temporary file, and get the total number of downloaded bytes. - total_downloaded = _download_fixed_amount_of_data(connection, temp_file, - required_length) + match_hostname(self.sock.getpeercert(), self.host) - # Does the total number of downloaded bytes match the required length? - _check_downloaded_length(total_downloaded, required_length, - STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH) - except: - # Close 'temp_file'; any written data is lost. - temp_file.close_temp_file() - logger.exception('Could not download URL: '+str(url)) - raise - else: - return temp_file - finally: - # NOTE: Not thread-safe. - # Restore previously saved values or functions. - httplib.HTTPConnection.response_class = previous_http_response_class - socket.setdefaulttimeout(previous_socket_timeout) + +class VerifiedHTTPSHandler(six.moves.urllib.request.HTTPSHandler): + """ + A HTTPSHandler that uses our own VerifiedHTTPSConnection. + + https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L109 + """ + + def __init__(self, connection_class = VerifiedHTTPSConnection): + self.specialized_conn_class = connection_class + six.moves.urllib.request.HTTPSHandler.__init__(self) + + def https_open(self, req): + return self.do_open(self.specialized_conn_class, req) diff --git a/tuf/ed25519_keys.py b/tuf/ed25519_keys.py index 82dd9c2a..381dfbdf 100755 --- a/tuf/ed25519_keys.py +++ b/tuf/ed25519_keys.py @@ -49,6 +49,7 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals # 'binascii' required for hexadecimal conversions. Signatures and # public/private keys are hexlified. @@ -84,6 +85,10 @@ # TODO: Version 0.2.3 of 'pynacl' prints: "UserWarning: reimporting '...' might # overwrite older definitions." when importing 'nacl.signing'. Suppress user # warnings temporarily (at least until this issue is fixed by PyNaCl). +# +# Note: A 'pragma: no cover' comment is intended for test 'coverage'. Lines +# or code blocks with this comment should not be flagged as uncovered. +# pynacl will always be install prior to running the unit tests. with warnings.catch_warnings(): warnings.simplefilter('ignore') try: @@ -92,7 +97,7 @@ # PyNaCl's 'cffi' dependency may raise an 'IOError' exception when importing # 'nacl.signing'. - except (ImportError, IOError): + except (ImportError, IOError): # pragma: no cover pass # The optimized pure Python implementation of ed25519 provided by TUF. If @@ -163,9 +168,9 @@ def generate_public_and_private(): # key generation. try: nacl_key = nacl.signing.SigningKey(seed) - public = str(nacl_key.verify_key) + public = nacl_key.verify_key.encode(encoder=nacl.encoding.RawEncoder()) - except NameError: + except NameError: # pragma: no cover message = 'The PyNaCl library and/or its dependencies unavailable.' raise tuf.UnsupportedLibraryError(message) @@ -187,7 +192,7 @@ def create_signature(public_key, private_key, data): A signature is a 64-byte string. >>> public, private = generate_public_and_private() - >>> data = 'The quick brown fox jumps over the lazy dog' + >>> data = b'The quick brown fox jumps over the lazy dog' >>> signature, method = \ create_signature(public, private, data) >>> tuf.formats.ED25519SIGNATURE_SCHEMA.matches(signature) @@ -250,13 +255,13 @@ def create_signature(public_key, private_key, data): nacl_sig = nacl_key.sign(data) signature = nacl_sig.signature - except NameError: + except NameError: # pragma: no cover message = 'The PyNaCl library and/or its dependencies unavailable.' raise tuf.UnsupportedLibraryError(message) - except (ValueError, TypeError, nacl.exceptions.CryptoError): + except (ValueError, TypeError, nacl.exceptions.CryptoError) as e: message = 'An "ed25519" signature could not be created with PyNaCl.' - raise tuf.CryptoError(message) + raise tuf.CryptoError(message + str(e)) return signature, method @@ -272,14 +277,14 @@ def verify_signature(public_key, method, signature, data, use_pynacl=False): 'sig', and 'data' arguments to complete the verification. >>> public, private = generate_public_and_private() - >>> data = 'The quick brown fox jumps over the lazy dog' + >>> data = b'The quick brown fox jumps over the lazy dog' >>> signature, method = \ create_signature(public, private, data) >>> verify_signature(public, method, signature, data, use_pynacl=False) True >>> verify_signature(public, method, signature, data, use_pynacl=True) True - >>> bad_data = 'The sly brown fox jumps over the lazy dog' + >>> bad_data = b'The sly brown fox jumps over the lazy dog' >>> bad_signature, method = \ create_signature(public, private, bad_data) >>> verify_signature(public, method, bad_signature, data, use_pynacl=False) @@ -348,10 +353,9 @@ def verify_signature(public_key, method, signature, data, use_pynacl=False): try: nacl_verify_key = nacl.signing.VerifyKey(public) nacl_message = nacl_verify_key.verify(data, signature) - if nacl_message == data: - valid_signature = True + valid_signature = True - except NameError: + except NameError: # pragma: no cover message = 'The PyNaCl library and/or its dependencies unavailable.' raise tuf.UnsupportedLibraryError(message) @@ -366,8 +370,9 @@ def verify_signature(public_key, method, signature, data, use_pynacl=False): # The pure Python implementation raises 'Exception' if 'signature' is # invalid. - except Exception, e: + except Exception as e: pass + else: message = 'Unsupported ed25519 signing method: '+repr(method)+'.\n'+ \ 'Supported methods: '+repr(_SUPPORTED_ED25519_SIGNING_METHODS)+'.' diff --git a/tuf/formats.py b/tuf/formats.py index 7c9f58fc..7d039c63 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ formats.py @@ -60,6 +62,14 @@ signable_object = make_signable(unsigned_object) """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import binascii import calendar import re @@ -69,7 +79,7 @@ import tuf import tuf.schema as SCHEMA - +import tuf._vendor.six as six # Note that in the schema definitions below, the 'SCHEMA.Object' types allow # additional keys which are not defined. Thus, any additions to them will be @@ -146,7 +156,7 @@ # The contents of an encrypted TUF key. Encrypted TUF keys are saved to files # in this format. -ENCRYPTEDKEY_SCHEMA = SCHEMA.AnyString() +ENCRYPTEDKEY_SCHEMA = SCHEMA.AnyBytes() # A value that is either True or False, on or off, etc. BOOLEAN_SCHEMA = SCHEMA.Boolean() @@ -169,7 +179,7 @@ NUMBINS_SCHEMA = SCHEMA.Integer(lo=1) # A PyCrypto signature. -PYCRYPTOSIGNATURE_SCHEMA = SCHEMA.AnyString() +PYCRYPTOSIGNATURE_SCHEMA = SCHEMA.AnyBytes() # An RSA key in PEM format. PEMRSA_SCHEMA = SCHEMA.AnyString() @@ -219,13 +229,13 @@ keyval = KEYVAL_SCHEMA) # An ED25519 raw public key, which must be 32 bytes. -ED25519PUBLIC_SCHEMA = SCHEMA.LengthString(32) +ED25519PUBLIC_SCHEMA = SCHEMA.LengthBytes(32) # An ED25519 raw seed key, which must be 32 bytes. -ED25519SEED_SCHEMA = SCHEMA.LengthString(32) +ED25519SEED_SCHEMA = SCHEMA.LengthBytes(32) # An ED25519 raw signature, which must be 64 bytes. -ED25519SIGNATURE_SCHEMA = SCHEMA.LengthString(64) +ED25519SIGNATURE_SCHEMA = SCHEMA.LengthBytes(64) # Required installation libraries expected by the repository tools and other # cryptography modules. @@ -359,6 +369,7 @@ name = SCHEMA.Optional(ROLENAME_SCHEMA), keyids = KEYIDS_SCHEMA, threshold = THRESHOLD_SCHEMA, + backtrack = SCHEMA.Optional(BOOLEAN_SCHEMA), paths = SCHEMA.Optional(RELPATHS_SCHEMA), path_hash_prefixes = SCHEMA.Optional(PATH_HASH_PREFIXES_SCHEMA)) @@ -494,7 +505,8 @@ class MetaFile(object): def __eq__(self, other): return isinstance(other, MetaFile) and self.info == other.info - + + __hash__ = None def __ne__(self, other): return not self.__eq__(other) @@ -506,10 +518,12 @@ def __getattr__(self, name): referred to directly without the info dict. The info dict is just to be able to do the __eq__ comparison generically. """ - + if name in self.info: return self.info[name] - raise AttributeError, name + + else: + raise AttributeError(name) @@ -735,7 +749,7 @@ def datetime_to_unix_timestamp(datetime_object): timestamp. For example, Python's time.time() returns a Unix timestamp, and includes the number of microseconds. 'datetime_object' is converted to UTC. - >>> datetime_object = datetime.datetime(1985, 10, 26, 01, 22) + >>> datetime_object = datetime.datetime(1985, 10, 26, 1, 22) >>> timestamp = datetime_to_unix_timestamp(datetime_object) >>> timestamp 499137720 @@ -820,7 +834,7 @@ def format_base64(data): data: - A string or buffer of data to convert. + Binary or buffer of data to convert. tuf.FormatError, if the base64 encoding fails or the argument @@ -834,9 +848,9 @@ def format_base64(data): """ try: - return binascii.b2a_base64(data).rstrip('=\n ') + return binascii.b2a_base64(data).decode('utf-8').rstrip('=\n ') - except (TypeError, binascii.Error), e: + except (TypeError, binascii.Error) as e: raise tuf.FormatError('Invalid base64 encoding: '+str(e)) @@ -864,7 +878,7 @@ def parse_base64(base64_string): 'base64_string'. """ - if not isinstance(base64_string, basestring): + if not isinstance(base64_string, six.string_types): message = 'Invalid argument: '+repr(base64_string) raise tuf.FormatError(message) @@ -874,9 +888,9 @@ def parse_base64(base64_string): base64_string = base64_string + padding try: - return binascii.a2b_base64(base64_string) + return binascii.a2b_base64(base64_string.encode('utf-8')) - except (TypeError, binascii.Error), e: + except (TypeError, binascii.Error) as e: raise tuf.FormatError('Invalid base64 encoding: '+str(e)) @@ -1194,10 +1208,8 @@ def _canonical_string_encoder(string): """ string = '"%s"' % re.sub(r'(["\\])', r'\\\1', string) - if isinstance(string, unicode): - return string.encode('utf-8') - else: - return string + + return string @@ -1207,7 +1219,7 @@ def _encode_canonical(object, output_function): # Helper for encode_canonical. Older versions of json.encoder don't # even let us replace the separators. - if isinstance(object, basestring): + if isinstance(object, six.string_types): output_function(_canonical_string_encoder(object)) elif object is True: output_function("true") @@ -1215,7 +1227,7 @@ def _encode_canonical(object, output_function): output_function("false") elif object is None: output_function("null") - elif isinstance(object, (int, long)): + elif isinstance(object, six.integer_types): output_function(str(object)) elif isinstance(object, (tuple, list)): output_function("[") @@ -1228,8 +1240,7 @@ def _encode_canonical(object, output_function): elif isinstance(object, dict): output_function("{") if len(object): - items = object.items() - items.sort() + items = sorted(six.iteritems(object)) for key, value in items[:-1]: output_function(_canonical_string_encoder(key)) output_function(":") diff --git a/tuf/hash.py b/tuf/hash.py index 4fe96455..172d00e1 100755 --- a/tuf/hash.py +++ b/tuf/hash.py @@ -18,17 +18,25 @@ available to TUF, simplifying the creation of digest objects, and providing a central location for hash routines are the main goals of this module. Support routines implemented include functions to - create digest objects given a filename or file object. - Hashlib and pycrypto hash algorithms currently supported. - + create digest objects given a filename or file object. Hashlib and PyCrypto + hash algorithms currently supported. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals import logging # Import tuf Exceptions. import tuf import tuf.log +import tuf._vendor.six as six + # Import tuf logger to log warning messages. logger = logging.getLogger('tuf.hash') @@ -126,7 +134,6 @@ def digest(algorithm=_DEFAULT_HASH_ALGORITHM, Digest object (e.g., hashlib.new(algorithm) or algorithm.new() # pycrypto). - """ # Was a hashlib digest object requested and is it supported? @@ -189,6 +196,7 @@ def digest_fileobject(file_object, algorithm=_DEFAULT_HASH_ALGORITHM, tuf.UnsupportedAlgorithmError + tuf.Error @@ -197,7 +205,6 @@ def digest_fileobject(file_object, algorithm=_DEFAULT_HASH_ALGORITHM, Digest object (e.g., hashlib.new(algorithm) or algorithm.new() # pycrypto). - """ # Digest object returned whose hash will be updated using 'file_object'. @@ -218,7 +225,13 @@ def digest_fileobject(file_object, algorithm=_DEFAULT_HASH_ALGORITHM, data = file_object.read(chunksize) if not data: break - digest_object.update(data_to_string(data)) + + if not isinstance(data, six.binary_type): + digest_object.update(data.encode('utf-8')) + + else: + digest_object.update(data) + return digest_object @@ -254,7 +267,6 @@ def digest_filename(filename, algorithm=_DEFAULT_HASH_ALGORITHM, Digest object (e.g., hashlib.new(algorithm) or algorithm.new() # pycrypto). - """ # Open 'filename' in read+binary mode. @@ -267,40 +279,5 @@ def digest_filename(filename, algorithm=_DEFAULT_HASH_ALGORITHM, digest_object = digest_fileobject(file_object, algorithm, hash_library) file_object.close() + return digest_object - - - - - -def data_to_string(data): - """ - - Return 'data' as a string. The update() function of a digest object - only accepts strings, however, TUF will often need to feed this function - non-strings. This utility function circumvents this issue and decides how - exactly to convert these objects TUF might use. - - - data: - The data object to be returned as a string. - - - None. - - - None. - - - String. - - """ - - if isinstance(data, str): - return data - - elif isinstance(data, unicode): - return data.encode("utf-8") - - else: - return str(data) diff --git a/tuf/keydb.py b/tuf/keydb.py index 1dbaec05..dbb34670 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -27,12 +27,21 @@ 'keyid' key (i.e., rsakey['keyid']). """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import logging import copy import tuf import tuf.formats import tuf.keys +import tuf._vendor.six as six # List of strings representing the key types supported by TUF. _SUPPORTED_KEY_TYPES = ['rsa', 'ed25519'] @@ -84,7 +93,7 @@ def create_keydb_from_root_metadata(root_metadata): # Iterate through the keys found in 'root_metadata' by converting # them to 'RSAKEY_SCHEMA' if their type is 'rsa', and then # adding them the database. Duplicates are avoided. - for keyid, key_metadata in root_metadata['keys'].items(): + for keyid, key_metadata in six.iteritems(root_metadata['keys']): if key_metadata['keytype'] in _SUPPORTED_KEY_TYPES: # 'key_metadata' is stored in 'KEY_SCHEMA' format. Call # create_from_metadata_format() to get the key in 'RSAKEY_SCHEMA' @@ -93,17 +102,17 @@ def create_keydb_from_root_metadata(root_metadata): try: add_key(key_dict, keyid) - except tuf.KeyAlreadyExistsError, e: - logger.warn(e) + except tuf.KeyAlreadyExistsError as e: + logger.warning(e) continue # 'tuf.Error' raised if keyid does not match the keyid for 'rsakey_dict'. - except tuf.Error, e: + except tuf.Error as e: logger.error(e) continue else: - logger.warn('Root Metadata file contains a key with an invalid keytype.') + logger.warning('Root Metadata file contains a key with an invalid keytype.') diff --git a/tuf/keys.py b/tuf/keys.py index 0e7ba84f..7a50736b 100755 --- a/tuf/keys.py +++ b/tuf/keys.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ keys.py @@ -44,6 +46,14 @@ key (i.e., rsakey['keyid']). """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + # Required for hexadecimal conversions. Signatures and public/private keys are # hexlified. import binascii @@ -303,13 +313,13 @@ def generate_ed25519_key(): # Generate the keyid of the ED25519 key. 'key_value' corresponds to the # 'keyval' entry of the 'ED25519KEY_SCHEMA' dictionary. The private key # information is not included in the generation of the 'keyid' identifier. - key_value = {'public': binascii.hexlify(public), + key_value = {'public': binascii.hexlify(public).decode(), 'private': ''} keyid = _get_keyid(keytype, key_value) # Build the 'ed25519_key' dictionary. Update 'key_value' with the ED25519 # private key prior to adding 'key_value' to 'ed25519_key'. - key_value['private'] = binascii.hexlify(private) + key_value['private'] = binascii.hexlify(private).decode() ed25519_key['keytype'] = keytype ed25519_key['keyid'] = keyid @@ -491,7 +501,7 @@ def _get_keyid(keytype, key_value): # Create a digest object and call update(), using the JSON # canonical format of 'rskey_meta' as the update data. digest_object = tuf.hash.digest(_KEY_ID_HASH_ALGORITHM) - digest_object.update(key_update_data) + digest_object.update(key_update_data.encode('utf-8')) # 'keyid' becomes the hexadecimal representation of the hash. keyid = digest_object.hexdigest() @@ -684,7 +694,7 @@ def create_signature(key_dict, data): # otherwise raise an exception. if keytype == 'rsa': if _RSA_CRYPTO_LIBRARY == 'pycrypto': - sig, method = tuf.pycrypto_keys.create_rsa_signature(private, data) + sig, method = tuf.pycrypto_keys.create_rsa_signature(private, data.encode('utf-8')) else: # pragma: no cover message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": '+\ @@ -692,10 +702,10 @@ def create_signature(key_dict, data): raise tuf.UnsupportedLibraryError(message) elif keytype == 'ed25519': - public = binascii.unhexlify(public) - private = binascii.unhexlify(private) + public = binascii.unhexlify(public.encode('utf-8')) + private = binascii.unhexlify(private.encode('utf-8')) if 'pynacl' in _available_crypto_libraries: - sig, method = tuf.ed25519_keys.create_signature(public, private, data) + sig, method = tuf.ed25519_keys.create_signature(public, private, data.encode('utf-8')) else: # pragma: no cover message = 'The required PyNaCl library is unavailable.' @@ -709,7 +719,7 @@ def create_signature(key_dict, data): # The hexadecimal representation of 'sig' is stored in the signature. signature['keyid'] = keyid signature['method'] = method - signature['sig'] = binascii.hexlify(sig) + signature['sig'] = binascii.hexlify(sig).decode() return signature @@ -798,7 +808,7 @@ def verify_signature(key_dict, signature, data): # key_dict['keyval']['private']. method = signature['method'] sig = signature['sig'] - sig = binascii.unhexlify(sig) + sig = binascii.unhexlify(sig.encode('utf-8')) public = key_dict['keyval']['public'] keytype = key_dict['keytype'] valid_signature = False @@ -807,7 +817,7 @@ def verify_signature(key_dict, signature, data): # generated across different platforms and Python key dictionaries. The # resulting 'data' is a string encoded in UTF-8 and compatible with the input # expected by the cryptography functions called below. - data = tuf.formats.encode_canonical(data) + data = tuf.formats.encode_canonical(data).encode('utf-8') # Call the appropriate cryptography libraries for the supported key types, # otherwise raise an exception. @@ -822,20 +832,20 @@ def verify_signature(key_dict, signature, data): else: valid_signature = tuf.pycrypto_keys.verify_rsa_signature(sig, method, public, data) - else: + else: # pragma: no cover message = 'Unsupported "tuf.conf.RSA_CRYPTO_LIBRARY": '+\ repr(_RSA_CRYPTO_LIBRARY)+'.' raise tuf.UnsupportedLibraryError(message) elif keytype == 'ed25519': - public = binascii.unhexlify(public) + public = binascii.unhexlify(public.encode('utf-8')) if _ED25519_CRYPTO_LIBRARY == 'pynacl' or \ 'pynacl' in _available_crypto_libraries: valid_signature = tuf.ed25519_keys.verify_signature(public, method, sig, data, use_pynacl=True) # Fall back to the optimized pure python implementation of ed25519. - else: + else: # pragma: no cover valid_signature = tuf.ed25519_keys.verify_signature(public, method, sig, data, use_pynacl=False) @@ -933,7 +943,7 @@ def import_rsakey_from_encrypted_pem(encrypted_pem, password): public, private = \ tuf.pycrypto_keys.create_rsa_public_and_private_from_encrypted_pem(encrypted_pem, password) - else: + else: #pragma: no cover message = 'Invalid crypto library: '+repr(_RSA_CRYPTO_LIBRARY)+'.' raise tuf.UnsupportedLibraryError(message) @@ -978,10 +988,6 @@ def format_rsakey_from_pem(pem): >>> rsa_key2 = format_rsakey_from_pem(public) >>> rsa_key == rsa_key2 True - >>> format_rsakey_from_pem('bad_pem') - Traceback (most recent call last): - ... - FormatError: The PEM string argument is improperly formatted. pem: @@ -1007,7 +1013,7 @@ def format_rsakey_from_pem(pem): # Ensure the PEM string starts with the required number of dashes. Although # a simple validation of 'pem' is performed here, a fully valid PEM string is # needed to successfully verify signatures. - if not pem.startswith(b'-----'): + if not pem.startswith('-----'): raise tuf.FormatError('The PEM string argument is improperly formatted.') # Begin building the RSA key dictionary. @@ -1057,7 +1063,7 @@ def encrypt_key(key_object, password): >>> ed25519_key = generate_ed25519_key() >>> password = 'secret' - >>> encrypted_key = encrypt_key(ed25519_key, password) + >>> encrypted_key = encrypt_key(ed25519_key, password).encode('utf-8') >>> tuf.formats.ENCRYPTEDKEY_SCHEMA.matches(encrypted_key) True @@ -1111,8 +1117,9 @@ def encrypt_key(key_object, password): if _GENERAL_CRYPTO_LIBRARY == 'pycrypto': encrypted_key = \ tuf.pycrypto_keys.encrypt_key(key_object, password) - - else: + + # check_crypto_libraries() should have fully verified _GENERAL_CRYPTO_LIBRARY. + else: # pragma: no cover message = 'Invalid crypto library: '+repr(_GENERAL_CRYPTO_LIBRARY)+'.' raise tuf.UnsupportedLibraryError(message) @@ -1147,7 +1154,7 @@ def decrypt_key(encrypted_key, passphrase): >>> ed25519_key = generate_ed25519_key() >>> password = 'secret' >>> encrypted_key = encrypt_key(ed25519_key, password) - >>> decrypted_key = decrypt_key(encrypted_key, password) + >>> decrypted_key = decrypt_key(encrypted_key.encode('utf-8'), password) >>> tuf.formats.ANYKEY_SCHEMA.matches(decrypted_key) True >>> decrypted_key == ed25519_key @@ -1209,7 +1216,8 @@ def decrypt_key(encrypted_key, passphrase): key_object = \ tuf.pycrypto_keys.decrypt_key(encrypted_key, passphrase) - else: + # check_crypto_libraries() should have fully verified _GENERAL_CRYPTO_LIBRARY. + else: # pragma: no cover message = 'Invalid crypto library: '+repr(_GENERAL_CRYPTO_LIBRARY)+'.' raise tuf.UnsupportedLibraryError(message) @@ -1278,7 +1286,7 @@ def create_rsa_encrypted_pem(private_key, passphrase): # Raise 'tuf.UnsupportedLibraryError' if the following libraries, specified in # 'tuf.conf', are unsupported or unavailable: - # 'tuf.conf.RSA_CRYPTO_LIBRARY' and 'tuf.conf.GENERAL_CRYPTO_LIBRARY'. + # 'tuf.conf.GENERAL_CRYPTO_LIBRARY' and 'tuf.conf.RSA_CRYPTO_LIBRARY'. check_crypto_libraries(['rsa', 'general']) encrypted_pem = None @@ -1290,8 +1298,9 @@ def create_rsa_encrypted_pem(private_key, passphrase): if _RSA_CRYPTO_LIBRARY == 'pycrypto': encrypted_pem = \ tuf.pycrypto_keys.create_rsa_encrypted_pem(private_key, passphrase) - - else: + + # check_crypto_libraries() should have fully verified _RSA_CRYPTO_LIBRARY. + else: # pragma: no cover message = 'Invalid crypto library: '+repr(_RSA_CRYPTO_LIBRARY)+'.' raise tuf.UnsupportedLibraryError(message) diff --git a/tuf/log.py b/tuf/log.py index ebbe65d6..92281f31 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -56,6 +56,13 @@ http://docs.python.org/2/howto/logging-cookbook.html """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals import logging import time @@ -312,7 +319,7 @@ def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL): logger.debug('Added a console handler.') else: - logger.warn('We already have a console handler.') + logger.warning('We already have a console handler.') @@ -346,4 +353,4 @@ def remove_console_handler(): logger.debug('Removed a console handler.') else: - logger.warn('We do not have a console handler.') + logger.warning('We do not have a console handler.') diff --git a/tuf/mirrors.py b/tuf/mirrors.py index 4ece3515..2fad086d 100755 --- a/tuf/mirrors.py +++ b/tuf/mirrors.py @@ -3,26 +3,34 @@ mirrors.py - Konstantin Andrianov + Konstantin Andrianov. Derived from original mirrors.py written by Geremy Condra. - March 12, 2012 + March 12, 2012. See LICENSE for licensing information. - To extract a list of mirror urls corresponding to the file type and - the location of the file with respect to the base url. + Extract a list of mirror urls corresponding to the file type and the location + of the file with respect to the base url. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os -import urllib import tuf import tuf.util import tuf.formats +import tuf._vendor.six as six # The type of file to be downloaded from a repository. The # 'get_list_of_mirrors' function supports these file types. @@ -74,6 +82,7 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): tuf.formats.MIRRORDICT_SCHEMA.check_match(mirrors_dict) tuf.formats.NAME_SCHEMA.check_match(file_type) + # Verify 'file_type' is supported. if file_type not in _SUPPORTED_FILE_TYPES: message = 'Invalid file_type argument. '+ \ 'Supported file types: '+repr(_SUPPORTED_FILE_TYPES) @@ -87,29 +96,26 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): in_confined_directory = tuf.util.file_in_confined_directories list_of_mirrors = [] - for mirror_name, mirror_info in mirrors_dict.items(): + for mirror_name, mirror_info in six.iteritems(mirrors_dict): if file_type == 'meta': base = mirror_info['url_prefix']+'/'+mirror_info['metadata_path'] - - elif file_type == 'target': + + # 'file_type' == 'target'. 'file_type' should have been verified to contain + # a supported string value above (either 'meta' or 'target'). + else: targets_path = mirror_info['targets_path'] full_filepath = os.path.join(targets_path, file_path) if not in_confined_directory(full_filepath, mirror_info['confined_target_dirs']): continue base = mirror_info['url_prefix']+'/'+mirror_info['targets_path'] - - else: - message = repr(file_type)+' is not a supported file type. '+ \ - 'Supported file types: '+repr(_SUPPORTED_FILE_TYPES) - raise tuf.Error(message) # urllib.quote(string) replaces special characters in string using the %xx # escape. This is done to avoid parsing issues of the URL on the server # side. Do *NOT* pass URLs with Unicode characters without first encoding # the URL as UTF-8. We need a long-term solution with #61. # http://bugs.python.org/issue1712522 - file_path = urllib.quote(file_path) + file_path = six.moves.urllib.parse.quote(file_path) url = base + '/' + file_path.lstrip(os.sep) list_of_mirrors.append(url) diff --git a/tuf/pycrypto_keys.py b/tuf/pycrypto_keys.py index def95632..72c7e8b7 100755 --- a/tuf/pycrypto_keys.py +++ b/tuf/pycrypto_keys.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ pycrypto_keys.py @@ -44,6 +46,14 @@ Derivation Function 1 (PBKF1) + MD5. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os import binascii import json @@ -205,7 +215,7 @@ def generate_rsa_public_and_private(bits=_DEFAULT_RSA_KEY_BITS): rsa_pubkey = rsa_key_object.publickey() public = rsa_pubkey.exportKey(format='PEM') - return public, private + return public.decode(), private.decode() @@ -224,13 +234,13 @@ def create_rsa_signature(private_key, data): http://www.ietf.org/rfc/rfc3447.txt >>> public, private = generate_rsa_public_and_private(2048) - >>> data = 'The quick brown fox jumps over the lazy dog' + >>> data = 'The quick brown fox jumps over the lazy dog'.encode('utf-8') >>> signature, method = create_rsa_signature(private, data) >>> tuf.formats.NAME_SCHEMA.matches(method) True >>> method == 'RSASSA-PSS' True - >>> tuf.formats.PYCRYPTOSIGNATURE_SCHEMA.matches(method) + >>> tuf.formats.PYCRYPTOSIGNATURE_SCHEMA.matches(signature) True @@ -267,8 +277,8 @@ def create_rsa_signature(private_key, data): signature = None # Verify the signature, but only if the private key has been set. The private - # key is a NULL string if unset. Although it may be clearer to explicit check - # that 'private_key' is not '', we can/should check for a value and not + # key is a NULL string if unset. Although it may be clearer to explicitly + # check that 'private_key' is not '', we can/should check for a value and not # compare identities with the 'is' keyword. Up to this point 'private_key' # has variable size and can be an empty string. if len(private_key): @@ -284,7 +294,7 @@ def create_rsa_signature(private_key, data): sha256_object = Crypto.Hash.SHA256.new(data) rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key) - except (ValueError, IndexError, TypeError), e: + except (ValueError, IndexError, TypeError) as e: message = 'Invalid private key or hash data: '+str(e) raise tuf.CryptoError(message) @@ -321,11 +331,11 @@ def verify_rsa_signature(signature, signature_method, public_key, data): and 'data' to complete the verification. >>> public, private = generate_rsa_public_and_private(2048) - >>> data = 'The quick brown fox jumps over the lazy dog' + >>> data = b'The quick brown fox jumps over the lazy dog' >>> signature, method = create_rsa_signature(private, data) >>> verify_rsa_signature(signature, method, public, data) True - >>> verify_rsa_signature(signature, method, public, 'bad_data') + >>> verify_rsa_signature(signature, method, public, b'bad_data') False @@ -383,7 +393,7 @@ def verify_rsa_signature(signature, signature_method, public_key, data): sha256_object = Crypto.Hash.SHA256.new(data) valid_signature = pkcs1_pss_verifier.verify(sha256_object, signature) - except (ValueError, IndexError, TypeError), e: + except (ValueError, IndexError, TypeError) as e: message = 'The RSA signature could not be verified.' raise tuf.CryptoError(message) @@ -463,7 +473,7 @@ def create_rsa_encrypted_pem(private_key, passphrase): encrypted_pem = rsa_key_object.exportKey(format='PEM', passphrase=passphrase) - except (ValueError, IndexError, TypeError), e: + except (ValueError, IndexError, TypeError) as e: message = 'An encrypted RSA key in PEM format cannot be generated: '+str(e) raise tuf.CryptoError(message) @@ -471,7 +481,7 @@ def create_rsa_encrypted_pem(private_key, passphrase): raise TypeError('The required private key is unset.') - return encrypted_pem + return encrypted_pem.decode() @@ -559,7 +569,7 @@ def create_rsa_public_and_private_from_encrypted_pem(encrypted_pem, passphrase): # (possibly because the passphrase is wrong)." # If the passphrase is incorrect, PyCrypto returns: "RSA key format is not # supported". - except (ValueError, IndexError, TypeError), e: + except (ValueError, IndexError, TypeError) as e: message = 'RSA (public, private) tuple cannot be generated from the'+\ ' encrypted PEM string: '+str(e) # Raise 'tuf.CryptoError' and PyCrypto's exception message. Avoid @@ -580,7 +590,7 @@ def create_rsa_public_and_private_from_encrypted_pem(encrypted_pem, passphrase): message = 'The public and private keys cannot be exported in PEM format.' raise tuf.CryptoError(message) - return public, private + return public.decode(), private.decode() @@ -616,7 +626,7 @@ def encrypt_key(key_object, password): '1f26964cc8d4f7ee5f3c5da2fbb7ab35811169573ac367b860a537e47789f8c4'}} >>> passphrase = 'secret' >>> encrypted_key = encrypt_key(ed25519_key, passphrase) - >>> tuf.formats.ENCRYPTEDKEY_SCHEMA.matches(encrypted_key) + >>> tuf.formats.ENCRYPTEDKEY_SCHEMA.matches(encrypted_key.encode('utf-8')) True @@ -706,7 +716,7 @@ def decrypt_key(encrypted_key, password): '1f26964cc8d4f7ee5f3c5da2fbb7ab35811169573ac367b860a537e47789f8c4'}} >>> passphrase = 'secret' >>> encrypted_key = encrypt_key(ed25519_key, passphrase) - >>> decrypted_key = decrypt_key(encrypted_key, passphrase) + >>> decrypted_key = decrypt_key(encrypted_key.encode('utf-8'), passphrase) >>> tuf.formats.ED25519KEY_SCHEMA.matches(decrypted_key) True >>> decrypted_key == ed25519_key @@ -751,11 +761,11 @@ def decrypt_key(encrypted_key, password): # Decrypt 'encrypted_key', using 'password' (and additional key derivation # data like salts and password iterations) to re-derive the decryption key. - json_data = _decrypt(encrypted_key, password) + json_data = _decrypt(encrypted_key.decode('utf-8'), password) # Raise 'tuf.Error' if 'json_data' cannot be deserialized to a valid # 'tuf.formats.ANYKEY_SCHEMA' key object. - key_object = tuf.util.load_json_string(json_data) + key_object = tuf.util.load_json_string(json_data.decode()) return key_object @@ -836,7 +846,7 @@ def _encrypt(key_data, derived_key_information): # encryption. iv = Crypto.Random.new().read(16) stateful_counter_128bit_blocks = Crypto.Util.Counter.new(128, - initial_value=long(iv.encode('hex'), 16)) + initial_value=int(binascii.hexlify(iv), 16)) symmetric_key = derived_key_information['derived_key'] aes_cipher = Crypto.Cipher.AES.new(symmetric_key, Crypto.Cipher.AES.MODE_CTR, @@ -851,7 +861,7 @@ def _encrypt(key_data, derived_key_information): # what circumstances. PyCrypto example given is to call encrypt() without # checking for exceptions. Avoid propogating the exception trace and only # raise 'tuf.CryptoError', along with the cause of encryption failure. - except (ValueError, IndexError, TypeError), e: + except (ValueError, IndexError, TypeError) as e: message = 'The key data cannot be encrypted: '+str(e) raise tuf.CryptoError(message) @@ -862,7 +872,7 @@ def _encrypt(key_data, derived_key_information): hmac_object = Crypto.Hash.HMAC.new(symmetric_key, ciphertext, Crypto.Hash.SHA256) hmac = hmac_object.hexdigest() - + # Store the number of PBKDF2 iterations used to derive the symmetric key so # that the decryption routine can regenerate the symmetric key successfully. # The pbkdf2 iterations are allowed to vary for the keys loaded and saved. @@ -873,11 +883,11 @@ def _encrypt(key_data, derived_key_information): # '_ENCRYPTION_DELIMITER' to make extraction easier. This delimiter is # arbitrarily chosen and should not occur in the hexadecimal representations # of the fields it is separating. - return binascii.hexlify(salt) + _ENCRYPTION_DELIMITER + \ - binascii.hexlify(str(iterations)) + _ENCRYPTION_DELIMITER + \ - binascii.hexlify(hmac) + _ENCRYPTION_DELIMITER + \ - binascii.hexlify(iv) + _ENCRYPTION_DELIMITER + \ - binascii.hexlify(ciphertext) + return binascii.hexlify(salt).decode() + _ENCRYPTION_DELIMITER + \ + str(iterations) + _ENCRYPTION_DELIMITER + \ + hmac + _ENCRYPTION_DELIMITER + \ + binascii.hexlify(iv).decode() + _ENCRYPTION_DELIMITER + \ + binascii.hexlify(ciphertext).decode() @@ -904,11 +914,10 @@ def _decrypt(file_contents, password): raise tuf.CryptoError('Invalid encrypted file.') # Ensure we have the expected raw data for the delimited cryptographic data. - salt = binascii.unhexlify(salt) - iterations = int(binascii.unhexlify(iterations)) - hmac = binascii.unhexlify(hmac) - iv = binascii.unhexlify(iv) - ciphertext = binascii.unhexlify(ciphertext) + salt = binascii.unhexlify(salt.encode('utf-8')) + iterations = int(iterations) + iv = binascii.unhexlify(iv.encode('utf-8')) + ciphertext = binascii.unhexlify(ciphertext.encode('utf-8')) # Generate derived key from 'password'. The salt and iterations are specified # so that the expected derived key is regenerated correctly. Discard the old @@ -928,7 +937,7 @@ def _decrypt(file_contents, password): # The following decryption routine assumes 'ciphertext' was encrypted with # AES-256. stateful_counter_128bit_blocks = Crypto.Util.Counter.new(128, - initial_value=long(iv.encode('hex'), 16)) + initial_value=int(binascii.hexlify(iv), 16)) aes_cipher = Crypto.Cipher.AES.new(derived_key, Crypto.Cipher.AES.MODE_CTR, counter=stateful_counter_128bit_blocks) @@ -939,7 +948,9 @@ def _decrypt(file_contents, password): # what circumstances. PyCrypto example given is to call decrypt() without # checking for exceptions. Avoid propogating the exception trace and only # raise 'tuf.CryptoError', along with the cause of decryption failure. - except (ValueError, IndexError, TypeError), e: + # Note: decryption failure, due to malicious ciphertext, should not occur here + # if the hmac check above passed. + except (ValueError, IndexError, TypeError) as e: # pragma: no cover raise tuf.CryptoError('Decryption failed: '+str(e)) return key_plaintext diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py new file mode 100755 index 00000000..b32d251e --- /dev/null +++ b/tuf/repository_lib.py @@ -0,0 +1,2231 @@ + +""" + + repository_lib.py + + + Vladimir Diaz + + + June 1, 2014 + + + See LICENSE for licensing information. + + + Provide a library for the repository tool that can create a TUF repository. + The repository tool can be used with the Python interpreter in interactive + mode, or imported directly into a Python module. See 'tuf/README' for the + complete guide to using 'tuf.repository_tool.py'. +""" + +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +import os +import errno +import sys +import time +import datetime +import getpass +import logging +import tempfile +import shutil +import json +import gzip +import random + +import tuf +import tuf.formats +import tuf.util +import tuf.keydb +import tuf.roledb +import tuf.keys +import tuf.sig +import tuf.log +import tuf.conf +import tuf._vendor.iso8601 as iso8601 +import tuf._vendor.six as six + + +# See 'log.py' to learn how logging is handled in TUF. +logger = logging.getLogger('tuf.repository_lib') + +# Recommended RSA key sizes: +# http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 +# According to the document above, revised May 6, 2003, RSA keys of +# size 3072 provide security through 2031 and beyond. 2048-bit keys +# are the recommended minimum and are good from the present through 2030. +DEFAULT_RSA_KEY_BITS = 3072 + +# The extension of TUF metadata. +METADATA_EXTENSION = '.json' + +# The targets and metadata directory names. Metadata files are written +# to the staged metadata directory instead of the "live" one. +METADATA_STAGED_DIRECTORY_NAME = 'metadata.staged' +METADATA_DIRECTORY_NAME = 'metadata' +TARGETS_DIRECTORY_NAME = 'targets' + +# The metadata filenames of the top-level roles. +ROOT_FILENAME = 'root' + METADATA_EXTENSION +TARGETS_FILENAME = 'targets' + METADATA_EXTENSION +SNAPSHOT_FILENAME = 'snapshot' + METADATA_EXTENSION +TIMESTAMP_FILENAME = 'timestamp' + METADATA_EXTENSION + +# Log warning when metadata expires in n days, or less. +# root = 1 month, snapshot = 1 day, targets = 10 days, timestamp = 1 day. +ROOT_EXPIRES_WARN_SECONDS = 2630000 +SNAPSHOT_EXPIRES_WARN_SECONDS = 86400 +TARGETS_EXPIRES_WARN_SECONDS = 864000 +TIMESTAMP_EXPIRES_WARN_SECONDS = 86400 + +# Supported key types. +SUPPORTED_KEY_TYPES = ['rsa', 'ed25519'] + +# The recognized compression extensions. +SUPPORTED_COMPRESSION_EXTENSIONS = ['.gz'] + +# The full list of supported TUF metadata extensions. +METADATA_EXTENSIONS = ['.json', '.json.gz'] + + +def _generate_and_write_metadata(rolename, metadata_filename, write_partial, + targets_directory, metadata_directory, + consistent_snapshot=False, filenames=None): + """ + Non-public function that can generate and write the metadata of the specified + top-level 'rolename'. It also increments version numbers if: + + 1. write_partial==True and the metadata is the first to be written. + + 2. write_partial=False (i.e., write()), the metadata was not loaded as + partially written, and a write_partial is not needed. + """ + + metadata = None + + # Retrieve the roleinfo of 'rolename' to extract the needed metadata + # attributes, such as version number, expiration, etc. + roleinfo = tuf.roledb.get_roleinfo(rolename) + snapshot_compressions = tuf.roledb.get_roleinfo('snapshot')['compressions'] + + # Generate the appropriate role metadata for 'rolename'. + if rolename == 'root': + metadata = generate_root_metadata(roleinfo['version'], + roleinfo['expires'], consistent_snapshot) + + _log_warning_if_expires_soon(ROOT_FILENAME, roleinfo['expires'], + ROOT_EXPIRES_WARN_SECONDS) + + # Check for the Targets role, including delegated roles. + elif rolename.startswith('targets'): + metadata = generate_targets_metadata(targets_directory, + roleinfo['paths'], + roleinfo['version'], + roleinfo['expires'], + roleinfo['delegations'], + consistent_snapshot) + if rolename == 'targets': + _log_warning_if_expires_soon(TARGETS_FILENAME, roleinfo['expires'], + TARGETS_EXPIRES_WARN_SECONDS) + + elif rolename == 'snapshot': + root_filename = filenames['root'] + targets_filename = filenames['targets'] + metadata = generate_snapshot_metadata(metadata_directory, + roleinfo['version'], + roleinfo['expires'], root_filename, + targets_filename, + consistent_snapshot) + + _log_warning_if_expires_soon(SNAPSHOT_FILENAME, roleinfo['expires'], + SNAPSHOT_EXPIRES_WARN_SECONDS) + + elif rolename == 'timestamp': + snapshot_filename = filenames['snapshot'] + metadata = generate_timestamp_metadata(snapshot_filename, + roleinfo['version'], + roleinfo['expires'], + snapshot_compressions) + + _log_warning_if_expires_soon(TIMESTAMP_FILENAME, roleinfo['expires'], + TIMESTAMP_EXPIRES_WARN_SECONDS) + + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + + # Check if the version number of 'rolename' may be automatically incremented, + # depending on whether if partial metadata is loaded or if the metadata is + # written with write() / write_partial(). + # Increment the version number if this is the first partial write. + if write_partial: + temp_signable = sign_metadata(metadata, [], metadata_filename) + temp_signable['signatures'].extend(roleinfo['signatures']) + status = tuf.sig.get_signature_status(temp_signable, rolename) + if len(status['good_sigs']) == 0: + metadata['version'] = metadata['version'] + 1 + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + # non-partial write() + else: + if tuf.sig.verify(signable, rolename) and not roleinfo['partial_loaded']: + metadata['version'] = metadata['version'] + 1 + signable = sign_metadata(metadata, roleinfo['signing_keyids'], + metadata_filename) + + # Write the metadata to file if contains a threshold of signatures. + signable['signatures'].extend(roleinfo['signatures']) + + if tuf.sig.verify(signable, rolename) or write_partial: + _remove_invalid_and_duplicate_signatures(signable) + compressions = roleinfo['compressions'] + filename = write_metadata_file(signable, metadata_filename, compressions, + consistent_snapshot) + + # The root and timestamp files should also be written without a digest if + # 'consistent_snaptshots' is True. Client may request a timestamp and root + # file without knowing its digest and file size. + if rolename == 'root' or rolename == 'timestamp': + write_metadata_file(signable, metadata_filename, compressions, + consistent_snapshot=False) + + + # 'signable' contains an invalid threshold of signatures. + else: + message = 'Not enough signatures for ' + repr(metadata_filename) + raise tuf.UnsignedMetadataError(message, signable) + + return signable, filename + + + + + +def _prompt(message, result_type=str): + """ + Non-public function that prompts the user for input by loging 'message', + converting the input to 'result_type', and returning the value to the + caller. + """ + + return result_type(six.moves.input(message)) + + + + + +def _get_password(prompt='Password: ', confirm=False): + """ + Non-public function that returns the password entered by the user. If + 'confirm' is True, the user is asked to enter the previously entered + password once again. If they match, the password is returned to the caller. + """ + + while True: + # getpass() prompts the user for a password without echoing + # the user input. + password = getpass.getpass(prompt, sys.stderr) + + if not confirm: + return password + password2 = getpass.getpass('Confirm: ', sys.stderr) + + if password == password2: + return password + + else: + print('Mismatch; try again.') + + + + + +def _metadata_is_partially_loaded(rolename, signable, roleinfo): + """ + Non-public function that determines whether 'rolename' is loaded with + at least zero good signatures, but an insufficient threshold (which means + 'rolename' was written to disk with repository.write_partial()). A repository + maintainer may write partial metadata without including a valid signature. + Howerver, the final repository.write() must include a threshold number of + signatures. + + If 'rolename' is found to be partially loaded, mark it as partially loaded in + its 'tuf.roledb' roleinfo. This function exists to assist in deciding whether + a role's version number should be incremented when write() or write_parital() + is called. Return True if 'rolename' was partially loaded, False otherwise. + """ + + # The signature status lists the number of good signatures, including + # bad, untrusted, unknown, etc. + status = tuf.sig.get_signature_status(signable, rolename) + + if len(status['good_sigs']) < status['threshold'] and \ + len(status['good_sigs']) >= 0: + return True + + else: + return False + + + + + +def _check_directory(directory): + """ + + Non-public function that ensures 'directory' is valid and it exists. This + is not a security check, but a way for the caller to determine the cause of + an invalid directory provided by the user. If the directory argument is + valid, it is returned normalized and as an absolute path. + + + directory: + The directory to check. + + + tuf.Error, if 'directory' could not be validated. + + tuf.FormatError, if 'directory' is not properly formatted. + + + None. + + + The normalized absolutized path of 'directory'. + """ + + # Does 'directory' have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(directory) + + # Check if the directory exists. + if not os.path.isdir(directory): + raise tuf.Error(repr(directory) + ' directory does not exist.') + + directory = os.path.abspath(directory) + + return directory + + + + + +def _check_role_keys(rolename): + """ + Non-public function that verifies the public and signing keys of 'rolename'. + If either contain an invalid threshold of keys, raise an exception. + 'rolename' is the full rolename (e.g., 'targets/unclaimed/django'). + """ + + # Extract the total number of public and private keys of 'rolename' from its + # roleinfo in 'tuf.roledb'. + roleinfo = tuf.roledb.get_roleinfo(rolename) + total_keyids = len(roleinfo['keyids']) + threshold = roleinfo['threshold'] + total_signatures = len(roleinfo['signatures']) + total_signing_keys = len(roleinfo['signing_keyids']) + + # Raise an exception for an invalid threshold of public keys. + if total_keyids < threshold: + message = repr(rolename) + ' role contains ' + \ + repr(total_keyids) + ' / ' + repr(threshold) + ' public keys.' + raise tuf.InsufficientKeysError(message) + + # Raise an exception for an invalid threshold of signing keys. + if total_signatures == 0 and total_signing_keys < threshold: + message = repr(rolename) + ' role contains ' + \ + repr(total_signing_keys) + ' / ' + repr(threshold) + ' signing keys.' + raise tuf.InsufficientKeysError(message) + + + + + +def _remove_invalid_and_duplicate_signatures(signable): + """ + Non-public function that removes invalid signatures from 'signable'. + 'signable' may contain signatures (invalid) from previous versions + of the metadata that were loaded with load_repository(). Invalid, or + duplicate signatures are removed from 'signable'. + """ + + # Store the keyids of valid signatures. 'signature_keyids' is checked + # for duplicates rather than comparing signature objects because PSS may + # generate duplicate valid signatures of the same data, yet contain different + # signatures. + signature_keyids = [] + + for signature in signable['signatures']: + signed = signable['signed'] + keyid = signature['keyid'] + key = None + + # Remove 'signature' from 'signable' if the listed keyid does not exist + # in 'tuf.keydb'. + try: + key = tuf.keydb.get_key(keyid) + + except tuf.UnknownKeyError as e: + signable['signatures'].remove(signature) + + # Remove 'signature' from 'signable' if it is an invalid signature. + if not tuf.keys.verify_signature(key, signature, signed): + signable['signatures'].remove(signature) + + # Although valid, it may still need removal if it is a duplicate. Check + # the keyid, rather than the signature, to remove duplicate PSS signatures. + # PSS may generate multiple different signatures for the same keyid. + else: + if keyid in signature_keyids: + signable['signatures'].remove(signature) + + # 'keyid' is valid and not a duplicate, so add it to 'signature_keyids'. + else: + signature_keyids.append(keyid) + + + + + +def _delete_obsolete_metadata(metadata_directory, snapshot_metadata, + consistent_snapshot): + """ + Non-public function that deletes metadata files marked as removed by + 'repository_tool.py'. Revoked metadata files are not actually deleted until + this function is called. Obsolete metadata should *not* be retained in + "metadata.staged", otherwise they may be re-loaded by 'load_repository()'. + Note: Obsolete metadata may not always be easily detected (by inspecting + top-level metadata during loading) due to partial metadata and top-level + metadata that have not been written yet. + """ + + # Walk the repository's metadata 'targets' sub-directory, where all the + # metadata of delegated roles is stored. + targets_metadata = os.path.join(metadata_directory, 'targets') + + # The 'targets.json' metadata is not visited, only its child delegations. + # The 'targets/unclaimed/django.json' role would be located in the + # '{repository_directory}/metadata/targets/unclaimed/' directory. + if os.path.exists(targets_metadata) and os.path.isdir(targets_metadata): + for directory_path, junk_directories, files in os.walk(targets_metadata): + + # 'files' here is a list of target file names. + for basename in files: + metadata_path = os.path.join(directory_path, basename) + # Strip the metadata dirname and the leading path separator. + # '{repository_directory}/metadata/targets/unclaimed/django.json' --> + # 'targets/unclaimed/django.json' + metadata_name = \ + metadata_path[len(metadata_directory):].lstrip(os.path.sep) + + # Strip the digest if 'consistent_snapshot' is True. + # Example: 'targets/unclaimed/13df98ab0.django.json' --> + # 'targets/unclaimed/django.json'. Consistent and non-consistent + # metadata might co-exist if write() and write(consistent_snapshot=True) + # are mixed, so ensure only 'digest.filename' metadata is stripped. + embeded_digest = None + if metadata_name not in snapshot_metadata['meta']: + metadata_name, embeded_digest = \ + _strip_consistent_snapshot_digest(metadata_name, consistent_snapshot) + + # Strip filename extensions. The role database does not include the + # metadata extension. + metadata_name_extension = metadata_name + for metadata_extension in METADATA_EXTENSIONS: + if metadata_name.endswith(metadata_extension): + metadata_name = metadata_name[:-len(metadata_extension)] + + # Delete the metadata file if it does not exist in 'tuf.roledb'. + # 'repository_tool.py' might have marked 'metadata_name' as removed, but + # its metadata file is not actually deleted yet. Do it now. + if not tuf.roledb.role_exists(metadata_name): + logger.info('Removing outdated metadata: ' + repr(metadata_path)) + os.remove(metadata_path) + + # Delete outdated consistent snapshots. snapshot metadata includes + # the file extension of roles. + if consistent_snapshot and embeded_digest is not None: + file_hashes = list(snapshot_metadata['meta'][metadata_name_extension] \ + ['hashes'].values()) + if embeded_digest not in file_hashes: + logger.info('Removing outdated metadata: ' + repr(metadata_path)) + os.remove(metadata_path) + + + + + +def _get_written_metadata_and_digests(metadata_signable): + """ + Non-public function that returns the actual content of written metadata and + its digest. + """ + + # Explicitly specify the JSON separators for Python 2 + 3 consistent. + written_metadata_content = \ + json.dumps(metadata_signable, indent=1, separators=(',', ': '), + sort_keys=True).encode('utf-8') + + written_metadata_digests = {} + + for hash_algorithm in tuf.conf.REPOSITORY_HASH_ALGORITHMS: + digest_object = tuf.hash.digest(hash_algorithm) + digest_object.update(written_metadata_content) + written_metadata_digests.update({hash_algorithm: digest_object.hexdigest()}) + + return written_metadata_content, written_metadata_digests + + + + + +def _strip_consistent_snapshot_digest(metadata_filename, consistent_snapshot): + """ + Strip from 'metadata_filename' any digest data (in the expected + '{dirname}/digest.filename' format) that it may contain, and return it. + """ + + embeded_digest = '' + + # Strip the digest if 'consistent_snapshot' is True. + # Example: 'targets/unclaimed/13df98ab0.django.json' --> + # 'targets/unclaimed/django.json' + if consistent_snapshot: + dirname, basename = os.path.split(metadata_filename) + embeded_digest = basename[:basename.find('.')] + + # Ensure the digest, including the period, is stripped. + basename = basename[basename.find('.') + 1:] + + metadata_filename = os.path.join(dirname, basename) + + + return metadata_filename, embeded_digest + + + + + +def _load_top_level_metadata(repository, top_level_filenames): + """ + Load the metadata of the Root, Timestamp, Targets, and Snapshot roles. + At a minimum, the Root role must exist and successfully load. + """ + + root_filename = top_level_filenames[ROOT_FILENAME] + targets_filename = top_level_filenames[TARGETS_FILENAME] + snapshot_filename = top_level_filenames[SNAPSHOT_FILENAME] + timestamp_filename = top_level_filenames[TIMESTAMP_FILENAME] + + root_metadata = None + targets_metadata = None + snapshot_metadata = None + timestamp_metadata = None + + # Load 'root.json'. A Root role file without a digest is always written. + if os.path.exists(root_filename): + # Initialize the key and role metadata of the top-level roles. + signable = tuf.util.load_json_file(root_filename) + tuf.formats.check_signable_object_format(signable) + root_metadata = signable['signed'] + tuf.keydb.create_keydb_from_root_metadata(root_metadata) + tuf.roledb.create_roledb_from_root_metadata(root_metadata) + + # Load Root's roleinfo and update 'tuf.roledb'. + roleinfo = tuf.roledb.get_roleinfo('root') + roleinfo['signatures'] = [] + for signature in signable['signatures']: + if signature not in roleinfo['signatures']: + roleinfo['signatures'].append(signature) + + if os.path.exists(root_filename+'.gz'): + roleinfo['compressions'].append('gz') + + # By default, roleinfo['partial_loaded'] of top-level roles should be set to + # False in 'create_roledb_from_root_metadata()'. Update this field, if + # necessary, now that we have its signable object. + if _metadata_is_partially_loaded('root', signable, roleinfo): + roleinfo['partial_loaded'] = True + + _log_warning_if_expires_soon(ROOT_FILENAME, roleinfo['expires'], + ROOT_EXPIRES_WARN_SECONDS) + + tuf.roledb.update_roleinfo('root', roleinfo) + + # Ensure the 'consistent_snapshot' field is extracted. + consistent_snapshot = root_metadata['consistent_snapshot'] + + else: + message = 'Cannot load the required root file: '+repr(root_filename) + raise tuf.RepositoryError(message) + + # Load 'timestamp.json'. A Timestamp role file without a digest is always + # written. + if os.path.exists(timestamp_filename): + signable = tuf.util.load_json_file(timestamp_filename) + timestamp_metadata = signable['signed'] + for signature in signable['signatures']: + repository.timestamp.add_signature(signature) + + # Load Timestamp's roleinfo and update 'tuf.roledb'. + roleinfo = tuf.roledb.get_roleinfo('timestamp') + roleinfo['expires'] = timestamp_metadata['expires'] + roleinfo['version'] = timestamp_metadata['version'] + if os.path.exists(timestamp_filename+'.gz'): + roleinfo['compressions'].append('gz') + + if _metadata_is_partially_loaded('timestamp', signable, roleinfo): + roleinfo['partial_loaded'] = True + + _log_warning_if_expires_soon(TIMESTAMP_FILENAME, roleinfo['expires'], + TIMESTAMP_EXPIRES_WARN_SECONDS) + + tuf.roledb.update_roleinfo('timestamp', roleinfo) + + else: + pass + + # Load 'snapshot.json'. A consistent snapshot of Snapshot must be calculated + # if 'consistent_snapshot' is True. + if consistent_snapshot: + snapshot_hashes = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['hashes'] + snapshot_digest = random.choice(list(snapshot_hashes.values())) + dirname, basename = os.path.split(snapshot_filename) + snapshot_filename = os.path.join(dirname, snapshot_digest + '.' + basename) + + if os.path.exists(snapshot_filename): + signable = tuf.util.load_json_file(snapshot_filename) + tuf.formats.check_signable_object_format(signable) + snapshot_metadata = signable['signed'] + for signature in signable['signatures']: + repository.snapshot.add_signature(signature) + + # Load Snapshot's roleinfo and update 'tuf.roledb'. + roleinfo = tuf.roledb.get_roleinfo('snapshot') + roleinfo['expires'] = snapshot_metadata['expires'] + roleinfo['version'] = snapshot_metadata['version'] + if os.path.exists(snapshot_filename+'.gz'): + roleinfo['compressions'].append('gz') + + if _metadata_is_partially_loaded('snapshot', signable, roleinfo): + roleinfo['partial_loaded'] = True + + _log_warning_if_expires_soon(SNAPSHOT_FILENAME, roleinfo['expires'], + SNAPSHOT_EXPIRES_WARN_SECONDS) + + tuf.roledb.update_roleinfo('snapshot', roleinfo) + + else: + pass + + # Load 'targets.json'. A consistent snapshot of Targets must be calculated if + # 'consistent_snapshot' is True. + if consistent_snapshot: + targets_hashes = snapshot_metadata['meta'][TARGETS_FILENAME]['hashes'] + targets_digest = random.choice(list(targets_hashes.values())) + dirname, basename = os.path.split(targets_filename) + targets_filename = os.path.join(dirname, targets_digest + '.' + basename) + + if os.path.exists(targets_filename): + signable = tuf.util.load_json_file(targets_filename) + tuf.formats.check_signable_object_format(signable) + targets_metadata = signable['signed'] + + for signature in signable['signatures']: + repository.targets.add_signature(signature) + + # Update 'targets.json' in 'tuf.roledb.py' + roleinfo = tuf.roledb.get_roleinfo('targets') + roleinfo['paths'] = list(targets_metadata['targets'].keys()) + roleinfo['version'] = targets_metadata['version'] + roleinfo['expires'] = targets_metadata['expires'] + roleinfo['delegations'] = targets_metadata['delegations'] + if os.path.exists(targets_filename+'.gz'): + roleinfo['compressions'].append('gz') + + if _metadata_is_partially_loaded('targets', signable, roleinfo): + roleinfo['partial_loaded'] = True + + _log_warning_if_expires_soon(TARGETS_FILENAME, roleinfo['expires'], + TARGETS_EXPIRES_WARN_SECONDS) + + tuf.roledb.update_roleinfo('targets', roleinfo) + + # Add the keys specified in the delegations field of the Targets role. + for key_metadata in six.itervalues(targets_metadata['delegations']['keys']): + key_object = tuf.keys.format_metadata_to_key(key_metadata) + + # Add 'key_object' to the list of recognized keys. Keys may be shared, + # so do not raise an exception if 'key_object' has already been loaded. + # In contrast to the methods that may add duplicate keys, do not log + # a warning as there may be many such duplicate key warnings. The + # repository maintainer should have also been made aware of the duplicate + # key when it was added. + try: + tuf.keydb.add_key(key_object) + + except tuf.KeyAlreadyExistsError as e: + pass + + for role in targets_metadata['delegations']['roles']: + rolename = role['name'] + roleinfo = {'name': role['name'], 'keyids': role['keyids'], + 'threshold': role['threshold'], 'compressions': [''], + 'signing_keyids': [], 'partial_loaded': False, + 'signatures': [], 'delegations': {'keys': {}, + 'roles': []}} + tuf.roledb.add_role(rolename, roleinfo) + + else: + pass + + return repository, consistent_snapshot + + + + +def _log_warning_if_expires_soon(rolename, expires_iso8601_timestamp, + seconds_remaining_to_warn): + """ + Non-public function that logs a warning if 'rolename' expires in + 'seconds_remaining_to_warn' seconds, or less. + """ + + # Metadata stores expiration datetimes in ISO8601 format. Convert to + # unix timestamp, subtract from from current time.time() (also in POSIX time) + # and compare against 'seconds_remaining_to_warn'. Log a warning message + # to console if 'rolename' expires soon. + datetime_object = iso8601.parse_date(expires_iso8601_timestamp) + expires_unix_timestamp = \ + tuf.formats.datetime_to_unix_timestamp(datetime_object) + seconds_until_expires = expires_unix_timestamp - int(time.time()) + + if seconds_until_expires <= seconds_remaining_to_warn: + days_until_expires = seconds_until_expires / 86400 + + message = repr(rolename) + ' expires ' + datetime_object.ctime() + \ + ' (UTC).\n' + repr(days_until_expires) + ' day(s) until it expires.' + + logger.warning(message) + + + + + +def generate_and_write_rsa_keypair(filepath, bits=DEFAULT_RSA_KEY_BITS, + password=None): + """ + + Generate an RSA key file, create an encrypted PEM string (using 'password' + as the pass phrase), and store it in 'filepath'. The public key portion of + the generated RSA key is stored in <'filepath'>.pub. Which cryptography + library performs the cryptographic decryption is determined by the string + set in 'tuf.conf.RSA_CRYPTO_LIBRARY'. PyCrypto currently supported. The + PEM private key is encrypted with 3DES and CBC the mode of operation. The + password is strengthened with PBKDF1-MD5. + + + filepath: + The public and private key files are saved to .pub, , + respectively. + + bits: + The number of bits of the generated RSA key. + + password: + The password used to encrypt 'filepath'. + + + tuf.FormatError, if the arguments are improperly formatted. + + + Writes key files to '' and '.pub'. + + + None. + """ + + # Do the arguments have the correct format? + # This check ensures arguments have the appropriate number of + # objects and object types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(filepath) + + # Does 'bits' have the correct format? + tuf.formats.RSAKEYBITS_SCHEMA.check_match(bits) + + # If the caller does not provide a password argument, prompt for one. + if password is None: # pragma: no cover + message = 'Enter a password for the RSA key file: ' + password = _get_password(message, confirm=True) + + # Does 'password' have the correct format? + tuf.formats.PASSWORD_SCHEMA.check_match(password) + + # Generate public and private RSA keys, encrypted the private portion + # and store them in PEM format. + rsa_key = tuf.keys.generate_rsa_key(bits) + public = rsa_key['keyval']['public'] + private = rsa_key['keyval']['private'] + encrypted_pem = tuf.keys.create_rsa_encrypted_pem(private, password) + + # Write public key (i.e., 'public', which is in PEM format) to + # '.pub'. If the parent directory of filepath does not exist, + # create it (and all its parent directories, if necessary). + tuf.util.ensure_parent_dir(filepath) + + # Create a tempororary file, write the contents of the public key, and move + # to final destination. + file_object = tuf.util.TempFile() + file_object.write(public.encode('utf-8')) + + # The temporary file is closed after the final move. + file_object.move(filepath + '.pub') + + # Write the private key in encrypted PEM format to ''. + # Unlike the public key file, the private key does not have a file + # extension. + file_object = tuf.util.TempFile() + file_object.write(encrypted_pem.encode('utf-8')) + file_object.move(filepath) + + + + + +def import_rsa_privatekey_from_file(filepath, password=None): + """ + + Import the encrypted PEM file in 'filepath', decrypt it, and return the key + object in 'tuf.formats.RSAKEY_SCHEMA' format. + + Which cryptography library performs the cryptographic decryption is + determined by the string set in 'tuf.conf.RSA_CRYPTO_LIBRARY'. PyCrypto + currently supported. + + The PEM private key is encrypted with 3DES and CBC the mode of operation. + The password is strengthened with PBKDF1-MD5. + + + filepath: + file, an RSA encrypted PEM file. Unlike the public RSA PEM + key file, 'filepath' does not have an extension. + + password: + The passphrase to decrypt 'filepath'. + + + tuf.FormatError, if the arguments are improperly formatted. + + tuf.CryptoError, if 'filepath' is not a valid encrypted key file. + + + The contents of 'filepath' is read, decrypted, and the key stored. + + + An RSA key object, conformant to 'tuf.formats.RSAKEY_SCHEMA'. + """ + + # Does 'filepath' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(filepath) + + # If the caller does not provide a password argument, prompt for one. + # Password confirmation disabled here, which should ideally happen only + # when creating encrypted key files (i.e., improve usability). + if password is None: # pragma: no cover + message = 'Enter a password for the encrypted RSA file: ' + password = _get_password(message, confirm=False) + + # Does 'password' have the correct format? + tuf.formats.PASSWORD_SCHEMA.check_match(password) + + encrypted_pem = None + + # Read the contents of 'filepath' that should be an encrypted PEM. + with open(filepath, 'rb') as file_object: + encrypted_pem = file_object.read().decode('utf-8') + + # Convert 'encrypted_pem' to 'tuf.formats.RSAKEY_SCHEMA' format. Raise + # 'tuf.CryptoError' if 'encrypted_pem' is invalid. + rsa_key = tuf.keys.import_rsakey_from_encrypted_pem(encrypted_pem, password) + + return rsa_key + + + + + +def import_rsa_publickey_from_file(filepath): + """ + + Import the RSA key stored in 'filepath'. The key object returned is a TUF + key, specifically 'tuf.formats.RSAKEY_SCHEMA'. If the RSA PEM in 'filepath' + contains a private key, it is discarded. + + Which cryptography library performs the cryptographic decryption is + determined by the string set in 'tuf.conf.RSA_CRYPTO_LIBRARY'. PyCrypto + currently supported. If the RSA PEM in 'filepath' contains a private key, + it is discarded. + + + filepath: + .pub file, an RSA PEM file. + + + tuf.FormatError, if 'filepath' is improperly formatted. + + tuf.Error, if a valid RSA key object cannot be generated. This may be + caused by an improperly formatted PEM file. + + + 'filepath' is read and its contents extracted. + + + An RSA key object conformant to 'tuf.formats.RSAKEY_SCHEMA'. + """ + + # Does 'filepath' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(filepath) + + # Read the contents of the key file that should be in PEM format and contains + # the public portion of the RSA key. + with open(filepath, 'rb') as file_object: + rsa_pubkey_pem = file_object.read().decode('utf-8') + + # Convert 'rsa_pubkey_pem' to 'tuf.formats.RSAKEY_SCHEMA' format. + try: + rsakey_dict = tuf.keys.format_rsakey_from_pem(rsa_pubkey_pem) + + except tuf.FormatError as e: + raise tuf.Error('Cannot import improperly formatted PEM file.') + + return rsakey_dict + + + + + +def generate_and_write_ed25519_keypair(filepath, password=None): + """ + + Generate an ED25519 key file, create an encrypted TUF key (using 'password' + as the pass phrase), and store it in 'filepath'. The public key portion of + the generated ED25519 key is stored in <'filepath'>.pub. Which cryptography + library performs the cryptographic decryption is determined by the string + set in 'tuf.conf.ED25519_CRYPTO_LIBRARY'. + + PyCrypto currently supported. The ED25519 private key is encrypted with + AES-256 and CTR the mode of operation. The password is strengthened with + PBKDF2-HMAC-SHA256. + + + filepath: + The public and private key files are saved to .pub and + , respectively. + + password: + The password, or passphrase, to encrypt the private portion of the + generated ed25519 key. A symmetric encryption key is derived from + 'password', so it is not directly used. + + + tuf.FormatError, if the arguments are improperly formatted. + + tuf.CryptoError, if 'filepath' cannot be encrypted. + + tuf.UnsupportedLibraryError, if 'filepath' cannot be encrypted due to an + invalid configuration setting (i.e., invalid 'tuf.conf.py' setting). + + + Writes key files to '' and '.pub'. + + + None. + """ + + # Does 'filepath' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(filepath) + + # If the caller does not provide a password argument, prompt for one. + if password is None: # pragma: no cover + message = 'Enter a password for the ED25519 key: ' + password = _get_password(message, confirm=True) + + # Does 'password' have the correct format? + tuf.formats.PASSWORD_SCHEMA.check_match(password) + + # Generate a new ED25519 key object and encrypt it. The cryptography library + # used is determined by the user, or by default (set in + # 'tuf.conf.ED25519_CRYPTO_LIBRARY'). Raise 'tuf.CryptoError' or + # 'tuf.UnsupportedLibraryError', if 'ed25519_key' cannot be encrypted. + ed25519_key = tuf.keys.generate_ed25519_key() + encrypted_key = tuf.keys.encrypt_key(ed25519_key, password) + + # ed25519 public key file contents in metadata format (i.e., does not include + # the keyid portion). + keytype = ed25519_key['keytype'] + keyval = ed25519_key['keyval'] + ed25519key_metadata_format = \ + tuf.keys.format_keyval_to_metadata(keytype, keyval, private=False) + + # Write the public key, conformant to 'tuf.formats.KEY_SCHEMA', to + # '.pub'. + tuf.util.ensure_parent_dir(filepath) + + # Create a tempororary file, write the contents of the public key, and move + # to final destination. + file_object = tuf.util.TempFile() + file_object.write(json.dumps(ed25519key_metadata_format).encode('utf-8')) + + # The temporary file is closed after the final move. + file_object.move(filepath + '.pub') + + # Write the encrypted key string, conformant to + # 'tuf.formats.ENCRYPTEDKEY_SCHEMA', to ''. + file_object = tuf.util.TempFile() + file_object.write(encrypted_key.encode('utf-8')) + file_object.move(filepath) + + + + + +def import_ed25519_publickey_from_file(filepath): + """ + + Load the ED25519 public key object (conformant to 'tuf.formats.KEY_SCHEMA') + stored in 'filepath'. Return 'filepath' in tuf.formats.ED25519KEY_SCHEMA + format. + + If the TUF key object in 'filepath' contains a private key, it is discarded. + + + filepath: + .pub file, a TUF public key file. + + + tuf.FormatError, if 'filepath' is improperly formatted or is an unexpected + key type. + + + The contents of 'filepath' is read and saved. + + + An ED25519 key object conformant to 'tuf.formats.ED25519KEY_SCHEMA'. + """ + + # Does 'filepath' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(filepath) + + # ED25519 key objects are saved in json and metadata format. Return the + # loaded key object in tuf.formats.ED25519KEY_SCHEMA' format that also + # includes the keyid. + ed25519_key_metadata = tuf.util.load_json_file(filepath) + ed25519_key = tuf.keys.format_metadata_to_key(ed25519_key_metadata) + + # Raise an exception if an unexpected key type is imported. + # Redundant validation of 'keytype'. 'tuf.keys.format_metadata_to_key()' + # should have fully validated 'ed25519_key_metadata'. + if ed25519_key['keytype'] != 'ed25519': # pragma: no cover + message = 'Invalid key type loaded: ' + repr(ed25519_key['keytype']) + raise tuf.FormatError(message) + + return ed25519_key + + + + + +def import_ed25519_privatekey_from_file(filepath, password=None): + """ + + Import the encrypted ed25519 TUF key file in 'filepath', decrypt it, and + return the key object in 'tuf.formats.ED25519KEY_SCHEMA' format. + + Which cryptography library performs the cryptographic decryption is + determined by the string set in 'tuf.conf.ED25519_CRYPTO_LIBRARY'. PyCrypto + currently supported. + + The TUF private key (may also contain the public part) is encrypted with AES + 256 and CTR the mode of operation. The password is strengthened with + PBKDF2-HMAC-SHA256. + + + filepath: + file, an RSA encrypted TUF key file. + + password: + The password, or passphrase, to import the private key (i.e., the + encrypted key file 'filepath' must be decrypted before the ed25519 key + object can be returned. + + + tuf.FormatError, if the arguments are improperly formatted or the imported + key object contains an invalid key type (i.e., not 'ed25519'). + + tuf.CryptoError, if 'filepath' cannot be decrypted. + + tuf.UnsupportedLibraryError, if 'filepath' cannot be decrypted due to an + invalid configuration setting (i.e., invalid 'tuf.conf.py' setting). + + + 'password' is used to decrypt the 'filepath' key file. + + + An ed25519 key object of the form: 'tuf.formats.ED25519KEY_SCHEMA'. + """ + + # Does 'filepath' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(filepath) + + # If the caller does not provide a password argument, prompt for one. + # Password confirmation disabled here, which should ideally happen only + # when creating encrypted key files (i.e., improve usability). + if password is None: # pragma: no cover + message = 'Enter a password for the encrypted ED25519 key: ' + password = _get_password(message, confirm=False) + + # Does 'password' have the correct format? + tuf.formats.PASSWORD_SCHEMA.check_match(password) + + # Store the encrypted contents of 'filepath' prior to calling the decryption + # routine. + encrypted_key = None + + with open(filepath, 'rb') as file_object: + encrypted_key = file_object.read() + + # Decrypt the loaded key file, calling the appropriate cryptography library + # (i.e., set by the user) and generating the derived encryption key from + # 'password'. Raise 'tuf.CryptoError' or 'tuf.UnsupportedLibraryError' if the + # decryption fails. + key_object = tuf.keys.decrypt_key(encrypted_key, password) + + # Raise an exception if an unexpected key type is imported. + if key_object['keytype'] != 'ed25519': + message = 'Invalid key type loaded: ' + repr(key_object['keytype']) + raise tuf.FormatError(message) + + return key_object + + + + + +def get_metadata_filenames(metadata_directory=None): + """ + + Return a dictionary containing the filenames of the top-level roles. + If 'metadata_directory' is set to 'metadata', the dictionary + returned would contain: + + filenames = {'root.json': 'metadata/root.json', + 'targets.json': 'metadata/targets.json', + 'snapshot.json': 'metadata/snapshot.json', + 'timestamp.json': 'metadata/timestamp.json'} + + If 'metadata_directory' is not set by the caller, the current directory is + used. + + + metadata_directory: + The directory containing the metadata files. + + + tuf.FormatError, if 'metadata_directory' is improperly formatted. + + + None. + + + A dictionary containing the expected filenames of the top-level + metadata files, such as 'root.json' and 'snapshot.json'. + """ + + if metadata_directory is None: + metadata_directory = os.getcwd() + + # Does 'metadata_directory' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(metadata_directory) + + # Store the filepaths of the top-level roles, including the + # 'metadata_directory' for each one. + filenames = {} + + filenames[ROOT_FILENAME] = \ + os.path.join(metadata_directory, ROOT_FILENAME) + + filenames[TARGETS_FILENAME] = \ + os.path.join(metadata_directory, TARGETS_FILENAME) + + filenames[SNAPSHOT_FILENAME] = \ + os.path.join(metadata_directory, SNAPSHOT_FILENAME) + + filenames[TIMESTAMP_FILENAME] = \ + os.path.join(metadata_directory, TIMESTAMP_FILENAME) + + return filenames + + + + + +def get_metadata_fileinfo(filename): + """ + + Retrieve the file information of 'filename'. The object returned + conforms to 'tuf.formats.FILEINFO_SCHEMA'. The information + generated for 'filename' is stored in metadata files like 'targets.json'. + The fileinfo object returned has the form: + + fileinfo = {'length': 1024, + 'hashes': {'sha256': 1233dfba312, ...}, + 'custom': {...}} + + + filename: + The metadata file whose file information is needed. It must exist. + + + tuf.FormatError, if 'filename' is improperly formatted. + + tuf.Error, if 'filename' doesn't exist. + + + The file is opened and information about the file is generated, + such as file size and its hash. + + + A dictionary conformant to 'tuf.formats.FILEINFO_SCHEMA'. This + dictionary contains the length, hashes, and custom data about the + 'filename' metadata file. SHA256 hashes are generated by default. + """ + + # Does 'filename' have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(filename) + + if not os.path.isfile(filename): + message = repr(filename) + ' is not a file.' + raise tuf.Error(message) + + # Note: 'filehashes' is a dictionary of the form + # {'sha256': 1233dfba312, ...}. 'custom' is an optional + # dictionary that a client might define to include additional + # file information, such as the file's author, version/revision + # numbers, etc. + filesize, filehashes = \ + tuf.util.get_file_details(filename, tuf.conf.REPOSITORY_HASH_ALGORITHMS) + custom = None + + return tuf.formats.make_fileinfo(filesize, filehashes, custom) + + + + + + +def get_target_hash(target_filepath): + """ + + Compute the hash of 'target_filepath'. This is useful in conjunction with + the "path_hash_prefixes" attribute in a delegated targets role, which + tells us which paths it is implicitly responsible for. + + The repository may optionally organize targets into hashed bins to ease + target delegations and role metadata management. The use of consistent + hashing allows for a uniform distribution of targets into bins. + + + target_filepath: + The path to the target file on the repository. This will be relative to + the 'targets' (or equivalent) directory on a given mirror. + + + None. + + + None. + + + The hash of 'target_filepath'. + """ + + return tuf.util.get_target_hash(target_filepath) + + + + + +def generate_root_metadata(version, expiration_date, consistent_snapshot): + """ + + Create the root metadata. 'tuf.roledb.py' and 'tuf.keydb.py' are read and + the information returned by these modules is used to generate the root + metadata object. + + + version: + The metadata version number. Clients use the version number to + determine if the downloaded version is newer than the one currently + trusted. + + expiration_date: + The expiration date of the metadata file. Conformant to + 'tuf.formats.ISO8601_DATETIME_SCHEMA'. + + consistent_snapshot: + Boolean. If True, a file digest is expected to be prepended to the + filename of any target file located in the targets directory. Each digest + is stripped from the target filename and listed in the snapshot metadata. + + + tuf.FormatError, if the generated root metadata object could not + be generated with the correct format. + + tuf.Error, if an error is encountered while generating the root + metadata object (e.g., a required top-level role not found in 'tuf.roledb'.) + + + The contents of 'tuf.keydb.py' and 'tuf.roledb.py' are read. + + + A root metadata object, conformant to 'tuf.formats.ROOT_SCHEMA'. + """ + + # Do the arguments have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if any of the arguments are improperly formatted. + tuf.formats.METADATAVERSION_SCHEMA.check_match(version) + tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) + tuf.formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot) + + # The role and key dictionaries to be saved in the root metadata object. + # Conformant to 'ROLEDICT_SCHEMA' and 'KEYDICT_SCHEMA', respectively. + roledict = {} + keydict = {} + + # Extract the role, threshold, and keyid information of the top-level roles, + # which Root stores in its metadata. The necessary role metadata is generated + # from this information. + for rolename in ['root', 'targets', 'snapshot', 'timestamp']: + + # If a top-level role is missing from 'tuf.roledb.py', raise an exception. + if not tuf.roledb.role_exists(rolename): + raise tuf.Error(repr(rolename) + ' not in "tuf.roledb".') + + # Keep track of the keys loaded to avoid duplicates. + keyids = [] + + # Generate keys for the keyids listed by the role being processed. + for keyid in tuf.roledb.get_role_keyids(rolename): + key = tuf.keydb.get_key(keyid) + + # If 'key' is an RSA key, it would conform to 'tuf.formats.RSAKEY_SCHEMA', + # and have the form: + # {'keytype': 'rsa', + # 'keyid': keyid, + # 'keyval': {'public': '-----BEGIN RSA PUBLIC KEY----- ...', + # 'private': '-----BEGIN RSA PRIVATE KEY----- ...'}} + keyid = key['keyid'] + if keyid not in keydict: + + # This appears to be a new keyid. Generate the key for it. + if key['keytype'] in ['rsa', 'ed25519']: + keytype = key['keytype'] + keyval = key['keyval'] + keydict[keyid] = \ + tuf.keys.format_keyval_to_metadata(keytype, keyval, private=False) + + # This is not a recognized key. Raise an exception. + else: + raise tuf.Error('Unsupported keytype: '+keyid) + + # Do we have a duplicate? + if keyid in keyids: + raise tuf.Error('Same keyid listed twice: '+keyid) + + # Add the loaded keyid for the role being processed. + keyids.append(keyid) + + # Generate and store the role data belonging to the processed role. + role_threshold = tuf.roledb.get_role_threshold(rolename) + role_metadata = tuf.formats.make_role_metadata(keyids, role_threshold) + roledict[rolename] = role_metadata + + # Generate the root metadata object. + root_metadata = tuf.formats.RootFile.make_metadata(version, expiration_date, + keydict, roledict, + consistent_snapshot) + + return root_metadata + + + + + +def generate_targets_metadata(targets_directory, target_files, version, + expiration_date, delegations=None, + write_consistent_targets=False): + """ + + Generate the targets metadata object. The targets in 'target_files' must + exist at the same path they should on the repo. 'target_files' is a list of + targets. The 'custom' field of the targets metadata is not currently + supported. + + + targets_directory: + The directory containing the target files and directories of the + repository. + + target_files: + The target files tracked by 'targets.json'. 'target_files' is a list of + target paths that are relative to the targets directory (e.g., + ['file1.txt', 'Django/module.py']). + + version: + The metadata version number. Clients use the version number to + determine if the downloaded version is newer than the one currently + trusted. + + expiration_date: + The expiration date of the metadata file. Conformant to + 'tuf.formats.ISO8601_DATETIME_SCHEMA'. + + delegations: + The delegations made by the targets role to be generated. 'delegations' + must match 'tuf.formats.DELEGATIONS_SCHEMA'. + + write_consistent_targets: + Boolean that indicates whether file digests should be prepended to the + target files. + + + tuf.FormatError, if an error occurred trying to generate the targets + metadata object. + + tuf.Error, if any of the target files cannot be read. + + + The target files are read and file information generated about them. + + + A targets metadata object, conformant to 'tuf.formats.TARGETS_SCHEMA'. + """ + + # Do the arguments have the correct format? + # Ensure the arguments have the appropriate number of objects and object + # types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(targets_directory) + tuf.formats.PATHS_SCHEMA.check_match(target_files) + tuf.formats.METADATAVERSION_SCHEMA.check_match(version) + tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) + tuf.formats.BOOLEAN_SCHEMA.check_match(write_consistent_targets) + + if delegations is not None: + tuf.formats.DELEGATIONS_SCHEMA.check_match(delegations) + + # Store the file attributes of targets in 'target_files'. 'filedict', + # conformant to 'tuf.formats.FILEDICT_SCHEMA', is added to the targets + # metadata object returned. + filedict = {} + + # Ensure the user is aware of a non-existent 'target_directory', and convert + # it to its abosolute path, if it exists. + targets_directory = _check_directory(targets_directory) + + # Generate the fileinfo of all the target files listed in 'target_files'. + for target in target_files: + + # The root-most folder of the targets directory should not be included in + # target paths listed in targets metadata. + # (e.g., 'targets/more_targets/somefile.txt' -> 'more_targets/somefile.txt') + relative_targetpath = target + + # Note: join() discards 'targets_directory' if 'target' contains a leading + # path separator (i.e., is treated as an absolute path). + target_path = os.path.join(targets_directory, target.lstrip(os.sep)) + + # Ensure all target files listed in 'target_files' exist. If just one of + # these files does not exist, raise an exception. + if not os.path.exists(target_path): + message = repr(target_path) + ' cannot be read. Unable to generate '+ \ + 'targets metadata.' + raise tuf.Error(message) + + filedict[relative_targetpath] = get_metadata_fileinfo(target_path) + + if write_consistent_targets: + for target_digest in filedict[relative_targetpath]['hashes']: + dirname, basename = os.path.split(target_path) + digest_filename = target_digest + '.' + basename + digest_target = os.path.join(dirname, digest_filename) + + if not os.path.exists(digest_target): + logger.warning('Hard linking target file to ' + repr(digest_target)) + os.link(target_path, digest_target) + + # Generate the targets metadata object. + targets_metadata = tuf.formats.TargetsFile.make_metadata(version, + expiration_date, + filedict, + delegations) + + return targets_metadata + + + + + +def generate_snapshot_metadata(metadata_directory, version, expiration_date, + root_filename, targets_filename, + consistent_snapshot=False): + """ + + Create the snapshot metadata. The minimum metadata must exist + (i.e., 'root.json' and 'targets.json'). This will also look through + the 'targets/' directory in 'metadata_directory' and the resulting + snapshot file will list all the delegated roles. + + + metadata_directory: + The directory containing the 'root.json' and 'targets.json' metadata + files. + + version: + The metadata version number. Clients use the version number to + determine if the downloaded version is newer than the one currently + trusted. + + expiration_date: + The expiration date of the metadata file. + Conformant to 'tuf.formats.ISO8601_DATETIME_SCHEMA'. + + root_filename: + The filename of the top-level root role. The hash and file size of this + file is listed in the snapshot role. + + targets_filename: + The filename of the top-level targets role. The hash and file size of + this file is listed in the snapshot role. + + consistent_snapshot: + Boolean. If True, a file digest is expected to be prepended to the + filename of any target file located in the targets directory. Each digest + is stripped from the target filename and listed in the snapshot metadata. + + + tuf.FormatError, if the arguments are improperly formatted. + + tuf.Error, if an error occurred trying to generate the snapshot metadata + object. + + + The 'root.json' and 'targets.json' files are read. + + + The snapshot metadata object, conformant to 'tuf.formats.SNAPSHOT_SCHEMA'. + """ + + # Do the arguments have the correct format? + # This check ensures arguments have the appropriate number of objects and + # object types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.PATH_SCHEMA.check_match(metadata_directory) + tuf.formats.METADATAVERSION_SCHEMA.check_match(version) + tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) + tuf.formats.PATH_SCHEMA.check_match(root_filename) + tuf.formats.PATH_SCHEMA.check_match(targets_filename) + tuf.formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot) + + metadata_directory = _check_directory(metadata_directory) + + # Retrieve the fileinfo of 'root.json' and 'targets.json'. This file + # information includes data such as file length, hashes of the file, etc. + filedict = {} + filedict[ROOT_FILENAME] = get_metadata_fileinfo(root_filename) + filedict[TARGETS_FILENAME] = get_metadata_fileinfo(targets_filename) + + # Add compressed versions of the 'targets.json' and 'root.json' metadata, + # if they exist. + for extension in SUPPORTED_COMPRESSION_EXTENSIONS: + compressed_root_filename = root_filename+extension + compressed_targets_filename = targets_filename+extension + + # If the compressed versions of the root and targets metadata is found, + # add their file attributes to 'filedict'. + if os.path.exists(compressed_root_filename): + filedict[ROOT_FILENAME+extension] = \ + get_metadata_fileinfo(compressed_root_filename) + if os.path.exists(compressed_targets_filename): + filedict[TARGETS_FILENAME+extension] = \ + get_metadata_fileinfo(compressed_targets_filename) + + # Walk the 'targets/' directory and generate the fileinfo of all the role + # files found. This information is stored in the 'meta' field of the snapshot + # metadata object. + targets_metadata = os.path.join(metadata_directory, 'targets') + if os.path.exists(targets_metadata) and os.path.isdir(targets_metadata): + for directory_path, junk_directories, files in os.walk(targets_metadata): + + # 'files' here is a list of file names. + for basename in files: + metadata_path = os.path.join(directory_path, basename) + metadata_name = \ + metadata_path[len(metadata_directory):].lstrip(os.path.sep) + + # Strip the digest if 'consistent_snapshot' is True. + # Example: 'targets/unclaimed/13df98ab0.django.json' --> + # 'targets/unclaimed/django.json' + metadata_name, digest_junk = \ + _strip_consistent_snapshot_digest(metadata_name, consistent_snapshot) + + # All delegated roles are added to the snapshot file, including + # compressed versions. + for metadata_extension in METADATA_EXTENSIONS: + if metadata_name.endswith(metadata_extension): + rolename = metadata_name[:-len(metadata_extension)] + + # Obsolete role files may still be found. Ensure only roles loaded + # in the roledb are included in the snapshot metadata. + if tuf.roledb.role_exists(rolename): + filedict[metadata_name] = get_metadata_fileinfo(metadata_path) + + # Generate the snapshot metadata object. + snapshot_metadata = tuf.formats.SnapshotFile.make_metadata(version, + expiration_date, + filedict) + + return snapshot_metadata + + + + + +def generate_timestamp_metadata(snapshot_filename, version, + expiration_date, compressions=()): + """ + + Generate the timestamp metadata object. The 'snapshot.json' file must + exist. + + + snapshot_filename: + The required filename of the snapshot metadata file. The timestamp role + needs to the calculate the file size and hash of this file. + + version: + The timestamp's version number. Clients use the version number to + determine if the downloaded version is newer than the one currently + trusted. + + expiration_date: + The expiration date of the metadata file, conformant to + 'tuf.formats.ISO8601_DATETIME_SCHEMA'. + + compressions: + Compression extensions (e.g., 'gz'). If 'snapshot.json' is also saved in + compressed form, these compression extensions should be stored in + 'compressions' so the compressed timestamp files can be added to the + timestamp metadata object. + + + tuf.FormatError, if the generated timestamp metadata object cannot be + formatted correctly, or one of the arguments is improperly formatted. + + + None. + + + A timestamp metadata object, conformant to 'tuf.formats.TIMESTAMP_SCHEMA'. + """ + + # Do the arguments have the correct format? + # This check ensures arguments have the appropriate number of objects and + # object types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.PATH_SCHEMA.check_match(snapshot_filename) + tuf.formats.METADATAVERSION_SCHEMA.check_match(version) + tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) + tuf.formats.COMPRESSIONS_SCHEMA.check_match(compressions) + + # Retrieve the fileinfo of the snapshot metadata file. + # This file information contains hashes, file length, custom data, etc. + fileinfo = {} + fileinfo[SNAPSHOT_FILENAME] = get_metadata_fileinfo(snapshot_filename) + + # Save the fileinfo of the compressed versions of 'timestamp.json' + # in 'fileinfo'. Log the files included in 'fileinfo'. + for file_extension in compressions: + if not len(file_extension): + continue + + compressed_filename = snapshot_filename + '.' + file_extension + try: + compressed_fileinfo = get_metadata_fileinfo(compressed_filename) + + except: + logger.warning('Cannot get fileinfo about ' + repr(compressed_filename)) + + else: + logger.info('Including fileinfo about ' + repr(compressed_filename)) + fileinfo[SNAPSHOT_FILENAME + '.' + file_extension] = compressed_fileinfo + + # Generate the timestamp metadata object. + timestamp_metadata = tuf.formats.TimestampFile.make_metadata(version, + expiration_date, + fileinfo) + + return timestamp_metadata + + + + + +def sign_metadata(metadata_object, keyids, filename): + """ + + Sign a metadata object. If any of the keyids have already signed the file, + the old signature is replaced. The keys in 'keyids' must already be + loaded in 'tuf.keydb'. + + + metadata_object: + The metadata object to sign. For example, 'metadata' might correspond to + 'tuf.formats.ROOT_SCHEMA' or 'tuf.formats.TARGETS_SCHEMA'. + + keyids: + The keyids list of the signing keys. + + filename: + The intended filename of the signed metadata object. + For example, 'root.json' or 'targets.json'. This function + does NOT save the signed metadata to this filename. + + + tuf.FormatError, if a valid 'signable' object could not be generated or + the arguments are improperly formatted. + + tuf.Error, if an invalid keytype was found in the keystore. + + + None. + + + A signable object conformant to 'tuf.formats.SIGNABLE_SCHEMA'. + """ + + # Do the arguments have the correct format? + # This check ensures arguments have the appropriate number of objects and + # object types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.ANYROLE_SCHEMA.check_match(metadata_object) + tuf.formats.KEYIDS_SCHEMA.check_match(keyids) + tuf.formats.PATH_SCHEMA.check_match(filename) + + # Make sure the metadata is in 'signable' format. That is, + # it contains a 'signatures' field containing the result + # of signing the 'signed' field of 'metadata' with each + # keyid of 'keyids'. + signable = tuf.formats.make_signable(metadata_object) + + # Sign the metadata with each keyid in 'keyids'. + for keyid in keyids: + + # Load the signing key. + key = tuf.keydb.get_key(keyid) + logger.info('Signing ' + repr(filename) + ' with ' + key['keyid']) + + # Create a new signature list. If 'keyid' is encountered, do not add it + # to the new list. + signatures = [] + for signature in signable['signatures']: + if not keyid == signature['keyid']: + signatures.append(signature) + signable['signatures'] = signatures + + # Generate the signature using the appropriate signing method. + if key['keytype'] in SUPPORTED_KEY_TYPES: + if len(key['keyval']['private']): + signed = signable['signed'] + signature = tuf.keys.create_signature(key, signed) + signable['signatures'].append(signature) + + else: + logger.warning('Private key unset. Skipping: ' + repr(keyid)) + + else: + raise tuf.Error('The keydb contains a key with an invalid key type.') + + # Raise 'tuf.FormatError' if the resulting 'signable' is not formatted + # correctly. + tuf.formats.check_signable_object_format(signable) + + return signable + + + + + +def write_metadata_file(metadata, filename, compressions, consistent_snapshot): + """ + + If necessary, write the 'metadata' signable object to 'filename', and the + compressed version of the metadata file if 'compression' is set. + Note: Compression algorithms like gzip attach a timestamp to compressed + files, so a metadata file compressed multiple times may generate different + digests even though the uncompressed content has not changed. + + + metadata: + The object that will be saved to 'filename', conformant to + 'tuf.formats.SIGNABLE_SCHEMA'. + + filename: + The filename of the metadata to be written (e.g., 'root.json'). + If a compression algorithm is specified in 'compressions', the + compression extention is appended to 'filename'. + + compressions: + Specify the algorithms, as a list of strings, used to compress the file; + The only currently available compression option is 'gz' (gzip). + + consistent_snapshot: + Boolean that determines whether the metadata file's digest should be + prepended to the filename. + + + tuf.FormatError, if the arguments are improperly formatted. + + tuf.Error, if the directory of 'filename' does not exist. + + Any other runtime (e.g., IO) exception. + + + The 'filename' (or the compressed filename) file is created, or overwritten + if it exists. + + + None. + """ + + # Do the arguments have the correct format? + # This check ensures arguments have the appropriate number of objects and + # object types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.SIGNABLE_SCHEMA.check_match(metadata) + tuf.formats.PATH_SCHEMA.check_match(filename) + tuf.formats.COMPRESSIONS_SCHEMA.check_match(compressions) + tuf.formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot) + + # Verify the directory of 'filename', and convert 'filename' to its absolute + # path so that temporary files are moved to their expected destinations. + filename = os.path.abspath(filename) + written_filename = filename + _check_directory(os.path.dirname(filename)) + consistent_filenames = [] + + # Generate the actual metadata file content of 'metadata'. Metadata is + # saved as json and includes formatting, such as indentation and sorted + # objects. The new digest of 'metadata' is also calculated to help determine + # if re-saving is required. + file_content, new_digests = _get_written_metadata_and_digests(metadata) + + if consistent_snapshot: + for new_digest in six.itervalues(new_digests): + dirname, basename = os.path.split(filename) + digest_and_filename = new_digest + '.' + basename + consistent_filenames.append(os.path.join(dirname, digest_and_filename)) + written_filename = consistent_filenames.pop() + + # Verify whether new metadata needs to be written (i.e., has not been + # previously written or has changed. + write_new_metadata = False + + # Has the uncompressed metadata changed? Does it exist? If so, set + # 'write_compressed_version' to True so that it is written. + # compressed metadata should only be written if it does not exist or the + # uncompressed version has changed). + try: + file_length_junk, old_digests = tuf.util.get_file_details(written_filename) + if old_digests != new_digests: + write_new_metadata = True + + # 'tuf.Error' raised if 'filename' does not exist. + except tuf.Error as e: + write_new_metadata = True + + if write_new_metadata: + # The 'metadata' object is written to 'file_object', including compressed + # versions. To avoid partial metadata from being written, 'metadata' is + # first written to a temporary location (i.e., 'file_object') and then moved + # to 'filename'. + file_object = tuf.util.TempFile() + + # Serialize 'metadata' to the file-like object and then write + # 'file_object' to disk. The dictionary keys of 'metadata' are sorted + # and indentation is used. The 'tuf.util.TempFile' file-like object is + # automically closed after the final move. + file_object.write(file_content) + logger.info('Saving ' + repr(written_filename)) + file_object.move(written_filename) + + for consistent_filename in consistent_filenames: + logger.info('Linking ' + repr(consistent_filename)) + os.link(written_filename, consistent_filename) + + + # Generate the compressed versions of 'metadata', if necessary. A compressed + # file may be written (without needing to write the uncompressed version) if + # the repository maintainer adds compression after writing the uncompressed + # version. + for compression in compressions: + file_object = None + + # Ignore the empty string that signifies non-compression. The uncompressed + # file was previously written above, if necessary. + if not len(compression): + continue + + elif compression == 'gz': + file_object = tuf.util.TempFile() + compressed_filename = filename + '.gz' + + # Instantiate a gzip object, but save compressed content to + # 'file_object' (i.e., GzipFile instance is based on its 'fileobj' + # argument). + gzip_object = gzip.GzipFile(fileobj=file_object, mode='wb') + try: + gzip_object.write(file_content) + finally: + gzip_object.close() + + else: + raise tuf.FormatError('Unknown compression algorithm: '+repr(compression)) + + # Save the compressed version, ensuring an unchanged file is not re-saved. + # Re-saving the same compressed version may cause its digest to unexpectedly + # change (gzip includes a timestamp) even though content has not changed. + _write_compressed_metadata(file_object, compressed_filename, + write_new_metadata, consistent_snapshot) + return written_filename + + + + + +def _write_compressed_metadata(file_object, compressed_filename, + write_new_metadata, consistent_snapshot): + """ + Write compressed versions of metadata, ensuring compressed file that have + not changed are not re-written, the digest of the compressed file is properly + added to the compressed filename, and consistent snapshots are also saved. + Ensure compressed files are written to a temporary location, and then + moved to their destinations. + """ + + # If a consistent snapshot is unneeded, 'file_object' may be simply moved + # 'compressed_filename' if not already written. + if not consistent_snapshot: + if not os.path.exists(compressed_filename) or write_new_metadata: + file_object.move(compressed_filename) + + # The temporary file must be closed if 'file_object.move()' is not used. + # tuf.util.TempFile() automatically closes the temp file when move() is + # called + else: + file_object.close_temp_file() + + # Consistent snapshots = True. Ensure the file's digest is included in the + # compressed filename written, provided it does not already exist. + else: + compressed_content = file_object.read() + new_digests = [] + consistent_filenames = [] + + # Multiple snapshots may be written if the repository uses multiple + # hash algorithms. Generate the digest of the compressed content. + for hash_algorithm in tuf.conf.REPOSITORY_HASH_ALGORITHMS: + digest_object = tuf.hash.digest(hash_algorithm) + digest_object.update(compressed_content) + new_digests.append(digest_object.hexdigest()) + + # Attach each digest to the compressed consistent snapshot filename. + for new_digest in new_digests: + dirname, basename = os.path.split(compressed_filename) + digest_and_filename = new_digest + '.' + basename + consistent_filenames.append(os.path.join(dirname, digest_and_filename)) + + # Move the 'tuf.util.TempFile' object to one of the filenames so that it is + # saved and the temporary file closed. Any remaining consistent snapshots + # may still need to be copied or linked. + compressed_filename = consistent_filenames.pop() + if not os.path.exists(compressed_filename): + logger.info('Saving ' + repr(compressed_filename)) + file_object.move(compressed_filename) + + # Save any remaining compressed consistent snapshots. + for consistent_filename in consistent_filenames: + if not os.path.exists(consistent_filename): + logger.info('Linking ' + repr(consistent_filename)) + os.link(compressed_filename, consistent_filename) + + + + + +def _log_status_of_top_level_roles(targets_directory, metadata_directory): + """ + Non-public function that logs whether any of the top-level roles contain an + invalid number of public and private keys, or an insufficient threshold of + signatures. Considering that the top-level metadata have to be verified in + the expected root -> targets -> snapshot -> timestamp order, this function + logs the error message and returns as soon as a required metadata file is + found to be invalid. It is assumed here that the delegated roles have been + written and verified. Example output: + + 'root' role contains 1 / 1 signatures. + 'targets' role contains 1 / 1 signatures. + 'snapshot' role contains 1 / 1 signatures. + 'timestamp' role contains 1 / 1 signatures. + + Note: Temporary metadata is generated so that file hashes & sizes may be + computed and verified against the attached signatures. 'metadata_directory' + should be a directory in a temporary repository directory. + """ + + # The expected full filenames of the top-level roles needed to write them to + # disk. + filenames = get_metadata_filenames(metadata_directory) + root_filename = filenames[ROOT_FILENAME] + targets_filename = filenames[TARGETS_FILENAME] + snapshot_filename = filenames[SNAPSHOT_FILENAME] + timestamp_filename = filenames[TIMESTAMP_FILENAME] + + # Verify that the top-level roles contain a valid number of public keys and + # that their corresponding private keys have been loaded. + for rolename in ['root', 'targets', 'snapshot', 'timestamp']: + try: + _check_role_keys(rolename) + + except tuf.InsufficientKeysError as e: + logger.info(str(e)) + return + + # Do the top-level roles contain a valid threshold of signatures? Top-level + # metadata is verified in Root -> Targets -> Snapshot -> Timestamp order. + # Verify the metadata of the Root role. + try: + signable, root_filename = \ + _generate_and_write_metadata('root', root_filename, False, + targets_directory, metadata_directory) + _log_status('root', signable) + + # 'tuf.UnsignedMetadataError' raised if metadata contains an invalid threshold + # of signatures. log the valid/threshold message, where valid < threshold. + except tuf.UnsignedMetadataError as e: + _log_status('root', e.signable) + return + + # Verify the metadata of the Targets role. + try: + signable, targets_filename = \ + _generate_and_write_metadata('targets', targets_filename, False, + targets_directory, metadata_directory) + _log_status('targets', signable) + + except tuf.UnsignedMetadataError as e: + _log_status('targets', e.signable) + return + + # Verify the metadata of the snapshot role. + filenames = {'root': root_filename, 'targets': targets_filename} + try: + signable, snapshot_filename = \ + _generate_and_write_metadata('snapshot', snapshot_filename, False, + targets_directory, metadata_directory, + False, filenames) + _log_status('snapshot', signable) + + except tuf.UnsignedMetadataError as e: + _log_status('snapshot', e.signable) + return + + # Verify the metadata of the Timestamp role. + filenames = {'snapshot': snapshot_filename} + try: + signable, snapshot_filename = \ + _generate_and_write_metadata('timestamp', snapshot_filename, False, + targets_directory, metadata_directory, + False, filenames) + _log_status('timestamp', signable) + + except tuf.UnsignedMetadataError as e: + _log_status('timestamp', e.signable) + return + + + + +def _log_status(rolename, signable): + """ + Non-public function logs the number of (good/threshold) signatures of + 'rolename'. + """ + + status = tuf.sig.get_signature_status(signable, rolename) + + message = repr(rolename) + ' role contains ' + repr(len(status['good_sigs']))+\ + ' / ' + repr(status['threshold']) + ' signatures.' + logger.info(message) + + + + + +def create_tuf_client_directory(repository_directory, client_directory): + """ + + Create a client directory structure that the 'tuf.interposition' package + and 'tuf.client.updater' module expect of clients. Metadata files + downloaded from a remote TUF repository are saved to 'client_directory'. + The Root file must initially exist before an update request can be + satisfied. create_tuf_client_directory() ensures the minimum metadata + is copied and that required directories ('previous' and 'current') are + created in 'client_directory'. Software updaters integrating TUF may + use the client directory created as an initial copy of the repository's + metadadata. + + + repository_directory: + The path of the root repository directory. The 'metadata' and 'targets' + sub-directories should be available in 'repository_directory'. The + metadata files of 'repository_directory' are copied to 'client_directory'. + + client_directory: + The path of the root client directory. The 'current' and 'previous' + sub-directies are created and will store the metadata files copied + from 'repository_directory'. 'client_directory' will store metadata + and target files downloaded from a TUF repository. + + + tuf.FormatError, if the arguments are improperly formatted. + + tuf.RepositoryError, if the metadata directory in 'client_directory' + already exists. + + + Copies metadata files and directories from 'repository_directory' to + 'client_directory'. Parent directories are created if they do not exist. + + + None. + """ + + # Do the arguments have the correct format? + # This check ensures arguments have the appropriate number of objects and + # object types, and that all dict keys are properly named. + # Raise 'tuf.FormatError' if the check fails. + tuf.formats.PATH_SCHEMA.check_match(repository_directory) + tuf.formats.PATH_SCHEMA.check_match(client_directory) + + # Set the absolute path of the Repository's metadata directory. The metadata + # directory should be the one served by the Live repository. At a minimum, + # the repository's root file must be copied. + repository_directory = os.path.abspath(repository_directory) + metadata_directory = os.path.join(repository_directory, + METADATA_DIRECTORY_NAME) + + # Set the client's metadata directory, which will store the metadata copied + # from the repository directory set above. + client_directory = os.path.abspath(client_directory) + client_metadata_directory = os.path.join(client_directory, + METADATA_DIRECTORY_NAME) + + # If the client's metadata directory does not already exist, create it and + # any of its parent directories, otherwise raise an exception. An exception + # is raised to avoid accidently overwritting previous metadata. + try: + os.makedirs(client_metadata_directory) + + except OSError as e: + if e.errno == errno.EEXIST: + message = 'Cannot create a fresh client metadata directory: '+ \ + repr(client_metadata_directory)+'. Already exists.' + raise tuf.RepositoryError(message) + + else: + raise + + # Move all metadata to the client's 'current' and 'previous' directories. + # The root metadata file MUST exist in '{client_metadata_directory}/current'. + # 'tuf.interposition' and 'tuf.client.updater.py' expect the 'current' and + # 'previous' directories to exist under 'metadata'. + client_current = os.path.join(client_metadata_directory, 'current') + client_previous = os.path.join(client_metadata_directory, 'previous') + shutil.copytree(metadata_directory, client_current) + shutil.copytree(metadata_directory, client_previous) + + + +def disable_console_log_messages(): + """ + + Disable logger messages printed to the console. For example, repository + maintainers may want to call this function if many roles will be sharing + keys, otherwise detected duplicate keys will continually log a warning + message. + + + None. + + + None. + + + Removes the 'tuf.log' console handler, added by default when + 'tuf.repository_tool.py' is imported. + + + None. + """ + + tuf.log.remove_console_handler() + + + +if __name__ == '__main__': + # The interactive sessions of the documentation strings can + # be tested by running repository_tool.py as a standalone module: + # $ python repository_lib.py. + import doctest + doctest.testmod() diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 84c278ef..598e20fb 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ repository_tool.py @@ -24,18 +26,16 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division +from __future__ import unicode_literals import os import errno -import sys import time import datetime -import getpass import logging import tempfile import shutil import json -import gzip import random import tuf @@ -47,7 +47,18 @@ import tuf.sig import tuf.log import tuf.conf +import tuf.repository_lib as repo_lib +from tuf.repository_lib import generate_and_write_rsa_keypair +from tuf.repository_lib import generate_and_write_ed25519_keypair +from tuf.repository_lib import import_rsa_publickey_from_file +from tuf.repository_lib import import_ed25519_publickey_from_file +from tuf.repository_lib import import_rsa_privatekey_from_file +from tuf.repository_lib import import_ed25519_privatekey_from_file +from tuf.repository_lib import create_tuf_client_directory +from tuf.repository_lib import disable_console_log_messages import tuf._vendor.iso8601 as iso8601 +import tuf._vendor.six as six + # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.repository_tool') @@ -57,41 +68,19 @@ tuf.log.add_console_handler() tuf.log.set_console_log_level(logging.WARNING) -# Recommended RSA key sizes: -# http://www.emc.com/emc-plus/rsa-labs/historical/twirl-and-rsa-key-size.htm#table1 -# According to the document above, revised May 6, 2003, RSA keys of -# size 3072 provide security through 2031 and beyond. 2048-bit keys -# are the recommended minimum and are good from the present through 2030. -DEFAULT_RSA_KEY_BITS = 3072 - # The algorithm used by the repository to generate the digests of the # target filepaths, which are included in metadata files and may be prepended # to the filenames of consistent snapshots. HASH_FUNCTION = 'sha256' -# The extension of TUF metadata. -METADATA_EXTENSION = '.json' - -# The metadata filenames of the top-level roles. -ROOT_FILENAME = 'root' + METADATA_EXTENSION -TARGETS_FILENAME = 'targets' + METADATA_EXTENSION -SNAPSHOT_FILENAME = 'snapshot' + METADATA_EXTENSION -TIMESTAMP_FILENAME = 'timestamp' + METADATA_EXTENSION - # The targets and metadata directory names. Metadata files are written # to the staged metadata directory instead of the "live" one. METADATA_STAGED_DIRECTORY_NAME = 'metadata.staged' METADATA_DIRECTORY_NAME = 'metadata' TARGETS_DIRECTORY_NAME = 'targets' -# The full list of supported TUF metadata extensions. -METADATA_EXTENSIONS = ['.json', '.json.gz'] - -# The recognized compression extensions. -SUPPORTED_COMPRESSION_EXTENSIONS = ['.gz'] - -# Supported key types. -SUPPORTED_KEY_TYPES = ['rsa', 'ed25519'] +# The extension of TUF metadata. +METADATA_EXTENSION = '.json' # Expiration date delta, in seconds, of the top-level roles. A metadata # expiration date is set by taking the current time and adding the expiration @@ -109,13 +98,6 @@ # Initial 'timestamp.json' expiration time of 1 day. TIMESTAMP_EXPIRATION = 86400 -# Log warning when metadata expires in n days, or less. -# root = 1 month, snapshot = 1 day, targets = 10 days, timestamp = 1 day. -ROOT_EXPIRES_WARN_SECONDS = 2630000 -SNAPSHOT_EXPIRES_WARN_SECONDS = 86400 -TARGETS_EXPIRES_WARN_SECONDS = 864000 -TIMESTAMP_EXPIRES_WARN_SECONDS = 86400 - try: tuf.keys.check_crypto_libraries(['rsa', 'ed25519', 'general']) @@ -137,7 +119,7 @@ class Repository(object): access by default: repository.root.version = 2 - repository.timestamp.expiration = datetime.datetime(2015, 08, 08, 12, 00) + repository.timestamp.expiration = datetime.datetime(2015, 8, 8, 12, 0) repository.snapshot.add_verification_key(...) repository.targets.delegate('unclaimed', ...) @@ -264,84 +246,67 @@ def write(self, write_partial=False, consistent_snapshot=False): # sub-directory. tuf.util.ensure_parent_dir(delegated_filename) - try: - _generate_and_write_metadata(delegated_rolename, delegated_filename, - write_partial, self._targets_directory, - self._metadata_directory, - consistent_snapshot) - - # Include only the exception message. - except tuf.UnsignedMetadataError, e: - raise tuf.UnsignedMetadataError(e[0]) - + repo_lib._generate_and_write_metadata(delegated_rolename, + delegated_filename, + write_partial, + self._targets_directory, + self._metadata_directory, + consistent_snapshot) + # Generate the 'root.json' metadata file. # _generate_and_write_metadata() raises a 'tuf.Error' exception if the # metadata cannot be written. - root_filename = 'root' + METADATA_EXTENSION + root_filename = repo_lib.ROOT_FILENAME root_filename = os.path.join(self._metadata_directory, root_filename) - try: - signable_junk, root_filename = \ - _generate_and_write_metadata('root', root_filename, write_partial, - self._targets_directory, - self._metadata_directory, - consistent_snapshot) - - # Include only the exception message. - except tuf.UnsignedMetadataError, e: - raise tuf.UnsignedMetadataError(e[0]) + signable_junk, root_filename = \ + repo_lib._generate_and_write_metadata('root', root_filename, write_partial, + self._targets_directory, + self._metadata_directory, + consistent_snapshot) + # Generate the 'targets.json' metadata file. - targets_filename = 'targets' + METADATA_EXTENSION + targets_filename = repo_lib.TARGETS_FILENAME targets_filename = os.path.join(self._metadata_directory, targets_filename) - try: - signable_junk, targets_filename = \ - _generate_and_write_metadata('targets', targets_filename, write_partial, - self._targets_directory, - self._metadata_directory, - consistent_snapshot) - # Include only the exception message. - except tuf.UnsignedMetadataError, e: - raise tuf.UnsignedMetadataError(e[0]) + signable_junk, targets_filename = \ + repo_lib._generate_and_write_metadata('targets', targets_filename, + write_partial, + self._targets_directory, + self._metadata_directory, + consistent_snapshot) # Generate the 'snapshot.json' metadata file. - snapshot_filename = os.path.join(self._metadata_directory, 'snapshot') - snapshot_filename = 'snapshot' + METADATA_EXTENSION + snapshot_filename = repo_lib.SNAPSHOT_FILENAME snapshot_filename = os.path.join(self._metadata_directory, snapshot_filename) filenames = {'root': root_filename, 'targets': targets_filename} snapshot_signable = None - try: - snapshot_signable, snapshot_filename = \ - _generate_and_write_metadata('snapshot', snapshot_filename, write_partial, - self._targets_directory, - self._metadata_directory, - consistent_snapshot, filenames) - - # Include only the exception message. - except tuf.UnsignedMetadataError, e: - raise tuf.UnsignedMetadataError(e[0]) + snapshot_signable, snapshot_filename = \ + repo_lib._generate_and_write_metadata('snapshot', snapshot_filename, + write_partial, + self._targets_directory, + self._metadata_directory, + consistent_snapshot, filenames) + # Generate the 'timestamp.json' metadata file. - timestamp_filename = 'timestamp' + METADATA_EXTENSION + timestamp_filename = repo_lib.TIMESTAMP_FILENAME timestamp_filename = os.path.join(self._metadata_directory, timestamp_filename) filenames = {'snapshot': snapshot_filename} - try: - _generate_and_write_metadata('timestamp', timestamp_filename, write_partial, - self._targets_directory, - self._metadata_directory, consistent_snapshot, - filenames) - # Include only the exception message. - except tuf.UnsignedMetadataError, e: - raise tuf.UnsignedMetadataError(e[0]) - + repo_lib._generate_and_write_metadata('timestamp', timestamp_filename, + write_partial, + self._targets_directory, + self._metadata_directory, + consistent_snapshot, filenames) # Delete the metadata of roles no longer in 'tuf.roledb'. Obsolete roles # may have been revoked and should no longer have their metadata files # available on disk, otherwise loading a repository may unintentionally load # them. - _delete_obsolete_metadata(self._metadata_directory, - snapshot_signable['signed'], consistent_snapshot) + repo_lib._delete_obsolete_metadata(self._metadata_directory, + snapshot_signable['signed'], + consistent_snapshot) @@ -427,16 +392,17 @@ def status(self): # Append any invalid roles to the 'insufficient_keys' and # 'insufficient_signatures' lists try: - _check_role_keys(delegated_role) + repo_lib._check_role_keys(delegated_role) - except tuf.InsufficientKeysError, e: + except tuf.InsufficientKeysError as e: insufficient_keys.append(delegated_role) continue try: - _generate_and_write_metadata(delegated_role, filename, False, - targets_directory, metadata_directory) - except tuf.UnsignedMetadataError, e: + repo_lib._generate_and_write_metadata(delegated_role, filename, False, + targets_directory, + metadata_directory) + except tuf.UnsignedMetadataError as e: insufficient_signatures.append(delegated_role) # Log the verification results of the delegated roles and return @@ -455,7 +421,8 @@ def status(self): return # Verify the top-level roles and log the results. - _log_status_of_top_level_roles(targets_directory, metadata_directory) + repo_lib._log_status_of_top_level_roles(targets_directory, + metadata_directory) finally: shutil.rmtree(temp_repository_directory, ignore_errors=True) @@ -598,9 +565,9 @@ def add_verification_key(self, key): try: tuf.keydb.add_key(key) - except tuf.KeyAlreadyExistsError, e: + except tuf.KeyAlreadyExistsError as e: message = 'Adding a verification key that has already been used.' - logger.warn(message) + logger.warning(message) keyid = key['keyid'] roleinfo = tuf.roledb.get_roleinfo(self.rolename) @@ -708,7 +675,7 @@ def load_signing_key(self, key): try: tuf.keydb.add_key(key) - except tuf.KeyAlreadyExistsError, e: + except tuf.KeyAlreadyExistsError as e: tuf.keydb.remove_key(key['keyid']) tuf.keydb.add_key(key) @@ -1209,8 +1176,8 @@ def compressions(self): A getter method that returns a list of the file compression algorithms used when the metadata is written to disk. If ['gz'] is set for the - 'targets.json' role, the metadata files 'targets.json' and 'targets.json.gz' - are written. + 'targets.json' role, the metadata files 'targets.json' and + 'targets.json.gz' are written. >>> >>> @@ -1334,7 +1301,7 @@ def __init__(self): try: tuf.roledb.add_role(self._rolename, roleinfo) - except tuf.RoleAlreadyExistsError, e: + except tuf.RoleAlreadyExistsError as e: pass @@ -1396,7 +1363,7 @@ def __init__(self): try: tuf.roledb.add_role(self.rolename, roleinfo) - except tuf.RoleAlreadyExistsError, e: + except tuf.RoleAlreadyExistsError as e: pass @@ -1452,7 +1419,7 @@ def __init__(self): try: tuf.roledb.add_role(self._rolename, roleinfo) - except tuf.RoleAlreadyExistsError, e: + except tuf.RoleAlreadyExistsError as e: pass @@ -1545,7 +1512,7 @@ def __init__(self, targets_directory, rolename='targets', roleinfo=None): try: tuf.roledb.add_role(self.rolename, roleinfo) - except tuf.RoleAlreadyExistsError, e: + except tuf.RoleAlreadyExistsError as e: pass @@ -1953,9 +1920,8 @@ def get_delegated_rolenames(self): return tuf.roledb.get_delegated_rolenames(self.rolename) - - def delegate(self, rolename, public_keys, list_of_targets, - threshold=1, restricted_paths=None, path_hash_prefixes=None): + def delegate(self, rolename, public_keys, list_of_targets, threshold=1, + backtrack=True, restricted_paths=None, path_hash_prefixes=None): """ Create a new delegation, where 'rolename' is a child delegation of this @@ -1986,6 +1952,17 @@ def delegate(self, rolename, public_keys, list_of_targets, threshold: The threshold number of keys of 'rolename'. + + backtrack: + Boolean that indicates whether this role allows the updater client + to continue searching for targets (target files it is trusted to list + but has not yet specified) in other delegations. If 'backtrack' is + False and 'updater.target()' does not find 'example_target.tar.gz' in + this role, a 'tuf.UnknownTargetError' exception should be raised. If + 'backtrack' is True (default), and 'target/other_role' is also trusted + with 'example_target.tar.gz' and has listed it, updater.target() + should backtrack and return the target file specified by + 'target/other_role'. restricted_paths: A list of restricted directory or file paths of 'rolename'. Any target @@ -2019,17 +1996,21 @@ def delegate(self, rolename, public_keys, list_of_targets, tuf.formats.ANYKEYLIST_SCHEMA.check_match(public_keys) tuf.formats.RELPATHS_SCHEMA.check_match(list_of_targets) tuf.formats.THRESHOLD_SCHEMA.check_match(threshold) + tuf.formats.BOOLEAN_SCHEMA.check_match(backtrack) + if restricted_paths is not None: tuf.formats.RELPATHS_SCHEMA.check_match(restricted_paths) + if path_hash_prefixes is not None: tuf.formats.PATH_HASH_PREFIXES_SCHEMA.check_match(path_hash_prefixes) - + + # Check if 'rolename' is not already a delegation. 'tuf.roledb' expects the # full rolename. - full_rolename = self._rolename+'/'+rolename + full_rolename = self._rolename + '/' + rolename if tuf.roledb.role_exists(full_rolename): - raise tuf.Error(repr(full_rolename)+' already delegated.') + raise tuf.Error(repr(full_rolename) + ' already delegated.') # Keep track of the valid keyids (added to the new Targets object) and their # keydicts (added to this Targets delegations). @@ -2101,9 +2082,12 @@ def delegate(self, rolename, public_keys, list_of_targets, roleinfo = {'name': full_rolename, 'keyids': roleinfo['keyids'], 'threshold': roleinfo['threshold'], + 'backtrack': backtrack, 'paths': roleinfo['paths']} + if restricted_paths is not None: roleinfo['paths'] = relative_restricted_paths + if path_hash_prefixes is not None: roleinfo['path_hash_prefixes'] = path_hash_prefixes # A role in a delegations must list either 'path_hash_prefixes' @@ -2274,7 +2258,7 @@ def delegate_hashed_bins(self, list_of_targets, keys_of_hashed_bins, # target path, reduced to the first 'prefix_length' hex digits, is # calculated to determine which 'bin_index' is should go. target_paths_in_bin = {} - for bin_index in xrange(total_hash_prefixes): + for bin_index in six.moves.xrange(total_hash_prefixes): target_paths_in_bin[bin_index] = [] # Assign every path to its bin. Ensure every target is located under the @@ -2291,7 +2275,7 @@ def delegate_hashed_bins(self, list_of_targets, keys_of_hashed_bins, # '{repository_root}/targets/file1.txt' -> 'file1.txt'. relative_path = target_path[len(self._targets_directory):] digest_object = tuf.hash.digest(algorithm=HASH_FUNCTION) - digest_object.update(relative_path) + digest_object.update(relative_path.encode('utf-8')) relative_path_hash = digest_object.hexdigest() relative_path_hash_prefix = relative_path_hash[:prefix_length] @@ -2315,7 +2299,7 @@ def delegate_hashed_bins(self, list_of_targets, keys_of_hashed_bins, # The parent roles will list bin roles starting from "0" to # 'total_hash_prefixes' in 'bin_offset' increments. The skipped bin roles # are listed in 'path_hash_prefixes' of 'outer_bin_index. - for outer_bin_index in xrange(0, total_hash_prefixes, bin_offset): + for outer_bin_index in six.moves.xrange(0, total_hash_prefixes, bin_offset): # The bin index is hex padded from the left with zeroes for up to the # 'prefix_length' (e.g., 'targets/unclaimed/000-003'). Ensure the correct # hash bin name is generated if a prefix range is unneeded. @@ -2331,7 +2315,7 @@ def delegate_hashed_bins(self, list_of_targets, keys_of_hashed_bins, path_hash_prefixes = [] bin_rolename_targets = [] - for inner_bin_index in xrange(outer_bin_index, outer_bin_index+bin_offset): + for inner_bin_index in six.moves.xrange(outer_bin_index, outer_bin_index+bin_offset): # 'inner_bin_rolename' needed in padded hex. For example, "00b". inner_bin_rolename = hex(inner_bin_index)[2:].zfill(prefix_length) path_hash_prefixes.append(inner_bin_rolename) @@ -2424,7 +2408,7 @@ def add_target_to_bin(self, target_filepath): # '{repository_root}/targets/file1.txt' -> '/file1.txt'. relative_path = filepath[len(self._targets_directory):] digest_object = tuf.hash.digest(algorithm=HASH_FUNCTION) - digest_object.update(relative_path) + digest_object.update(relative_path.encode('utf-8')) path_hash = digest_object.hexdigest() path_hash_prefix = path_hash[:prefix_length] @@ -2475,528 +2459,7 @@ def delegations(self): A list containing the Targets objects of this Targets' delegations. """ - return self._delegated_roles.values() - - - - - -def _generate_and_write_metadata(rolename, metadata_filename, write_partial, - targets_directory, metadata_directory, - consistent_snapshot=False, filenames=None): - """ - Non-public function that can generate and write the metadata of the specified - top-level 'rolename'. It also increments version numbers if: - - 1. write_partial==True and the metadata is the first to be written. - - 2. write_partial=False (i.e., write()), the metadata was not loaded as - partially written, and a write_partial is not needed. - """ - - metadata = None - - # Retrieve the roleinfo of 'rolename' to extract the needed metadata - # attributes, such as version number, expiration, etc. - roleinfo = tuf.roledb.get_roleinfo(rolename) - snapshot_compressions = tuf.roledb.get_roleinfo('snapshot')['compressions'] - - # Generate the appropriate role metadata for 'rolename'. - if rolename == 'root': - metadata = generate_root_metadata(roleinfo['version'], - roleinfo['expires'], consistent_snapshot) - - _log_warning_if_expires_soon(ROOT_FILENAME, roleinfo['expires'], - ROOT_EXPIRES_WARN_SECONDS) - - # Check for the Targets role, including delegated roles. - elif rolename.startswith('targets'): - metadata = generate_targets_metadata(targets_directory, - roleinfo['paths'], - roleinfo['version'], - roleinfo['expires'], - roleinfo['delegations'], - consistent_snapshot) - if rolename == 'targets': - _log_warning_if_expires_soon(TARGETS_FILENAME, roleinfo['expires'], - TARGETS_EXPIRES_WARN_SECONDS) - - elif rolename == 'snapshot': - root_filename = filenames['root'] - targets_filename = filenames['targets'] - metadata = generate_snapshot_metadata(metadata_directory, - roleinfo['version'], - roleinfo['expires'], root_filename, - targets_filename, - consistent_snapshot) - - _log_warning_if_expires_soon(SNAPSHOT_FILENAME, roleinfo['expires'], - SNAPSHOT_EXPIRES_WARN_SECONDS) - - elif rolename == 'timestamp': - snapshot_filename = filenames['snapshot'] - metadata = generate_timestamp_metadata(snapshot_filename, - roleinfo['version'], - roleinfo['expires'], - snapshot_compressions) - - _log_warning_if_expires_soon(TIMESTAMP_FILENAME, roleinfo['expires'], - TIMESTAMP_EXPIRES_WARN_SECONDS) - - signable = sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename) - - # Check if the version number of 'rolename' may be automatically incremented, - # depending on whether if partial metadata is loaded or if the metadata is - # written with write() / write_partial(). - # Increment the version number if this is the first partial write. - if write_partial: - temp_signable = sign_metadata(metadata, [], metadata_filename) - temp_signable['signatures'].extend(roleinfo['signatures']) - status = tuf.sig.get_signature_status(temp_signable, rolename) - if len(status['good_sigs']) == 0: - metadata['version'] = metadata['version'] + 1 - signable = sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename) - # non-partial write() - else: - if tuf.sig.verify(signable, rolename) and not roleinfo['partial_loaded']: - metadata['version'] = metadata['version'] + 1 - signable = sign_metadata(metadata, roleinfo['signing_keyids'], - metadata_filename) - - # Write the metadata to file if contains a threshold of signatures. - signable['signatures'].extend(roleinfo['signatures']) - - if tuf.sig.verify(signable, rolename) or write_partial: - _remove_invalid_and_duplicate_signatures(signable) - compressions = roleinfo['compressions'] - filename = write_metadata_file(signable, metadata_filename, compressions, - consistent_snapshot) - - # The root and timestamp files should also be written without a digest if - # 'consistent_snaptshots' is True. Client may request a timestamp and root - # file without knowing its digest and file size. - if rolename == 'root' or rolename == 'timestamp': - write_metadata_file(signable, metadata_filename, compressions, - consistent_snapshot=False) - - - # 'signable' contains an invalid threshold of signatures. - else: - message = 'Not enough signatures for '+repr(metadata_filename) - raise tuf.UnsignedMetadataError(message, signable) - - return signable, filename - - - - - -def _log_status_of_top_level_roles(targets_directory, metadata_directory): - """ - Non-public function that logs whether any of the top-level roles contain an - invalid number of public and private keys, or an insufficient threshold of - signatures. Considering that the top-level metadata have to be verified in - the expected root -> targets -> snapshot -> timestamp order, this function - logs the error message and returns as soon as a required metadata file is - found to be invalid. It is assumed here that the delegated roles have been - written and verified. Example output: - - 'root' role contains 1 / 1 signatures. - 'targets' role contains 1 / 1 signatures. - 'snapshot' role contains 1 / 1 signatures. - 'timestamp' role contains 1 / 1 signatures. - - Note: Temporary metadata is generated so that file hashes & sizes may be - computed and verified against the attached signatures. 'metadata_directory' - should be a directory in a temporary repository directory. - """ - - # The expected full filenames of the top-level roles needed to write them to - # disk. - filenames = get_metadata_filenames(metadata_directory) - root_filename = filenames[ROOT_FILENAME] - targets_filename = filenames[TARGETS_FILENAME] - snapshot_filename = filenames[SNAPSHOT_FILENAME] - timestamp_filename = filenames[TIMESTAMP_FILENAME] - - # Verify that the top-level roles contain a valid number of public keys and - # that their corresponding private keys have been loaded. - for rolename in ['root', 'targets', 'snapshot', 'timestamp']: - try: - _check_role_keys(rolename) - - except tuf.InsufficientKeysError, e: - logger.info(str(e)) - return - - # Do the top-level roles contain a valid threshold of signatures? Top-level - # metadata is verified in Root -> Targets -> Snapshot -> Timestamp order. - # Verify the metadata of the Root role. - try: - signable, root_filename = \ - _generate_and_write_metadata('root', root_filename, False, - targets_directory, metadata_directory) - _log_status('root', signable) - - # 'tuf.UnsignedMetadataError' raised if metadata contains an invalid threshold - # of signatures. log the valid/threshold message, where valid < threshold. - except tuf.UnsignedMetadataError, e: - signable = e[1] - _log_status('root', signable) - return - - # Verify the metadata of the Targets role. - try: - signable, targets_filename = \ - _generate_and_write_metadata('targets', targets_filename, False, - targets_directory, metadata_directory) - _log_status('targets', signable) - - except tuf.UnsignedMetadataError, e: - signable = e[1] - _log_status('targets', signable) - return - - # Verify the metadata of the snapshot role. - filenames = {'root': root_filename, 'targets': targets_filename} - try: - signable, snapshot_filename = \ - _generate_and_write_metadata('snapshot', snapshot_filename, False, - targets_directory, metadata_directory, - False, filenames) - _log_status('snapshot', signable) - - except tuf.UnsignedMetadataError, e: - signable = e[1] - _log_status('snapshot', signable) - return - - # Verify the metadata of the Timestamp role. - filenames = {'snapshot': snapshot_filename} - try: - signable, snapshot_filename = \ - _generate_and_write_metadata('timestamp', snapshot_filename, False, - targets_directory, metadata_directory, - False, filenames) - _log_status('timestamp', signable) - - except tuf.UnsignedMetadataError, e: - signable = e[1] - _log_status('timestamp', signable) - return - - - - -def _log_status(rolename, signable): - """ - Non-public function logs the number of (good/threshold) signatures of - 'rolename'. - """ - - status = tuf.sig.get_signature_status(signable, rolename) - - message = repr(rolename)+' role contains '+ repr(len(status['good_sigs']))+\ - ' / '+repr(status['threshold'])+' signatures.' - logger.info(message) - - - - - -def _prompt(message, result_type=str): - """ - Non-public function that prompts the user for input by loging 'message', - converting the input to 'result_type', and returning the value to the - caller. - """ - - return result_type(raw_input(message)) - - - - - -def _get_password(prompt='Password: ', confirm=False): - """ - Non-public function that returns the password entered by the user. If - 'confirm' is True, the user is asked to enter the previously entered - password once again. If they match, the password is returned to the caller. - """ - - while True: - # getpass() prompts the user for a password without echoing - # the user input. - password = getpass.getpass(prompt, sys.stderr) - if not confirm: - return password - password2 = getpass.getpass('Confirm: ', sys.stderr) - if password == password2: - return password - else: - print('Mismatch; try again.') - - - - - -def _metadata_is_partially_loaded(rolename, signable, roleinfo): - """ - Non-public function that determines whether 'rolename' is loaded with - at least 1 good signature, but an insufficient threshold (which means - 'rolename' was written to disk with repository.write_partial(). If 'rolename' - is found to be partially loaded, mark it as partially loaded in its - 'tuf.roledb' roleinfo. This function exists to assist in deciding whether - a role's version number should be incremented when write() or write_parital() - is called. Return True if 'rolename' was partially loaded, False otherwise. - """ - - # The signature status lists the number of good signatures, including - # bad, untrusted, unknown, etc. - status = tuf.sig.get_signature_status(signable, rolename) - - if len(status['good_sigs']) < status['threshold'] and \ - len(status['good_sigs']) >= 1: - return True - - else: - return False - - - - - -def _check_directory(directory): - """ - - Non-public function that ensures 'directory' is valid and it exists. This - is not a security check, but a way for the caller to determine the cause of - an invalid directory provided by the user. If the directory argument is - valid, it is returned normalized and as an absolute path. - - - directory: - The directory to check. - - - tuf.Error, if 'directory' could not be validated. - - tuf.FormatError, if 'directory' is not properly formatted. - - - None. - - - The normalized absolutized path of 'directory'. - """ - - # Does 'directory' have the correct format? - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(directory) - - # Check if the directory exists. - if not os.path.isdir(directory): - raise tuf.Error(repr(directory)+' directory does not exist.') - - directory = os.path.abspath(directory) - - return directory - - - - - -def _check_role_keys(rolename): - """ - Non-public function that verifies the public and signing keys of 'rolename'. - If either contain an invalid threshold of keys, raise an exception. - 'rolename' is the full rolename (e.g., 'targets/unclaimed/django'). - """ - - # Extract the total number of public and private keys of 'rolename' from its - # roleinfo in 'tuf.roledb'. - roleinfo = tuf.roledb.get_roleinfo(rolename) - total_keyids = len(roleinfo['keyids']) - threshold = roleinfo['threshold'] - total_signatures = len(roleinfo['signatures']) - total_signing_keys = len(roleinfo['signing_keyids']) - - # Raise an exception for an invalid threshold of public keys. - if total_keyids < threshold: - message = repr(rolename)+' role contains '+repr(total_keyids)+' / '+ \ - repr(threshold)+' public keys.' - raise tuf.InsufficientKeysError(message) - - # Raise an exception for an invalid threshold of signing keys. - if total_signatures == 0 and total_signing_keys < threshold: - message = repr(rolename)+' role contains '+repr(total_signing_keys)+' / '+ \ - repr(threshold)+' signing keys.' - raise tuf.InsufficientKeysError(message) - - - - - -def _remove_invalid_and_duplicate_signatures(signable): - """ - Non-public function that removes invalid signatures from 'signable'. - 'signable' may contain signatures (invalid) from previous versions - of the metadata that were loaded with load_repository(). Invalid, or - duplicate signatures are removed from 'signable'. - """ - - # Store the keyids of valid signatures. 'signature_keyids' is checked - # for duplicates rather than comparing signature objects because PSS may - # generate duplicate valid signatures of the same data, yet contain different - # signatures. - signature_keyids = [] - - for signature in signable['signatures']: - signed = signable['signed'] - keyid = signature['keyid'] - key = None - - # Remove 'signature' from 'signable' if the listed keyid does not exist - # in 'tuf.keydb'. - try: - key = tuf.keydb.get_key(keyid) - - except tuf.UnknownKeyError, e: - signable['signatures'].remove(signature) - - # Remove 'signature' from 'signable' if it is an invalid signature. - if not tuf.keys.verify_signature(key, signature, signed): - signable['signatures'].remove(signature) - - # Although valid, it may still need removal if it is a duplicate. Check - # the keyid, rather than the signature, to remove duplicate PSS signatures. - # PSS may generate multiple different signatures for the same keyid. - else: - if keyid in signature_keyids: - signable['signatures'].remove(signature) - - # 'keyid' is valid and not a duplicate, so add it to 'signature_keyids'. - else: - signature_keyids.append(keyid) - - - - - -def _delete_obsolete_metadata(metadata_directory, snapshot_metadata, - consistent_snapshot): - """ - Non-public function that deletes metadata files marked as removed by - 'repository_tool.py'. Revoked metadata files are not actually deleted until - this function is called. Obsolete metadata should *not* be retained in - "metadata.staged", otherwise they may be re-loaded by 'load_repository()'. - Note: Obsolete metadata may not always be easily detected (by inspecting - top-level metadata during loading) due to partial metadata and top-level - metadata that have not been written yet. - """ - - # Walk the repository's metadata 'targets' sub-directory, where all the - # metadata of delegated roles is stored. - targets_metadata = os.path.join(metadata_directory, 'targets') - - # The 'targets.json' metadata is not visited, only its child delegations. - # The 'targets/unclaimed/django.json' role would be located in the - # '{repository_directory}/metadata/targets/unclaimed/' directory. - if os.path.exists(targets_metadata) and os.path.isdir(targets_metadata): - for directory_path, junk_directories, files in os.walk(targets_metadata): - - # 'files' here is a list of target file names. - for basename in files: - metadata_path = os.path.join(directory_path, basename) - # Strip the metadata dirname and the leading path separator. - # '{repository_directory}/metadata/targets/unclaimed/django.json' --> - # 'targets/unclaimed/django.json' - metadata_name = \ - metadata_path[len(metadata_directory):].lstrip(os.path.sep) - - # Strip the digest if 'consistent_snapshot' is True. - # Example: 'targets/unclaimed/13df98ab0.django.json' --> - # 'targets/unclaimed/django.json'. Consistent and non-consistent - # metadata might co-exist if write() and write(consistent_snapshot=True) - # are mixed, so ensure only 'digest.filename' metadata is stripped. - embeded_digest = None - if metadata_name not in snapshot_metadata['meta']: - metadata_name, embeded_digest = \ - _strip_consistent_snapshot_digest(metadata_name, consistent_snapshot) - - # Strip filename extensions. The role database does not include the - # metadata extension. - metadata_name_extension = metadata_name - for metadata_extension in METADATA_EXTENSIONS: - if metadata_name.endswith(metadata_extension): - metadata_name = metadata_name[:-len(metadata_extension)] - - # Delete the metadata file if it does not exist in 'tuf.roledb'. - # 'repository_tool.py' might have marked 'metadata_name' as removed, but - # its metadata file is not actually deleted yet. Do it now. - if not tuf.roledb.role_exists(metadata_name): - logger.info('Removing outdated metadata: ' + repr(metadata_path)) - os.remove(metadata_path) - - # Delete outdated consistent snapshots. snapshot metadata includes - # the file extension of roles. - if consistent_snapshot and embeded_digest is not None: - file_hashes = snapshot_metadata['meta'][metadata_name_extension] \ - ['hashes'].values() - if embeded_digest not in file_hashes: - logger.info('Removing outdated metadata: ' + repr(metadata_path)) - os.remove(metadata_path) - - - - - -def _get_written_metadata_and_digests(metadata_signable): - """ - Non-public function that returns the actual content of written metadata and - its digest. - """ - - written_metadata_content = unicode(json.dumps(metadata_signable, indent=1, - sort_keys=True)) - written_metadata_digests = {} - - for hash_algorithm in tuf.conf.REPOSITORY_HASH_ALGORITHMS: - digest_object = tuf.hash.digest(hash_algorithm) - digest_object.update(written_metadata_content) - written_metadata_digests.update({hash_algorithm: digest_object.hexdigest()}) - - return written_metadata_content, written_metadata_digests - - - - - -def _strip_consistent_snapshot_digest(metadata_filename, consistent_snapshot): - """ - Strip from 'metadata_filename' any digest data (in the expected - '{dirname}/digest.filename' format) that it may contain, and return it. - """ - - embeded_digest = '' - - # Strip the digest if 'consistent_snapshot' is True. - # Example: 'targets/unclaimed/13df98ab0.django.json' --> - # 'targets/unclaimed/django.json' - if consistent_snapshot: - dirname, basename = os.path.split(metadata_filename) - embeded_digest = basename[:basename.find('.')] - - # Ensure the digest, including the period, is stripped. - basename = basename[basename.find('.')+1:] - - metadata_filename = os.path.join(dirname, basename) - - - return metadata_filename, embeded_digest - + return list(self._delegated_roles.values()) @@ -3048,7 +2511,7 @@ def create_new_repository(repository_directory): # 'OSError' raised if the leaf directory already exists or cannot be created. # Check for case where 'repository_directory' has already been created. - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass else: @@ -3071,7 +2534,7 @@ def create_new_repository(repository_directory): os.mkdir(metadata_directory) # 'OSError' raised if the leaf directory already exists or cannot be created. - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass else: @@ -3083,7 +2546,7 @@ def create_new_repository(repository_directory): logger.info(message) os.mkdir(targets_directory) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass else: @@ -3099,6 +2562,8 @@ def create_new_repository(repository_directory): + + def load_repository(repository_directory): """ @@ -3139,7 +2604,7 @@ def load_repository(repository_directory): repository = Repository(repository_directory, metadata_directory, targets_directory) - filenames = get_metadata_filenames(metadata_directory) + filenames = repo_lib.get_metadata_filenames(metadata_directory) # The Root file is always available without a consistent snapshots digest # attached to the filename. Store the 'consistent_snapshot' value read the @@ -3149,8 +2614,8 @@ def load_repository(repository_directory): # Load the metadata of the top-level roles (i.e., Root, Timestamp, Targets, # and Snapshot). - repository, consistent_snapshot = _load_top_level_metadata(repository, - filenames) + repository, consistent_snapshot = repo_lib._load_top_level_metadata(repository, + filenames) # Load delegated targets metadata. # Walk the 'targets/' directory and generate the fileinfo of all the files @@ -3175,7 +2640,8 @@ def load_repository(repository_directory): # Example: 'targets/unclaimed/13df98ab0.django.json' --> # 'targets/unclaimed/django.json' metadata_name, digest_junk = \ - _strip_consistent_snapshot_digest(metadata_name, consistent_snapshot) + repo_lib._strip_consistent_snapshot_digest(metadata_name, + consistent_snapshot) if metadata_name.endswith(METADATA_EXTENSION): extension_length = len(METADATA_EXTENSION) @@ -3195,7 +2661,7 @@ def load_repository(repository_directory): try: signable = tuf.util.load_json_file(metadata_path) - except (ValueError, IOError), e: + except (ValueError, IOError) as e: continue metadata_object = signable['signed'] @@ -3206,15 +2672,15 @@ def load_repository(repository_directory): roleinfo['signatures'].extend(signable['signatures']) roleinfo['version'] = metadata_object['version'] roleinfo['expires'] = metadata_object['expires'] - roleinfo['paths'] = metadata_object['targets'].keys() + roleinfo['paths'] = list(metadata_object['targets'].keys()) roleinfo['delegations'] = metadata_object['delegations'] - if os.path.exists(metadata_path+'.gz'): + if os.path.exists(metadata_path + '.gz'): roleinfo['compressions'].append('gz') # The roleinfo of 'metadata_name' should have been initialized with # defaults when it was loaded from its parent role. - if _metadata_is_partially_loaded(metadata_name, signable, roleinfo): + if repo_lib._metadata_is_partially_loaded(metadata_name, signable, roleinfo): roleinfo['partial_loaded'] = True tuf.roledb.update_roleinfo(metadata_name, roleinfo) @@ -3237,12 +2703,12 @@ def load_repository(repository_directory): # log a warning here as there may be many such duplicate key warnings. # The repository maintainer should have also been made aware of the # duplicate key when it was added. - for key_metadata in metadata_object['delegations']['keys'].values(): + for key_metadata in six.itervalues(metadata_object['delegations']['keys']): key_object = tuf.keys.format_metadata_to_key(key_metadata) try: tuf.keydb.add_key(key_object) - except tuf.KeyAlreadyExistsError, e: + except tuf.KeyAlreadyExistsError as e: pass # Add the delegated role's initial roleinfo, to be fully populated @@ -3264,1601 +2730,6 @@ def load_repository(repository_directory): -def _load_top_level_metadata(repository, top_level_filenames): - """ - Load the metadata of the Root, Timestamp, Targets, and Snapshot roles. - At a minimum, the Root role must exist and successfully load. - """ - - root_filename = top_level_filenames[ROOT_FILENAME] - targets_filename = top_level_filenames[TARGETS_FILENAME] - snapshot_filename = top_level_filenames[SNAPSHOT_FILENAME] - timestamp_filename = top_level_filenames[TIMESTAMP_FILENAME] - - root_metadata = None - targets_metadata = None - snapshot_metadata = None - timestamp_metadata = None - - # Load 'root.json'. A Root role file without a digest is always written. - if os.path.exists(root_filename): - # Initialize the key and role metadata of the top-level roles. - signable = tuf.util.load_json_file(root_filename) - tuf.formats.check_signable_object_format(signable) - root_metadata = signable['signed'] - tuf.keydb.create_keydb_from_root_metadata(root_metadata) - tuf.roledb.create_roledb_from_root_metadata(root_metadata) - - # Load Root's roleinfo and update 'tuf.roledb'. - roleinfo = tuf.roledb.get_roleinfo('root') - roleinfo['signatures'] = [] - for signature in signable['signatures']: - if signature not in roleinfo['signatures']: - roleinfo['signatures'].append(signature) - - if os.path.exists(root_filename+'.gz'): - roleinfo['compressions'].append('gz') - - # By default, roleinfo['partial_loaded'] of top-level roles should be set to - # False in 'create_roledb_from_root_metadata()'. Update this field, if - # necessary, now that we have its signable object. - if _metadata_is_partially_loaded('root', signable, roleinfo): - roleinfo['partial_loaded'] = True - - _log_warning_if_expires_soon(ROOT_FILENAME, roleinfo['expires'], - ROOT_EXPIRES_WARN_SECONDS) - - tuf.roledb.update_roleinfo('root', roleinfo) - - # Ensure the 'consistent_snapshot' field is extracted. - consistent_snapshot = root_metadata['consistent_snapshot'] - - else: - message = 'Cannot load the required root file: '+repr(root_filename) - raise tuf.RepositoryError(message) - - # Load 'timestamp.json'. A Timestamp role file without a digest is always - # written. - if os.path.exists(timestamp_filename): - signable = tuf.util.load_json_file(timestamp_filename) - timestamp_metadata = signable['signed'] - for signature in signable['signatures']: - repository.timestamp.add_signature(signature) - - # Load Timestamp's roleinfo and update 'tuf.roledb'. - roleinfo = tuf.roledb.get_roleinfo('timestamp') - roleinfo['expires'] = timestamp_metadata['expires'] - roleinfo['version'] = timestamp_metadata['version'] - if os.path.exists(timestamp_filename+'.gz'): - roleinfo['compressions'].append('gz') - - if _metadata_is_partially_loaded('timestamp', signable, roleinfo): - roleinfo['partial_loaded'] = True - - _log_warning_if_expires_soon(TIMESTAMP_FILENAME, roleinfo['expires'], - TIMESTAMP_EXPIRES_WARN_SECONDS) - - tuf.roledb.update_roleinfo('timestamp', roleinfo) - - else: - pass - - # Load 'snapshot.json'. A consistent snapshot of Snapshot must be calculated - # if 'consistent_snapshot' is True. - if consistent_snapshot: - snapshot_hashes = timestamp_metadata['meta'][SNAPSHOT_FILENAME]['hashes'] - snapshot_digest = random.choice(snapshot_hashes.values()) - dirname, basename = os.path.split(snapshot_filename) - snapshot_filename = os.path.join(dirname, snapshot_digest + '.' + basename) - - if os.path.exists(snapshot_filename): - signable = tuf.util.load_json_file(snapshot_filename) - tuf.formats.check_signable_object_format(signable) - snapshot_metadata = signable['signed'] - for signature in signable['signatures']: - repository.snapshot.add_signature(signature) - - # Load Snapshot's roleinfo and update 'tuf.roledb'. - roleinfo = tuf.roledb.get_roleinfo('snapshot') - roleinfo['expires'] = snapshot_metadata['expires'] - roleinfo['version'] = snapshot_metadata['version'] - if os.path.exists(snapshot_filename+'.gz'): - roleinfo['compressions'].append('gz') - - if _metadata_is_partially_loaded('snapshot', signable, roleinfo): - roleinfo['partial_loaded'] = True - - _log_warning_if_expires_soon(SNAPSHOT_FILENAME, roleinfo['expires'], - SNAPSHOT_EXPIRES_WARN_SECONDS) - - tuf.roledb.update_roleinfo('snapshot', roleinfo) - - else: - pass - - # Load 'targets.json'. A consistent snapshot of Targets must be calculated if - # 'consistent_snapshot' is True. - if consistent_snapshot: - targets_hashes = snapshot_metadata['meta'][TARGETS_FILENAME]['hashes'] - targets_digest = random.choice(targets_hashes.values()) - dirname, basename = os.path.split(targets_filename) - targets_filename = os.path.join(dirname, targets_digest + '.' + basename) - - if os.path.exists(targets_filename): - signable = tuf.util.load_json_file(targets_filename) - tuf.formats.check_signable_object_format(signable) - targets_metadata = signable['signed'] - - for signature in signable['signatures']: - repository.targets.add_signature(signature) - - # Update 'targets.json' in 'tuf.roledb.py' - roleinfo = tuf.roledb.get_roleinfo('targets') - roleinfo['paths'] = targets_metadata['targets'].keys() - roleinfo['version'] = targets_metadata['version'] - roleinfo['expires'] = targets_metadata['expires'] - roleinfo['delegations'] = targets_metadata['delegations'] - if os.path.exists(targets_filename+'.gz'): - roleinfo['compressions'].append('gz') - - if _metadata_is_partially_loaded('targets', signable, roleinfo): - roleinfo['partial_loaded'] = True - - _log_warning_if_expires_soon(TARGETS_FILENAME, roleinfo['expires'], - TARGETS_EXPIRES_WARN_SECONDS) - - tuf.roledb.update_roleinfo('targets', roleinfo) - - # Add the keys specified in the delegations field of the Targets role. - for key_metadata in targets_metadata['delegations']['keys'].values(): - key_object = tuf.keys.format_metadata_to_key(key_metadata) - - # Add 'key_object' to the list of recognized keys. Keys may be shared, - # so do not raise an exception if 'key_object' has already been loaded. - # In contrast to the methods that may add duplicate keys, do not log - # a warning as there may be many such duplicate key warnings. The - # repository maintainer should have also been made aware of the duplicate - # key when it was added. - try: - tuf.keydb.add_key(key_object) - - except tuf.KeyAlreadyExistsError, e: - pass - - for role in targets_metadata['delegations']['roles']: - rolename = role['name'] - roleinfo = {'name': role['name'], 'keyids': role['keyids'], - 'threshold': role['threshold'], 'compressions': [''], - 'signing_keyids': [], 'partial_loaded': False, - 'signatures': [], 'delegations': {'keys': {}, - 'roles': []}} - tuf.roledb.add_role(rolename, roleinfo) - - else: - pass - - return repository, consistent_snapshot - - - - -def _log_warning_if_expires_soon(rolename, expires_iso8601_timestamp, - seconds_remaining_to_warn): - """ - Non-public function that logs a warning if 'rolename' expires in - 'seconds_remaining_to_warn' seconds, or less. - """ - - # Metadata stores expiration datetimes in ISO8601 format. Convert to - # unix timestamp, subtract from from current time.time() (also in POSIX time) - # and compare against 'seconds_remaining_to_warn'. Log a warning message - # to console if 'rolename' expires soon. - datetime_object = iso8601.parse_date(expires_iso8601_timestamp) - expires_unix_timestamp = \ - tuf.formats.datetime_to_unix_timestamp(datetime_object) - seconds_until_expires = expires_unix_timestamp - int(time.time()) - - if seconds_until_expires <= seconds_remaining_to_warn: - days_until_expires = seconds_until_expires / 86400 - - message = repr(rolename) + ' expires ' + datetime_object.ctime() + \ - ' (UTC).\n' + repr(days_until_expires) + ' day(s) until it expires.' - - logger.warn(message) - - - - - -def generate_and_write_rsa_keypair(filepath, bits=DEFAULT_RSA_KEY_BITS, - password=None): - """ - - Generate an RSA key file, create an encrypted PEM string (using 'password' - as the pass phrase), and store it in 'filepath'. The public key portion of - the generated RSA key is stored in <'filepath'>.pub. Which cryptography - library performs the cryptographic decryption is determined by the string - set in 'tuf.conf.RSA_CRYPTO_LIBRARY'. PyCrypto currently supported. The - PEM private key is encrypted with 3DES and CBC the mode of operation. The - password is strengthened with PBKDF1-MD5. - - - filepath: - The public and private key files are saved to .pub, , - respectively. - - bits: - The number of bits of the generated RSA key. - - password: - The password used to encrypt 'filepath'. - - - tuf.FormatError, if the arguments are improperly formatted. - - - Writes key files to '' and '.pub'. - - - None. - """ - - # Do the arguments have the correct format? - # This check ensures arguments have the appropriate number of - # objects and object types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(filepath) - - # Does 'bits' have the correct format? - tuf.formats.RSAKEYBITS_SCHEMA.check_match(bits) - - # If the caller does not provide a password argument, prompt for one. - if password is None: - message = 'Enter a password for the RSA key file: ' - password = _get_password(message, confirm=True) - - # Does 'password' have the correct format? - tuf.formats.PASSWORD_SCHEMA.check_match(password) - - # Generate public and private RSA keys, encrypted the private portion - # and store them in PEM format. - rsa_key = tuf.keys.generate_rsa_key(bits) - public = rsa_key['keyval']['public'] - private = rsa_key['keyval']['private'] - encrypted_pem = tuf.keys.create_rsa_encrypted_pem(private, password) - - # Write public key (i.e., 'public', which is in PEM format) to - # '.pub'. If the parent directory of filepath does not exist, - # create it (and all its parent directories, if necessary). - tuf.util.ensure_parent_dir(filepath) - - # Create a tempororary file, write the contents of the public key, and move - # to final destination. - file_object = tuf.util.TempFile() - file_object.write(public) - - # The temporary file is closed after the final move. - file_object.move(filepath+'.pub') - - # Write the private key in encrypted PEM format to ''. - # Unlike the public key file, the private key does not have a file - # extension. - file_object = tuf.util.TempFile() - file_object.write(encrypted_pem) - file_object.move(filepath) - - - - - -def import_rsa_privatekey_from_file(filepath, password=None): - """ - - Import the encrypted PEM file in 'filepath', decrypt it, and return the key - object in 'tuf.formats.RSAKEY_SCHEMA' format. - - Which cryptography library performs the cryptographic decryption is - determined by the string set in 'tuf.conf.RSA_CRYPTO_LIBRARY'. PyCrypto - currently supported. - - The PEM private key is encrypted with 3DES and CBC the mode of operation. - The password is strengthened with PBKDF1-MD5. - - - filepath: - file, an RSA encrypted PEM file. Unlike the public RSA PEM - key file, 'filepath' does not have an extension. - - password: - The passphrase to decrypt 'filepath'. - - - tuf.FormatError, if the arguments are improperly formatted. - - tuf.CryptoError, if 'filepath' is not a valid encrypted key file. - - - The contents of 'filepath' is read, decrypted, and the key stored. - - - An RSA key object, conformant to 'tuf.formats.RSAKEY_SCHEMA'. - """ - - # Does 'filepath' have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(filepath) - - # If the caller does not provide a password argument, prompt for one. - # Password confirmation disabled here, which should ideally happen only - # when creating encrypted key files (i.e., improve usability). - if password is None: - message = 'Enter a password for the encrypted RSA file: ' - password = _get_password(message, confirm=False) - - # Does 'password' have the correct format? - tuf.formats.PASSWORD_SCHEMA.check_match(password) - - encrypted_pem = None - - # Read the contents of 'filepath' that should be an encrypted PEM. - with open(filepath, 'rb') as file_object: - encrypted_pem = file_object.read() - - # Convert 'encrypted_pem' to 'tuf.formats.RSAKEY_SCHEMA' format. Raise - # 'tuf.CryptoError' if 'encrypted_pem' is invalid. - rsa_key = tuf.keys.import_rsakey_from_encrypted_pem(encrypted_pem, password) - - return rsa_key - - - - - -def import_rsa_publickey_from_file(filepath): - """ - - Import the RSA key stored in 'filepath'. The key object returned is a TUF - key, specifically 'tuf.formats.RSAKEY_SCHEMA'. If the RSA PEM in 'filepath' - contains a private key, it is discarded. - - Which cryptography library performs the cryptographic decryption is - determined by the string set in 'tuf.conf.RSA_CRYPTO_LIBRARY'. PyCrypto - currently supported. If the RSA PEM in 'filepath' contains a private key, - it is discarded. - - - filepath: - .pub file, an RSA PEM file. - - - tuf.FormatError, if 'filepath' is improperly formatted. - - tuf.Error, if a valid RSA key object cannot be generated. This may be - caused by an improperly formatted PEM file. - - - 'filepath' is read and its contents extracted. - - - An RSA key object conformant to 'tuf.formats.RSAKEY_SCHEMA'. - """ - - # Does 'filepath' have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(filepath) - - # Read the contents of the key file that should be in PEM format and contains - # the public portion of the RSA key. - with open(filepath, 'rb') as file_object: - rsa_pubkey_pem = file_object.read() - - # Convert 'rsa_pubkey_pem' to 'tuf.formats.RSAKEY_SCHEMA' format. - try: - rsakey_dict = tuf.keys.format_rsakey_from_pem(rsa_pubkey_pem) - - except tuf.FormatError, e: - raise tuf.Error('Cannot import improperly formatted PEM file.') - - return rsakey_dict - - - - - -def generate_and_write_ed25519_keypair(filepath, password=None): - """ - - Generate an ED25519 key file, create an encrypted TUF key (using 'password' - as the pass phrase), and store it in 'filepath'. The public key portion of - the generated ED25519 key is stored in <'filepath'>.pub. Which cryptography - library performs the cryptographic decryption is determined by the string - set in 'tuf.conf.ED25519_CRYPTO_LIBRARY'. - - PyCrypto currently supported. The ED25519 private key is encrypted with - AES-256 and CTR the mode of operation. The password is strengthened with - PBKDF2-HMAC-SHA256. - - - filepath: - The public and private key files are saved to .pub and - , respectively. - - password: - The password, or passphrase, to encrypt the private portion of the - generated ed25519 key. A symmetric encryption key is derived from - 'password', so it is not directly used. - - - tuf.FormatError, if the arguments are improperly formatted. - - tuf.CryptoError, if 'filepath' cannot be encrypted. - - tuf.UnsupportedLibraryError, if 'filepath' cannot be encrypted due to an - invalid configuration setting (i.e., invalid 'tuf.conf.py' setting). - - - Writes key files to '' and '.pub'. - - - None. - """ - - # Does 'filepath' have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(filepath) - - # If the caller does not provide a password argument, prompt for one. - if password is None: - message = 'Enter a password for the ED25519 key: ' - password = _get_password(message, confirm=True) - - # Does 'password' have the correct format? - tuf.formats.PASSWORD_SCHEMA.check_match(password) - - # Generate a new ED25519 key object and encrypt it. The cryptography library - # used is determined by the user, or by default (set in - # 'tuf.conf.ED25519_CRYPTO_LIBRARY'). Raise 'tuf.CryptoError' or - # 'tuf.UnsupportedLibraryError', if 'ed25519_key' cannot be encrypted. - ed25519_key = tuf.keys.generate_ed25519_key() - encrypted_key = tuf.keys.encrypt_key(ed25519_key, password) - - # ed25519 public key file contents in metadata format (i.e., does not include - # the keyid portion). - keytype = ed25519_key['keytype'] - keyval = ed25519_key['keyval'] - ed25519key_metadata_format = \ - tuf.keys.format_keyval_to_metadata(keytype, keyval, private=False) - - # Write the public key, conformant to 'tuf.formats.KEY_SCHEMA', to - # '.pub'. - tuf.util.ensure_parent_dir(filepath) - - # Create a tempororary file, write the contents of the public key, and move - # to final destination. - file_object = tuf.util.TempFile() - file_object.write(json.dumps(ed25519key_metadata_format)) - - # The temporary file is closed after the final move. - file_object.move(filepath+'.pub') - - # Write the encrypted key string, conformant to - # 'tuf.formats.ENCRYPTEDKEY_SCHEMA', to ''. - file_object = tuf.util.TempFile() - file_object.write(encrypted_key) - file_object.move(filepath) - - - - - -def import_ed25519_publickey_from_file(filepath): - """ - - Load the ED25519 public key object (conformant to 'tuf.formats.KEY_SCHEMA') - stored in 'filepath'. Return 'filepath' in tuf.formats.ED25519KEY_SCHEMA - format. - - If the TUF key object in 'filepath' contains a private key, it is discarded. - - - filepath: - .pub file, a TUF public key file. - - - tuf.FormatError, if 'filepath' is improperly formatted or is an unexpected - key type. - - - The contents of 'filepath' is read and saved. - - - An ED25519 key object conformant to 'tuf.formats.ED25519KEY_SCHEMA'. - """ - - # Does 'filepath' have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(filepath) - - # ED25519 key objects are saved in json and metadata format. Return the - # loaded key object in tuf.formats.ED25519KEY_SCHEMA' format that also - # includes the keyid. - ed25519_key_metadata = tuf.util.load_json_file(filepath) - ed25519_key = tuf.keys.format_metadata_to_key(ed25519_key_metadata) - - # Raise an exception if an unexpected key type is imported. - if ed25519_key['keytype'] != 'ed25519': - message = 'Invalid key type loaded: '+repr(ed25519_key['keytype']) - raise tuf.FormatError(message) - - return ed25519_key - - - - - -def import_ed25519_privatekey_from_file(filepath, password=None): - """ - - Import the encrypted ed25519 TUF key file in 'filepath', decrypt it, and - return the key object in 'tuf.formats.ED25519KEY_SCHEMA' format. - - Which cryptography library performs the cryptographic decryption is - determined by the string set in 'tuf.conf.ED25519_CRYPTO_LIBRARY'. PyCrypto - currently supported. - - The TUF private key (may also contain the public part) is encrypted with AES - 256 and CTR the mode of operation. The password is strengthened with - PBKDF2-HMAC-SHA256. - - - filepath: - file, an RSA encrypted TUF key file. - - password: - The password, or passphrase, to import the private key (i.e., the - encrypted key file 'filepath' must be decrypted before the ed25519 key - object can be returned. - - - tuf.FormatError, if the arguments are improperly formatted or the imported - key object contains an invalid key type (i.e., not 'ed25519'). - - tuf.CryptoError, if 'filepath' cannot be decrypted. - - tuf.UnsupportedLibraryError, if 'filepath' cannot be decrypted due to an - invalid configuration setting (i.e., invalid 'tuf.conf.py' setting). - - - 'password' is used to decrypt the 'filepath' key file. - - - An ed25519 key object of the form: 'tuf.formats.ED25519KEY_SCHEMA'. - """ - - # Does 'filepath' have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(filepath) - - # If the caller does not provide a password argument, prompt for one. - # Password confirmation disabled here, which should ideally happen only - # when creating encrypted key files (i.e., improve usability). - if password is None: - message = 'Enter a password for the encrypted ED25519 key: ' - password = _get_password(message, confirm=False) - - # Does 'password' have the correct format? - tuf.formats.PASSWORD_SCHEMA.check_match(password) - - # Store the encrypted contents of 'filepath' prior to calling the decryption - # routine. - encrypted_key = None - - with open(filepath, 'rb') as file_object: - encrypted_key = file_object.read() - - # Decrypt the loaded key file, calling the appropriate cryptography library - # (i.e., set by the user) and generating the derived encryption key from - # 'password'. Raise 'tuf.CryptoError' or 'tuf.UnsupportedLibraryError' if the - # decryption fails. - key_object = tuf.keys.decrypt_key(encrypted_key, password) - - # Raise an exception if an unexpected key type is imported. - if key_object['keytype'] != 'ed25519': - message = 'Invalid key type loaded: '+repr(key_object['keytype']) - raise tuf.FormatError(message) - - return key_object - - - - - -def get_metadata_filenames(metadata_directory=None): - """ - - Return a dictionary containing the filenames of the top-level roles. - If 'metadata_directory' is set to 'metadata', the dictionary - returned would contain: - - filenames = {'root.json': 'metadata/root.json', - 'targets.json': 'metadata/targets.json', - 'snapshot.json': 'metadata/snapshot.json', - 'timestamp.json': 'metadata/timestamp.json'} - - If 'metadata_directory' is not set by the caller, the current directory is - used. - - - metadata_directory: - The directory containing the metadata files. - - - tuf.FormatError, if 'metadata_directory' is improperly formatted. - - - None. - - - A dictionary containing the expected filenames of the top-level - metadata files, such as 'root.json' and 'snapshot.json'. - """ - - if metadata_directory is None: - metadata_directory = os.getcwd() - - # Does 'metadata_directory' have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(metadata_directory) - - # Store the filepaths of the top-level roles, including the - # 'metadata_directory' for each one. - filenames = {} - - filenames[ROOT_FILENAME] = \ - os.path.join(metadata_directory, ROOT_FILENAME) - - filenames[TARGETS_FILENAME] = \ - os.path.join(metadata_directory, TARGETS_FILENAME) - - filenames[SNAPSHOT_FILENAME] = \ - os.path.join(metadata_directory, SNAPSHOT_FILENAME) - - filenames[TIMESTAMP_FILENAME] = \ - os.path.join(metadata_directory, TIMESTAMP_FILENAME) - - return filenames - - - - - -def get_metadata_fileinfo(filename): - """ - - Retrieve the file information of 'filename'. The object returned - conforms to 'tuf.formats.FILEINFO_SCHEMA'. The information - generated for 'filename' is stored in metadata files like 'targets.json'. - The fileinfo object returned has the form: - - fileinfo = {'length': 1024, - 'hashes': {'sha256': 1233dfba312, ...}, - 'custom': {...}} - - - filename: - The metadata file whose file information is needed. It must exist. - - - tuf.FormatError, if 'filename' is improperly formatted. - - tuf.Error, if 'filename' doesn't exist. - - - The file is opened and information about the file is generated, - such as file size and its hash. - - - A dictionary conformant to 'tuf.formats.FILEINFO_SCHEMA'. This - dictionary contains the length, hashes, and custom data about the - 'filename' metadata file. SHA256 hashes are generated by default. - """ - - # Does 'filename' have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(filename) - - if not os.path.isfile(filename): - message = repr(filename)+' is not a file.' - raise tuf.Error(message) - - # Note: 'filehashes' is a dictionary of the form - # {'sha256': 1233dfba312, ...}. 'custom' is an optional - # dictionary that a client might define to include additional - # file information, such as the file's author, version/revision - # numbers, etc. - filesize, filehashes = \ - tuf.util.get_file_details(filename, tuf.conf.REPOSITORY_HASH_ALGORITHMS) - custom = None - - return tuf.formats.make_fileinfo(filesize, filehashes, custom) - - - - - - -def get_target_hash(target_filepath): - """ - - Compute the hash of 'target_filepath'. This is useful in conjunction with - the "path_hash_prefixes" attribute in a delegated targets role, which - tells us which paths it is implicitly responsible for. - - The repository may optionally organize targets into hashed bins to ease - target delegations and role metadata management. The use of consistent - hashing allows for a uniform distribution of targets into bins. - - - target_filepath: - The path to the target file on the repository. This will be relative to - the 'targets' (or equivalent) directory on a given mirror. - - - None. - - - None. - - - The hash of 'target_filepath'. - """ - - return tuf.util.get_target_hash(target_filepath) - - - - - -def generate_root_metadata(version, expiration_date, consistent_snapshot): - """ - - Create the root metadata. 'tuf.roledb.py' and 'tuf.keydb.py' are read and - the information returned by these modules is used to generate the root - metadata object. - - - version: - The metadata version number. Clients use the version number to - determine if the downloaded version is newer than the one currently - trusted. - - expiration_date: - The expiration date of the metadata file. Conformant to - 'tuf.formats.ISO8601_DATETIME_SCHEMA'. - - consistent_snapshot: - Boolean. If True, a file digest is expected to be prepended to the - filename of any target file located in the targets directory. Each digest - is stripped from the target filename and listed in the snapshot metadata. - - - tuf.FormatError, if the generated root metadata object could not - be generated with the correct format. - - tuf.Error, if an error is encountered while generating the root - metadata object (e.g., a required top-level role not found in 'tuf.roledb'.) - - - The contents of 'tuf.keydb.py' and 'tuf.roledb.py' are read. - - - A root metadata object, conformant to 'tuf.formats.ROOT_SCHEMA'. - """ - - # Do the arguments have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if any of the arguments are improperly formatted. - tuf.formats.METADATAVERSION_SCHEMA.check_match(version) - tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) - tuf.formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot) - - # The role and key dictionaries to be saved in the root metadata object. - # Conformant to 'ROLEDICT_SCHEMA' and 'KEYDICT_SCHEMA', respectively. - roledict = {} - keydict = {} - - # Extract the role, threshold, and keyid information of the top-level roles, - # which Root stores in its metadata. The necessary role metadata is generated - # from this information. - for rolename in ['root', 'targets', 'snapshot', 'timestamp']: - - # If a top-level role is missing from 'tuf.roledb.py', raise an exception. - if not tuf.roledb.role_exists(rolename): - raise tuf.Error(repr(rolename)+' not in "tuf.roledb".') - - # Keep track of the keys loaded to avoid duplicates. - keyids = [] - - # Generate keys for the keyids listed by the role being processed. - for keyid in tuf.roledb.get_role_keyids(rolename): - key = tuf.keydb.get_key(keyid) - - # If 'key' is an RSA key, it would conform to 'tuf.formats.RSAKEY_SCHEMA', - # and have the form: - # {'keytype': 'rsa', - # 'keyid': keyid, - # 'keyval': {'public': '-----BEGIN RSA PUBLIC KEY----- ...', - # 'private': '-----BEGIN RSA PRIVATE KEY----- ...'}} - keyid = key['keyid'] - if keyid not in keydict: - - # This appears to be a new keyid. Generate the key for it. - if key['keytype'] in ['rsa', 'ed25519']: - keytype = key['keytype'] - keyval = key['keyval'] - keydict[keyid] = \ - tuf.keys.format_keyval_to_metadata(keytype, keyval, private=False) - - # This is not a recognized key. Raise an exception. - else: - raise tuf.Error('Unsupported keytype: '+keyid) - - # Do we have a duplicate? - if keyid in keyids: - raise tuf.Error('Same keyid listed twice: '+keyid) - - # Add the loaded keyid for the role being processed. - keyids.append(keyid) - - # Generate and store the role data belonging to the processed role. - role_threshold = tuf.roledb.get_role_threshold(rolename) - role_metadata = tuf.formats.make_role_metadata(keyids, role_threshold) - roledict[rolename] = role_metadata - - # Generate the root metadata object. - root_metadata = tuf.formats.RootFile.make_metadata(version, expiration_date, - keydict, roledict, - consistent_snapshot) - - return root_metadata - - - - - -def generate_targets_metadata(targets_directory, target_files, version, - expiration_date, delegations=None, - write_consistent_targets=False): - """ - - Generate the targets metadata object. The targets in 'target_files' must - exist at the same path they should on the repo. 'target_files' is a list of - targets. The 'custom' field of the targets metadata is not currently - supported. - - - targets_directory: - The directory containing the target files and directories of the - repository. - - target_files: - The target files tracked by 'targets.json'. 'target_files' is a list of - target paths that are relative to the targets directory (e.g., - ['file1.txt', 'Django/module.py']). - - version: - The metadata version number. Clients use the version number to - determine if the downloaded version is newer than the one currently - trusted. - - expiration_date: - The expiration date of the metadata file. Conformant to - 'tuf.formats.ISO8601_DATETIME_SCHEMA'. - - delegations: - The delegations made by the targets role to be generated. 'delegations' - must match 'tuf.formats.DELEGATIONS_SCHEMA'. - - write_consistent_targets: - Boolean that indicates whether file digests should be prepended to the - target files. - - - tuf.FormatError, if an error occurred trying to generate the targets - metadata object. - - tuf.Error, if any of the target files cannot be read. - - - The target files are read and file information generated about them. - - - A targets metadata object, conformant to 'tuf.formats.TARGETS_SCHEMA'. - """ - - # Do the arguments have the correct format? - # Ensure the arguments have the appropriate number of objects and object - # types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if there is a mismatch. - tuf.formats.PATH_SCHEMA.check_match(targets_directory) - tuf.formats.PATHS_SCHEMA.check_match(target_files) - tuf.formats.METADATAVERSION_SCHEMA.check_match(version) - tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) - tuf.formats.BOOLEAN_SCHEMA.check_match(write_consistent_targets) - - if delegations is not None: - tuf.formats.DELEGATIONS_SCHEMA.check_match(delegations) - - # Store the file attributes of targets in 'target_files'. 'filedict', - # conformant to 'tuf.formats.FILEDICT_SCHEMA', is added to the targets - # metadata object returned. - filedict = {} - - # Ensure the user is aware of a non-existent 'target_directory', and convert - # it to its abosolute path, if it exists. - targets_directory = _check_directory(targets_directory) - - # Generate the fileinfo of all the target files listed in 'target_files'. - for target in target_files: - - # The root-most folder of the targets directory should not be included in - # target paths listed in targets metadata. - # (e.g., 'targets/more_targets/somefile.txt' -> 'more_targets/somefile.txt') - relative_targetpath = target - - # Note: join() discards 'targets_directory' if 'target' contains a leading - # path separator (i.e., is treated as an absolute path). - target_path = os.path.join(targets_directory, target.lstrip(os.sep)) - - # Ensure all target files listed in 'target_files' exist. If just one of - # these files does not exist, raise an exception. - if not os.path.exists(target_path): - message = repr(target_path)+' cannot be read. Unable to generate '+ \ - 'targets metadata.' - raise tuf.Error(message) - - filedict[relative_targetpath] = get_metadata_fileinfo(target_path) - - if write_consistent_targets: - for target_digest in filedict[relative_targetpath]['hashes'].values(): - dirname, basename = os.path.split(target_path) - digest_filename = target_digest + '.' + basename - digest_target = os.path.join(dirname, digest_filename) - - if not os.path.exists(digest_target): - logger.warn('Hard linking target file to ' + repr(digest_target)) - os.link(target_path, digest_target) - - # Generate the targets metadata object. - targets_metadata = tuf.formats.TargetsFile.make_metadata(version, - expiration_date, - filedict, - delegations) - - return targets_metadata - - - - - -def generate_snapshot_metadata(metadata_directory, version, expiration_date, - root_filename, targets_filename, - consistent_snapshot=False): - """ - - Create the snapshot metadata. The minimum metadata must exist - (i.e., 'root.json' and 'targets.json'). This will also look through - the 'targets/' directory in 'metadata_directory' and the resulting - snapshot file will list all the delegated roles. - - - metadata_directory: - The directory containing the 'root.json' and 'targets.json' metadata - files. - - version: - The metadata version number. Clients use the version number to - determine if the downloaded version is newer than the one currently - trusted. - - expiration_date: - The expiration date of the metadata file. - Conformant to 'tuf.formats.ISO8601_DATETIME_SCHEMA'. - - root_filename: - The filename of the top-level root role. The hash and file size of this - file is listed in the snapshot role. - - targets_filename: - The filename of the top-level targets role. The hash and file size of - this file is listed in the snapshot role. - - consistent_snapshot: - Boolean. If True, a file digest is expected to be prepended to the - filename of any target file located in the targets directory. Each digest - is stripped from the target filename and listed in the snapshot metadata. - - - tuf.FormatError, if the arguments are improperly formatted. - - tuf.Error, if an error occurred trying to generate the snapshot metadata - object. - - - The 'root.json' and 'targets.json' files are read. - - - The snapshot metadata object, conformant to 'tuf.formats.SNAPSHOT_SCHEMA'. - """ - - # Do the arguments have the correct format? - # This check ensures arguments have the appropriate number of objects and - # object types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if the check fails. - tuf.formats.PATH_SCHEMA.check_match(metadata_directory) - tuf.formats.METADATAVERSION_SCHEMA.check_match(version) - tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) - tuf.formats.PATH_SCHEMA.check_match(root_filename) - tuf.formats.PATH_SCHEMA.check_match(targets_filename) - tuf.formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot) - - metadata_directory = _check_directory(metadata_directory) - - # Retrieve the fileinfo of 'root.json' and 'targets.json'. This file - # information includes data such as file length, hashes of the file, etc. - filedict = {} - filedict[ROOT_FILENAME] = get_metadata_fileinfo(root_filename) - filedict[TARGETS_FILENAME] = get_metadata_fileinfo(targets_filename) - - # Add compressed versions of the 'targets.json' and 'root.json' metadata, - # if they exist. - for extension in SUPPORTED_COMPRESSION_EXTENSIONS: - compressed_root_filename = root_filename+extension - compressed_targets_filename = targets_filename+extension - - # If the compressed versions of the root and targets metadata is found, - # add their file attributes to 'filedict'. - if os.path.exists(compressed_root_filename): - filedict[ROOT_FILENAME+extension] = \ - get_metadata_fileinfo(compressed_root_filename) - if os.path.exists(compressed_targets_filename): - filedict[TARGETS_FILENAME+extension] = \ - get_metadata_fileinfo(compressed_targets_filename) - - # Walk the 'targets/' directory and generate the fileinfo of all the role - # files found. This information is stored in the 'meta' field of the snapshot - # metadata object. - targets_metadata = os.path.join(metadata_directory, 'targets') - if os.path.exists(targets_metadata) and os.path.isdir(targets_metadata): - for directory_path, junk_directories, files in os.walk(targets_metadata): - - # 'files' here is a list of file names. - for basename in files: - metadata_path = os.path.join(directory_path, basename) - metadata_name = \ - metadata_path[len(metadata_directory):].lstrip(os.path.sep) - - # Strip the digest if 'consistent_snapshot' is True. - # Example: 'targets/unclaimed/13df98ab0.django.json' --> - # 'targets/unclaimed/django.json' - metadata_name, digest_junk = \ - _strip_consistent_snapshot_digest(metadata_name, consistent_snapshot) - - # All delegated roles are added to the snapshot file, including - # compressed versions. - for metadata_extension in METADATA_EXTENSIONS: - if metadata_name.endswith(metadata_extension): - rolename = metadata_name[:-len(metadata_extension)] - - # Obsolete role files may still be found. Ensure only roles loaded - # in the roledb are included in the snapshot metadata. - if tuf.roledb.role_exists(rolename): - filedict[metadata_name] = get_metadata_fileinfo(metadata_path) - - # Generate the snapshot metadata object. - snapshot_metadata = tuf.formats.SnapshotFile.make_metadata(version, - expiration_date, - filedict) - - return snapshot_metadata - - - - - -def generate_timestamp_metadata(snapshot_filename, version, - expiration_date, compressions=()): - """ - - Generate the timestamp metadata object. The 'snapshot.json' file must - exist. - - - snapshot_filename: - The required filename of the snapshot metadata file. The timestamp role - needs to the calculate the file size and hash of this file. - - version: - The timestamp's version number. Clients use the version number to - determine if the downloaded version is newer than the one currently - trusted. - - expiration_date: - The expiration date of the metadata file, conformant to - 'tuf.formats.ISO8601_DATETIME_SCHEMA'. - - compressions: - Compression extensions (e.g., 'gz'). If 'snapshot.json' is also saved in - compressed form, these compression extensions should be stored in - 'compressions' so the compressed timestamp files can be added to the - timestamp metadata object. - - - tuf.FormatError, if the generated timestamp metadata object cannot be - formatted correctly, or one of the arguments is improperly formatted. - - - None. - - - A timestamp metadata object, conformant to 'tuf.formats.TIMESTAMP_SCHEMA'. - """ - - # Do the arguments have the correct format? - # This check ensures arguments have the appropriate number of objects and - # object types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if the check fails. - tuf.formats.PATH_SCHEMA.check_match(snapshot_filename) - tuf.formats.METADATAVERSION_SCHEMA.check_match(version) - tuf.formats.ISO8601_DATETIME_SCHEMA.check_match(expiration_date) - tuf.formats.COMPRESSIONS_SCHEMA.check_match(compressions) - - # Retrieve the fileinfo of the snapshot metadata file. - # This file information contains hashes, file length, custom data, etc. - fileinfo = {} - fileinfo[SNAPSHOT_FILENAME] = get_metadata_fileinfo(snapshot_filename) - - # Save the fileinfo of the compressed versions of 'timestamp.json' - # in 'fileinfo'. Log the files included in 'fileinfo'. - for file_extension in compressions: - if not len(file_extension): - continue - - compressed_filename = snapshot_filename + '.' + file_extension - try: - compressed_fileinfo = get_metadata_fileinfo(compressed_filename) - - except: - logger.warn('Cannot get fileinfo about '+repr(compressed_filename)) - - else: - logger.info('Including fileinfo about '+repr(compressed_filename)) - fileinfo[SNAPSHOT_FILENAME + '.' + file_extension] = compressed_fileinfo - - # Generate the timestamp metadata object. - timestamp_metadata = tuf.formats.TimestampFile.make_metadata(version, - expiration_date, - fileinfo) - - return timestamp_metadata - - - - - -def sign_metadata(metadata_object, keyids, filename): - """ - - Sign a metadata object. If any of the keyids have already signed the file, - the old signature is replaced. The keys in 'keyids' must already be - loaded in 'tuf.keydb'. - - - metadata_object: - The metadata object to sign. For example, 'metadata' might correspond to - 'tuf.formats.ROOT_SCHEMA' or 'tuf.formats.TARGETS_SCHEMA'. - - keyids: - The keyids list of the signing keys. - - filename: - The intended filename of the signed metadata object. - For example, 'root.json' or 'targets.json'. This function - does NOT save the signed metadata to this filename. - - - tuf.FormatError, if a valid 'signable' object could not be generated or - the arguments are improperly formatted. - - tuf.Error, if an invalid keytype was found in the keystore. - - - None. - - - A signable object conformant to 'tuf.formats.SIGNABLE_SCHEMA'. - """ - - # Do the arguments have the correct format? - # This check ensures arguments have the appropriate number of objects and - # object types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if the check fails. - tuf.formats.ANYROLE_SCHEMA.check_match(metadata_object) - tuf.formats.KEYIDS_SCHEMA.check_match(keyids) - tuf.formats.PATH_SCHEMA.check_match(filename) - - # Make sure the metadata is in 'signable' format. That is, - # it contains a 'signatures' field containing the result - # of signing the 'signed' field of 'metadata' with each - # keyid of 'keyids'. - signable = tuf.formats.make_signable(metadata_object) - - # Sign the metadata with each keyid in 'keyids'. - for keyid in keyids: - - # Load the signing key. - key = tuf.keydb.get_key(keyid) - logger.info('Signing '+repr(filename)+' with '+key['keyid']) - - # Create a new signature list. If 'keyid' is encountered, - # do not add it to new list. - signatures = [] - for signature in signable['signatures']: - if not keyid == signature['keyid']: - signatures.append(signature) - signable['signatures'] = signatures - - # Generate the signature using the appropriate signing method. - if key['keytype'] in SUPPORTED_KEY_TYPES: - if len(key['keyval']['private']): - signed = signable['signed'] - signature = tuf.keys.create_signature(key, signed) - signable['signatures'].append(signature) - - else: - logger.warn('Private key unset. Skipping: '+repr(keyid)) - - else: - raise tuf.Error('The keydb contains a key with an invalid key type.') - - # Raise 'tuf.FormatError' if the resulting 'signable' is not formatted - # correctly. - tuf.formats.check_signable_object_format(signable) - - return signable - - - - - -def write_metadata_file(metadata, filename, compressions, consistent_snapshot): - """ - - If necessary, write the 'metadata' signable object to 'filename', and the - compressed version of the metadata file if 'compression' is set. - Note: Compression algorithms like gzip attach a timestamp to compressed - files, so a metadata file compressed multiple times may generate different - digests even though the uncompressed content has not changed. - - - metadata: - The object that will be saved to 'filename', conformant to - 'tuf.formats.SIGNABLE_SCHEMA'. - - filename: - The filename of the metadata to be written (e.g., 'root.json'). - If a compression algorithm is specified in 'compressions', the - compression extention is appended to 'filename'. - - compressions: - Specify the algorithms, as a list of strings, used to compress the file; - The only currently available compression option is 'gz' (gzip). - - consistent_snapshot: - Boolean that determines whether the metadata file's digest should be - prepended to the filename. - - - tuf.FormatError, if the arguments are improperly formatted. - - tuf.Error, if the directory of 'filename' does not exist. - - Any other runtime (e.g., IO) exception. - - - The 'filename' (or the compressed filename) file is created, or overwritten - if it exists. - - - None. - """ - - # Do the arguments have the correct format? - # This check ensures arguments have the appropriate number of objects and - # object types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if the check fails. - tuf.formats.SIGNABLE_SCHEMA.check_match(metadata) - tuf.formats.PATH_SCHEMA.check_match(filename) - tuf.formats.COMPRESSIONS_SCHEMA.check_match(compressions) - tuf.formats.BOOLEAN_SCHEMA.check_match(consistent_snapshot) - - # Verify the directory of 'filename', and convert 'filename' to its absolute - # path so that temporary files are moved to their expected destinations. - filename = os.path.abspath(filename) - written_filename = filename - _check_directory(os.path.dirname(filename)) - consistent_filenames = [] - - # Generate the actual metadata file content of 'metadata'. Metadata is - # saved as json and includes formatting, such as indentation and sorted - # objects. The new digest of 'metadata' is also calculated to help determine - # if re-saving is required. - file_content, new_digests = _get_written_metadata_and_digests(metadata) - - if consistent_snapshot: - for new_digest in new_digests.values(): - dirname, basename = os.path.split(filename) - digest_and_filename = new_digest + '.' + basename - consistent_filenames.append(os.path.join(dirname, digest_and_filename)) - written_filename = consistent_filenames.pop() - - # Verify whether new metadata needs to be written (i.e., has not been - # previously written or has changed. - write_new_metadata = False - - # Has the uncompressed metadata changed? Does it exist? If so, set - # 'write_compressed_version' to True so that it is written. - # compressed metadata should only be written if it does not exist or the - # uncompressed version has changed). - try: - file_length_junk, old_digests = tuf.util.get_file_details(written_filename) - if old_digests != new_digests: - write_new_metadata = True - - # 'tuf.Error' raised if 'filename' does not exist. - except tuf.Error, e: - write_new_metadata = True - - if write_new_metadata: - # The 'metadata' object is written to 'file_object', including compressed - # versions. To avoid partial metadata from being written, 'metadata' is - # first written to a temporary location (i.e., 'file_object') and then moved - # to 'filename'. - file_object = tuf.util.TempFile() - - # Serialize 'metadata' to the file-like object and then write - # 'file_object' to disk. The dictionary keys of 'metadata' are sorted - # and indentation is used. The 'tuf.util.TempFile' file-like object is - # automically closed after the final move. - file_object.write(file_content) - logger.info('Saving ' + repr(written_filename)) - file_object.move(written_filename) - - for consistent_filename in consistent_filenames: - logger.info('Linking ' + repr(consistent_filename)) - os.link(written_filename, consistent_filename) - - - # Generate the compressed versions of 'metadata', if necessary. A compressed - # file may be written (without needing to write the uncompressed version) if - # the repository maintainer adds compression after writing the uncompressed - # version. - for compression in compressions: - file_object = None - - # Ignore the empty string that signifies non-compression. The uncompressed - # file was previously written above, if necessary. - if not len(compression): - continue - - elif compression == 'gz': - file_object = tuf.util.TempFile() - compressed_filename = filename + '.gz' - - # Instantiate a gzip object, but save compressed content to - # 'file_object' (i.e., GzipFile instance is based on its 'fileobj' - # argument). - with gzip.GzipFile(fileobj=file_object, mode='wb') as gzip_object: - gzip_object.write(file_content) - - else: - raise tuf.FormatError('Unknown compression algorithm: '+repr(compression)) - - # Save the compressed version, ensuring an unchanged file is not re-saved. - # Re-saving the same compressed version may cause its digest to unexpectedly - # change (gzip includes a timestamp) even though content has not changed. - _write_compressed_metadata(file_object, compressed_filename, - write_new_metadata, consistent_snapshot) - return written_filename - - - - - -def _write_compressed_metadata(file_object, compressed_filename, - write_new_metadata, consistent_snapshot): - """ - Write compressed versions of metadata, ensuring compressed file that have - not changed are not re-written, the digest of the compressed file is properly - added to the compressed filename, and consistent snapshots are also saved. - Ensure compressed files are written to a temporary location, and then - moved to their destinations. - """ - - # If a consistent snapshot is unneeded, 'file_object' may be simply moved - # 'compressed_filename' if not already written. - if not consistent_snapshot: - if not os.path.exists(compressed_filename) or write_new_metadata: - file_object.move(compressed_filename) - - # The temporary file must be closed if 'file_object.move()' is not used. - # tuf.util.TempFile() automatically closes the temp file when move() is - # called - else: - file_object.close_temp_file() - - # Consistent snapshots = True. Ensure the file's digest is included in the - # compressed filename written, provided it does not already exist. - else: - compressed_content = file_object.read() - new_digests = [] - consistent_filenames = [] - - # Multiple snapshots may be written if the repository uses multiple - # hash algorithms. Generate the digest of the compressed content. - for hash_algorithm in tuf.conf.REPOSITORY_HASH_ALGORITHMS: - digest_object = tuf.hash.digest(hash_algorithm) - digest_object.update(compressed_content) - new_digests.append(digest_object.hexdigest()) - - # Attach each digest to the compressed consistent snapshot filename. - for new_digest in new_digests: - dirname, basename = os.path.split(compressed_filename) - digest_and_filename = new_digest + '.' + basename - consistent_filenames.append(os.path.join(dirname, digest_and_filename)) - - # Move the 'tuf.util.TempFile' object to one of the filenames so that it is - # saved and the temporary file closed. Any remaining consistent snapshots - # may still need to be copied or linked. - compressed_filename = consistent_filenames.pop() - if not os.path.exists(compressed_filename): - logger.info('Saving ' + repr(compressed_filename)) - file_object.move(compressed_filename) - - # Save any remaining compressed consistent snapshots. - for consistent_filename in consistent_filenames: - if not os.path.exists(consistent_filename): - logger.info('Linking ' + repr(consistent_filename)) - os.link(compressed_filename, consistent_filename) - - - - - -def create_tuf_client_directory(repository_directory, client_directory): - """ - - Create a client directory structure that the 'tuf.interposition' package - and 'tuf.client.updater' module expect of clients. Metadata files - downloaded from a remote TUF repository are saved to 'client_directory'. - The Root file must initially exist before an update request can be - satisfied. create_tuf_client_directory() ensures the minimum metadata - is copied and that required directories ('previous' and 'current') are - created in 'client_directory'. Software updaters integrating TUF may - use the client directory created as an initial copy of the repository's - metadadata. - - - repository_directory: - The path of the root repository directory. The 'metadata' and 'targets' - sub-directories should be available in 'repository_directory'. The - metadata files of 'repository_directory' are copied to 'client_directory'. - - client_directory: - The path of the root client directory. The 'current' and 'previous' - sub-directies are created and will store the metadata files copied - from 'repository_directory'. 'client_directory' will store metadata - and target files downloaded from a TUF repository. - - - tuf.FormatError, if the arguments are improperly formatted. - - tuf.RepositoryError, if the metadata directory in 'client_directory' - already exists. - - - Copies metadata files and directories from 'repository_directory' to - 'client_directory'. Parent directories are created if they do not exist. - - - None. - """ - - # Do the arguments have the correct format? - # This check ensures arguments have the appropriate number of objects and - # object types, and that all dict keys are properly named. - # Raise 'tuf.FormatError' if the check fails. - tuf.formats.PATH_SCHEMA.check_match(repository_directory) - tuf.formats.PATH_SCHEMA.check_match(client_directory) - - # Set the absolute path of the Repository's metadata directory. The metadata - # directory should be the one served by the Live repository. At a minimum, - # the repository's root file must be copied. - repository_directory = os.path.abspath(repository_directory) - metadata_directory = os.path.join(repository_directory, - METADATA_DIRECTORY_NAME) - - # Set the client's metadata directory, which will store the metadata copied - # from the repository directory set above. - client_directory = os.path.abspath(client_directory) - client_metadata_directory = os.path.join(client_directory, - METADATA_DIRECTORY_NAME) - - # If the client's metadata directory does not already exist, create it and - # any of its parent directories, otherwise raise an exception. An exception - # is raised to avoid accidently overwritting previous metadata. - try: - os.makedirs(client_metadata_directory) - - except OSError, e: - if e.errno == errno.EEXIST: - message = 'Cannot create a fresh client metadata directory: '+ \ - repr(client_metadata_directory)+'. Already exists.' - raise tuf.RepositoryError(message) - else: - raise - - # Move all metadata to the client's 'current' and 'previous' directories. - # The root metadata file MUST exist in '{client_metadata_directory}/current'. - # 'tuf.interposition' and 'tuf.client.updater.py' expect the 'current' and - # 'previous' directories to exist under 'metadata'. - client_current = os.path.join(client_metadata_directory, 'current') - client_previous = os.path.join(client_metadata_directory, 'previous') - shutil.copytree(metadata_directory, client_current) - shutil.copytree(metadata_directory, client_previous) - - - -def disable_console_log_messages(): - """ - - Disable logger messages printed to the console. For example, repository - maintainers may want to call this function if many roles will be sharing - keys, otherwise detected duplicate keys will continually log a warning - message. - - - None. - - - None. - - - Removes the 'tuf.log' console handler, added by default when - 'tuf.repository_tool.py' is imported. - - - None. - """ - - tuf.log.remove_console_handler() - - if __name__ == '__main__': # The interactive sessions of the documentation strings can # be tested by running repository_tool.py as a standalone module: diff --git a/tuf/roledb.py b/tuf/roledb.py index 835e5bb0..403b5dcb 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -35,12 +35,21 @@ optional. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import logging import copy import tuf import tuf.formats import tuf.log +import tuf._vendor.six as six # See 'tuf.log' to learn how logging is handled in TUF. logger = logging.getLogger('tuf.roledb') @@ -89,7 +98,7 @@ def create_roledb_from_root_metadata(root_metadata): # Iterate through the roles found in 'root_metadata' # and add them to '_roledb_dict'. Duplicates are avoided. - for rolename, roleinfo in root_metadata['roles'].items(): + for rolename, roleinfo in six.iteritems(root_metadata['roles']): if rolename == 'root': roleinfo['version'] = root_metadata['version'] roleinfo['expires'] = root_metadata['expires'] @@ -104,7 +113,7 @@ def create_roledb_from_root_metadata(root_metadata): try: add_role(rolename, roleinfo) # tuf.Error raised if the parent role of 'rolename' does not exist. - except tuf.Error, e: + except tuf.Error as e: logger.error(e) raise @@ -475,7 +484,7 @@ def get_rolenames(): A list of rolenames. """ - return _roledb_dict.keys() + return list(_roledb_dict.keys()) diff --git a/tuf/schema.py b/tuf/schema.py index 278e1899..ee719bbf 100755 --- a/tuf/schema.py +++ b/tuf/schema.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ schema.py @@ -40,12 +42,19 @@ can be found in 'formats.py'. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals import re import sys import tuf - +import tuf._vendor.six as six class Schema: """ @@ -141,7 +150,7 @@ class String(Schema): """ def __init__(self, string): - if not isinstance(string, basestring): + if not isinstance(string, six.string_types): raise tuf.FormatError('Expected a string but got '+repr(string)) self._string = string @@ -188,13 +197,51 @@ def __init__(self): def check_match(self, object): - if not isinstance(object, basestring): + if not isinstance(object, six.string_types): raise tuf.FormatError('Expected a string but got '+repr(object)) +class AnyBytes(Schema): + """ + + Matches any byte string, but not a non-byte object. This schema + can be viewed as the Any() schema applied to byte strings, but an + additional check is performed to ensure only strings are considered. + + Supported methods include + matches(): returns a Boolean result. + check_match(): raises 'tuf.FormatError' on a mismatch. + + + + >>> schema = AnyBytes() + >>> schema.matches(b'') + True + >>> schema.matches(b'a string') + True + >>> schema.matches(['a']) + False + >>> schema.matches(3) + False + >>> schema.matches({}) + False + """ + + def __init__(self): + pass + + + def check_match(self, object): + if not isinstance(object, six.binary_type): + raise tuf.FormatError('Expected a byte string but got '+repr(object)) + + + + + class LengthString(Schema): """ @@ -217,7 +264,7 @@ class LengthString(Schema): """ def __init__(self, length): - if isinstance(length, bool) or not isinstance(length, (int, long)): + if isinstance(length, bool) or not isinstance(length, six.integer_types): # We need to check for bool as a special case, since bool # is for historical reasons a subtype of int. raise tuf.FormatError('Got '+repr(length)+' instead of an integer.') @@ -226,7 +273,7 @@ def __init__(self, length): def check_match(self, object): - if not isinstance(object, basestring): + if not isinstance(object, six.string_types): raise tuf.FormatError('Expected a string but got '+repr(object)) if len(object) != self._string_length: @@ -237,6 +284,48 @@ def check_match(self, object): +class LengthBytes(Schema): + """ + + Matches any Bytes of a specified length. The argument object must be either + a str() in Python 2, or bytes() in Python 3. At instantiation, the bytes + length is set and any future comparisons are checked against this internal + bytes value length. + + Supported methods include + matches(): returns a Boolean result. + check_match(): raises 'tuf.FormatError' on a mismatch. + + + + >>> schema = LengthBytes(5) + >>> schema.matches(b'Hello') + True + >>> schema.matches(b'Hi') + False + """ + + def __init__(self, length): + if isinstance(length, bool) or not isinstance(length, six.integer_types): + # We need to check for bool as a special case, since bool + # is for historical reasons a subtype of int. + raise tuf.FormatError('Got '+repr(length)+' instead of an integer.') + + self._bytes_length = length + + + def check_match(self, object): + if not isinstance(object, six.binary_type): + raise tuf.FormatError('Expected a byte but got '+repr(object)) + + if len(object) != self._bytes_length: + raise tuf.FormatError('Expected a byte of length '+ + repr(self._bytes_length)) + + + + + class OneOf(Schema): """ @@ -401,7 +490,7 @@ class ListOf(Schema): False """ - def __init__(self, schema, min_count=0, max_count=sys.maxint, list_name='list'): + def __init__(self, schema, min_count=0, max_count=sys.maxsize, list_name='list'): """ Create a new ListOf schema. @@ -433,7 +522,7 @@ def check_match(self, object): for item in object: try: self._schema.check_match(item) - except tuf.FormatError, e: + except tuf.FormatError as e: raise tuf.FormatError(str(e)+' in '+repr(self._list_name)) # Raise exception if the number of items in the list is @@ -465,8 +554,6 @@ class Integer(Schema): True >>> schema.matches(False) False - >>> schema.matches(0L) - True >>> schema.matches('a string') False >>> Integer(lo=10, hi=30).matches(25) @@ -475,7 +562,7 @@ class Integer(Schema): False """ - def __init__(self, lo= -sys.maxint, hi=sys.maxint): + def __init__(self, lo = -2147483648, hi = 2147483647): """ Create a new Integer schema. @@ -490,7 +577,7 @@ def __init__(self, lo= -sys.maxint, hi=sys.maxint): def check_match(self, object): - if isinstance(object, bool) or not isinstance(object, (int, long)): + if isinstance(object, bool) or not isinstance(object, six.integer_types): # We need to check for bool as a special case, since bool # is for historical reasons a subtype of int. raise tuf.FormatError('Got '+repr(object)+' instead of an integer.') @@ -556,7 +643,7 @@ def check_match(self, object): if not isinstance(object, dict): raise tuf.FormatError('Expected a dict but got '+repr(object)) - for key, value in object.items(): + for key, value in six.iteritems(object): self._key_schema.check_match(key) self._value_schema.check_match(value) @@ -643,12 +730,12 @@ def __init__(self, object_name='object', **required): """ # Ensure valid arguments. - for key, schema in required.items(): + for key, schema in six.iteritems(required): if not isinstance(schema, Schema): raise tuf.FormatError('Expected Schema but got '+repr(schema)) self._object_name = object_name - self._required = required.items() + self._required = list(required.items()) def check_match(self, object): @@ -672,7 +759,7 @@ def check_match(self, object): else: try: schema.check_match(item) - except tuf.FormatError, e: + except tuf.FormatError as e: raise tuf.FormatError(str(e)+' in '+self._object_name+'.'+key) @@ -827,8 +914,9 @@ def __init__(self, pattern=None, modifiers=0, re_object=None, re_name=None): re_name: Identifier for the regular expression object. """ - if not isinstance(pattern, basestring): - raise tuf.FormatError(repr(pattern)+' is not a string.') + if not isinstance(pattern, six.string_types): + if pattern is not None: + raise tuf.FormatError(repr(pattern)+' is not a string.') if re_object is None: if pattern is None: @@ -848,13 +936,11 @@ def __init__(self, pattern=None, modifiers=0, re_object=None, re_name=None): def check_match(self, object): - if not isinstance(object, basestring) or not self._re_object.match(object): + if not isinstance(object, six.string_types) or not self._re_object.match(object): raise tuf.FormatError(repr(object)+' did not match '+repr(self._re_name)) - - if __name__ == '__main__': # The interactive sessions of the documentation strings can # be tested by running schema.py as a standalone module. diff --git a/tuf/sig.py b/tuf/sig.py index 43b8c538..0d7b5992 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -35,6 +35,14 @@ is also a function for that. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import tuf import tuf.formats import tuf.keydb diff --git a/tuf/unittest_toolbox.py b/tuf/unittest_toolbox.py index 7a5efb33..67ef7286 100755 --- a/tuf/unittest_toolbox.py +++ b/tuf/unittest_toolbox.py @@ -17,6 +17,14 @@ Specifically, Modified_TestCase is a derived class from unittest.TestCase. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os import sys import shutil @@ -108,7 +116,7 @@ def _destroy_temp_file(): def make_temp_data_file(self, suffix='', directory=None, data = 'junk data'): """Returns an absolute path of a temp file containing data.""" temp_file_path = self.make_temp_file(suffix=suffix, directory=directory) - temp_file = open(temp_file_path, 'wb') + temp_file = open(temp_file_path, 'wt') temp_file.write(data) temp_file.close() return temp_file_path diff --git a/tuf/util.py b/tuf/util.py index 7c93668d..a847fd47 100755 --- a/tuf/util.py +++ b/tuf/util.py @@ -18,6 +18,14 @@ TempFile class that generates a file-like object for temporary storage, etc. """ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os import sys import gzip @@ -29,6 +37,7 @@ import tuf.hash import tuf.conf import tuf.formats +import tuf._vendor.six as six # The algorithm used by the repository to generate the digests of the # target filepaths, which are included in metadata files and may be prepended @@ -54,8 +63,9 @@ def _default_temporary_directory(self, prefix): """__init__ helper.""" try: self.temporary_file = tempfile.NamedTemporaryFile(prefix=prefix) - except OSError, err: - logger.critical('Temp file in '+temp_dir+'failed: '+repr(err)) + + except OSError as err: # pragma: no cover + logger.critical('Cannot create a system temporary directory: '+repr(err)) raise tuf.Error(err) @@ -77,22 +87,26 @@ def __init__(self, prefix='tuf_temp_'): """ self._compression = None + # If compression is set then the original file is saved in 'self._orig_file'. self._orig_file = None temp_dir = tuf.conf.temporary_directory - if temp_dir is not None and isinstance(temp_dir, str): + if temp_dir is not None and tuf.formats.PATH_SCHEMA.matches(temp_dir): try: self.temporary_file = tempfile.NamedTemporaryFile(prefix=prefix, dir=temp_dir) - except OSError, err: - logger.error('Temp file in '+temp_dir+' failed: '+repr(err)) + except OSError as err: + logger.error('Temp file in ' + temp_dir + ' failed: '+repr(err)) logger.error('Will attempt to use system default temp dir.') self._default_temporary_directory(prefix) + else: self._default_temporary_directory(prefix) + + def get_compressed_length(self): """ @@ -134,6 +148,8 @@ def flush(self): + + def read(self, size=None): """ @@ -155,14 +171,19 @@ def read(self, size=None): self.temporary_file.seek(0) data = self.temporary_file.read() self.temporary_file.seek(0) + return data + else: if not (isinstance(size, int) and size > 0): raise tuf.FormatError + return self.temporary_file.read(size) + + def write(self, data, auto_flush=True): """ @@ -216,6 +237,8 @@ def move(self, destination_path): + + def seek(self, *args): """ @@ -239,6 +262,8 @@ def seek(self, *args): + + def decompress_temp_file_object(self, compression): """ @@ -293,7 +318,7 @@ def decompress_temp_file_object(self, compression): try: self.temporary_file = gzip.GzipFile(fileobj=self.temporary_file, mode='rb') - except Exception, exception: + except Exception as exception: raise tuf.DecompressionError(exception) @@ -415,8 +440,10 @@ def ensure_parent_dir(filename): # Split 'filename' into head and tail, check if head exists. directory = os.path.split(filename)[0] + if directory and not os.path.exists(directory): - os.makedirs(directory, 0700) + # mode = 'rwx------'. 448 (decimal) is 700 in octal. + os.makedirs(directory, 448) @@ -508,7 +535,7 @@ def find_delegated_role(roles, delegated_role): # The index of a role, if any, with the same name. role_index = None - for index in xrange(len(roles)): + for index in six.moves.xrange(len(roles)): role = roles[index] name = role.get('name') @@ -770,19 +797,10 @@ def get_target_hash(target_filepath): # Calculate the hash of the filepath to determine which bin to find the # target. The client currently assumes the repository uses - # 'HASH_FUNCTION' to generate hashes. + # 'HASH_FUNCTION' to generate hashes and 'utf-8'. digest_object = tuf.hash.digest(HASH_FUNCTION) - - try: - digest_object.update(target_filepath) - - except UnicodeEncodeError: - # Sometimes, there are Unicode characters in target paths. We assume a - # UTF-8 encoding and try to hash that. - digest_object = tuf.hash.digest(HASH_FUNCTION) - encoded_target_filepath = target_filepath.encode('utf-8') - digest_object.update(encoded_target_filepath) - + encoded_target_filepath = target_filepath.encode('utf-8') + digest_object.update(encoded_target_filepath) target_filepath_hash = digest_object.hexdigest() return target_filepath_hash @@ -899,7 +917,7 @@ def load_json_file(filepath): # The file is mostly likely gzipped. if filepath.endswith('.gz'): logger.debug('gzip.open('+str(filepath)+')') - fileobject = gzip.open(filepath) + fileobject = six.StringIO(gzip.open(filepath).read().decode('utf-8')) else: logger.debug('open('+str(filepath)+')') @@ -908,11 +926,12 @@ def load_json_file(filepath): try: deserialized_object = json.load(fileobject) - except ValueError, TypeError: + except (ValueError, TypeError) as e: message = 'Cannot deserialize to a Python object: '+repr(filepath) raise tuf.Error(message) else: + fileobject.close() return deserialized_object finally: