Merge branch 'develop' of github.com:theupdateframework/tuf into tuf1.0_version_numbers

MANIFEST.in

@@ -11,7 +11,6 @@ include tests/repository_data/keystore/targets_key
 include tests/repository_data/keystore/timestamp_key
 include tuf/_vendor/ed25519/test_data/ed25519
 include tuf/_vendor/ed25519/LICENSE
-include tuf/_vendor/iso8601/LICENSE 
 
 recursive-include docs *.txt
 recursive-include docs/papers *.pdf

README.rst

@@ -189,12 +189,12 @@ Instructions for Contributors
 
 Development: `https://github.com/theupdateframework/tuf <https://github.com/theupdateframework/tuf>`_
 
-`Virtualenv <https://virtualenv.pypa.io/en/latest/virtualenv.html#introduction>`_
+`Virtualenv <https://virtualenv.pypa.io/en/latest/index.html>`_
 is a tool to create isolated Python environments. It also includes
 ``pip`` and ``setuptools``, Python packages used to install TUF and its
 dependencies. All installation methods of virtualenv are outlined in the
 `installation
-section <https://virtualenv.pypa.io/en/latest/virtualenv.html#installation>`_
+section <https://virtualenv.pypa.io/en/latest/installation.html>`_
 and instructions for installing locally from source here:
 ::
 

docs/tuf-spec.txt

@@ -39,7 +39,7 @@ Version 0.9
    (http://www.geni.net/)
    (http://www.nsf.gov/)
 
-   TUF's Python implementation is based heavily on Thandy, the application
+   TUF's reference implementation is based heavily on Thandy, the application
    updater for Tor (http://www.torproject.org/). Its design and this spec are
    also largely based on Thandy's, with many parts being directly borrowed
    from Thandy. The Thandy spec can be found here:
@@ -277,7 +277,7 @@ Version 0.9
 
    To prevent an adversary from replaying an out-of-date signed metadata file
    whose signature has not yet expired, an automated process periodically signs
-   a timestamped statement containing the the hash of the snapshot file.  Even
+   a timestamped statement containing the hash of the snapshot file.  Even
    though this timestamp key must be kept online, the risk posed to clients by
    compromise of this key is minimal.
 
@@ -428,11 +428,12 @@ Version 0.9
           METHOD is the key signing method used to generate the signature.
           SIGNATURE is a signature of the canonical JSON form of ROLE.
 
-   The current Python implementation of TUF defines two signing methods,
+   The current reference implementation of TUF defines two signing methods,
    although TUF is not restricted to any particular key signing method,
    key type, or cryptographic library:
        
        "RSASSA-PSS" : RSA Probabilistic signature scheme with appendix.
+                      The underlying hash function is SHA256.
        
        "ed25519" : Elliptic curve digital signature algorithm based on Twisted
                   Edwards curves.
@@ -954,9 +955,11 @@ Version 0.9
 
 6.1. Key management and migration
 
-   All keys except the timestamp file signing key and the mirror list signing
-   key should be stored securely offline (e.g. encrypted and on a separate
-   machine, in special-purpose hardware, etc.).
+   All keys, except those for the timestamp and mirrors roles, should be
+   stored securely offline (e.g. encrypted and on a separate machine, in
+   special-purpose hardware, etc.).  This document does not prescribe how keys
+   should be encrypted and stored, and so it is left to implementers of
+   this document to decide how best to secure them. 
 
    To replace a compromised root key or any other top-level role key, the root
    role signs a new root.json file that lists the updated trusted keys for the

examples/README.md

@@ -0,0 +1,12 @@
+This directory contains an example of a TUF repository, metadata, and key and
+client files.
+
+## WARNING ##
+These examples were last updated 2 years ago. We have since made changes to the
+format of our metadata and key files, and will need to regenerate them so the
+new tools can properly load them. We are currently working on a 1.0 release
+that will make further tweaks to the format of metadata and key files, so these
+examples will be modified once again.
+
+Note: The examples that are up-to-date and normally tested are located here:
+https://github.com/theupdateframework/tuf/tree/develop/tests/repository_data/

setup.py

@@ -80,12 +80,12 @@
 
 setup(
   name = 'tuf',
-  version = '0.9.9',
+  version = '0.10.0',
   description = 'A secure updater framework for Python',
   long_description = long_description,
-  author = 'http://www.theupdateframework.com',
+  author = 'https://www.updateframework.com',
   author_email = 'theupdateframework@googlegroups.com',
-  url = 'http://www.theupdateframework.com',
+  url = 'https://www.updateframework.com',
   keywords = 'update updater secure authentication key compromise revocation',
   classifiers = [
     'Development Status :: 4 - Beta',
@@ -101,9 +101,9 @@
     'Programming Language :: Python :: 2.6',
     'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.2',
     'Programming Language :: Python :: 3.3',
     'Programming Language :: Python :: 3.4',
+    'Programming Language :: Python :: 3.5',
     'Programming Language :: Python :: Implementation :: CPython',
     'Topic :: Security',
     'Topic :: Software Development'

tests/aggregate_tests.py

@@ -64,6 +64,6 @@
 
 if __name__ == '__main__':
   suite = unittest.TestLoader().loadTestsFromNames(tests_without_extension)
-  all_tests_passed = unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful()
+  all_tests_passed = unittest.TextTestRunner(verbosity=1).run(suite).wasSuccessful()
   if not all_tests_passed:
     sys.exit(1)

tests/test_init.py

@@ -29,8 +29,9 @@
 import logging
 
 import tuf
+import tuf.log
 
-logger = logging.getLogger('tuf.test_keys')
+logger = logging.getLogger('tuf.test_init')
 
 class TestInit(unittest.TestCase):
   def setUp(self):

tests/test_log.py

@@ -112,6 +112,9 @@ def test_add_console_handler(self):
     # Test for invalid argument.
     self.assertRaises(tuf.FormatError, tuf.log.add_console_handler, 51)
 
+    # Test that an exception is printed to the console.  Note: A stack trace
+    # is not included in the exception output because 'log.py' applies a filter
+    # to minimize the amount of output to the console. 
     try:
       raise TypeError('Test exception output in the console.')
 

tests/test_pyca_crypto_keys.py

@@ -84,7 +84,7 @@ def test_create_rsa_signature(self):
                       crypto_keys.create_rsa_signature, '', data)
    
     # Check for invalid 'data'.
-    self.assertRaises(TypeError,
+    self.assertRaises(tuf.FormatError,
                       crypto_keys.create_rsa_signature, private_rsa, '')
     
     self.assertRaises(tuf.FormatError,

tests/test_pycrypto_keys.py

@@ -84,7 +84,8 @@ def test_create_rsa_signature(self):
                       pycrypto.create_rsa_signature, '', data)
    
     # Check for invalid 'data'.
-    pycrypto.create_rsa_signature(private_rsa, '')
+    self.assertRaises(tuf.FormatError,
+                      pycrypto.create_rsa_signature, private_rsa, '')
    
     # create_rsa_signature should reject non-string data.
     self.assertRaises(tuf.FormatError,

tox.ini

@@ -4,8 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-#envlist = py26, py27, py32, py33, py34
-envlist = py27
+envlist = py26, py27, py33, py34, py35
 
 [testenv]
 changedir = tests

tuf/__init__.py

@@ -28,14 +28,8 @@
 from __future__ import division
 from __future__ import unicode_literals
 
-import logging
-
-import tuf.log
-
 import six
 
-logging = logging.getLogger('tuf.__init__')
-
 # Import 'tuf.formats' if a module tries to import the
 # entire tuf package (i.e., from tuf import *). 
 __all__ = ['formats']

tuf/client/updater.py

@@ -1338,8 +1338,8 @@ def _get_file(self, filepath, verify_file_function, file_type,
       return file_object
     
     else:
-      logger.exception('Failed to update {0} from all mirrors: {1}'.format(
-                       filepath, file_mirror_errors))
+      logger.error('Failed to update {0} from all mirrors: {1}'.format(
+                   filepath, file_mirror_errors))
       raise tuf.NoWorkingMirrorError(file_mirror_errors)
 
 

tuf/conf.py

@@ -46,6 +46,14 @@
 # http://docs.python.org/2/library/ssl.html#certificates
 ssl_certificates = None
 
+# The 'log.py' module manages TUF's logging system.  Users have the option to
+# enable/disable logging to a file via 'ENABLE_FILE_LOGGING'
+ENABLE_FILE_LOGGING = True
+
+# If file logging is enabled via 'ENABLE_FILE_LOGGING', TUF log messages will
+# be saved to 'LOG_FILENAME'
+LOG_FILENAME = 'tuf.log'
+
 # Since the timestamp role does not have signed metadata about itself, we set a
 # default but sane upper bound for the number of bytes required to download it.
 DEFAULT_TIMESTAMP_REQUIRED_LENGTH = 16384 #bytes

tuf/formats.py

@@ -151,8 +151,8 @@
 NAME_SCHEMA = SCHEMA.AnyString()
 NAMES_SCHEMA = SCHEMA.ListOf(NAME_SCHEMA)
 
-# A string representing data.
-DATA_SCHEMA = SCHEMA.AnyString()
+# A byte string representing data.
+DATA_SCHEMA = SCHEMA.AnyBytes()
 
 # Supported hash algorithms.
 HASHALGORITHMS_SCHEMA = SCHEMA.ListOf(SCHEMA.OneOf(
@@ -187,6 +187,9 @@
 # A PyCrypto signature.
 PYCRYPTOSIGNATURE_SCHEMA = SCHEMA.AnyBytes()
 
+# A pyca-cryptography signature.
+PYCACRYPTOSIGNATURE_SCHEMA = SCHEMA.AnyBytes()
+
 # An RSA key in PEM format.
 PEMRSA_SCHEMA = SCHEMA.AnyString()
 

tuf/log.py

@@ -69,12 +69,12 @@
 
 import tuf
 import tuf.formats
+import tuf.conf
 
 # Setting a handler's log level filters only logging messages of that level
 # (and above).  For example, setting the built-in StreamHandler's log level to
 # 'logging.WARNING' will cause the stream handler to only process messages
 # of levels: WARNING, ERROR, and CRITICAL. 
-_DEFAULT_LOG_FILENAME = 'tuf.log'
 _DEFAULT_LOG_LEVEL = logging.DEBUG
 _DEFAULT_CONSOLE_LOG_LEVEL = logging.INFO
 _DEFAULT_FILE_LOG_LEVEL = logging.DEBUG
@@ -103,19 +103,22 @@
 # set by default.
 console_handler = None
 
-# Set the built-in file handler.  Messages will be logged to
-# '_DEFAULT_LOG_FILENAME', and only those messages with a log level of
-# '_DEFAULT_LOG_LEVEL'.  The log level of messages handled by 'file_handler'
-# may be modified with 'set_filehandler_log_level()'.  '_DEFAULT_LOG_FILENAME'
-# will be opened in append mode.
-file_handler = logging.FileHandler(_DEFAULT_LOG_FILENAME)
-file_handler.setLevel(_DEFAULT_FILE_LOG_LEVEL)
-file_handler.setFormatter(formatter)
-
 # Set the logger and its settings.
 logger = logging.getLogger('tuf')
 logger.setLevel(_DEFAULT_LOG_LEVEL)
-logger.addHandler(file_handler)
+
+# Set the built-in file handler.  Messages will be logged to
+# 'tuf.conf.LOG_FILENAME', and only those messages with a log level of
+# '_DEFAULT_LOG_LEVEL'.  The log level of messages handled by 'file_handler'
+# may be modified with 'set_filehandler_log_level()'.  'tuf.conf.LOG_FILENAME'
+# will be opened in append mode.
+if tuf.conf.ENABLE_FILE_LOGGING:
+  file_handler = logging.FileHandler(tuf.conf.LOG_FILENAME)
+  file_handler.setLevel(_DEFAULT_FILE_LOG_LEVEL)
+  file_handler.setFormatter(formatter)
+  logger.addHandler(file_handler)
+else:
+  pass
 
 # Silently ignore logger exceptions.
 logging.raiseExceptions = False

tuf/pyca_crypto_keys.py

@@ -264,7 +264,7 @@ def create_rsa_signature(private_key, data):
     True
     >>> method == 'RSASSA-PSS'
     True
-    >>> tuf.formats.PYCRYPTOSIGNATURE_SCHEMA.matches(signature)
+    >>> tuf.formats.PYCACRYPTOSIGNATURE_SCHEMA.matches(signature)
     True
 
   <Arguments>
@@ -404,7 +404,7 @@ def verify_rsa_signature(signature, signature_method, public_key, data):
   tuf.formats.NAME_SCHEMA.check_match(signature_method)
 
   # Does 'signature' have the correct format?
-  tuf.formats.PYCRYPTOSIGNATURE_SCHEMA.check_match(signature)
+  tuf.formats.PYCACRYPTOSIGNATURE_SCHEMA.check_match(signature)
   
   # What about 'data'?
   tuf.formats.DATA_SCHEMA.check_match(data)
@@ -428,7 +428,7 @@ def verify_rsa_signature(signature, signature_method, public_key, data):
                                   salt_length=hashes.SHA256().digest_size), 
                                   hashes.SHA256())
 
-      verifier.update(data.encode('utf-8'))
+      verifier.update(data)
       
       # verify() raises 'cryptograpahy.exceptions.InvalidSignature' if the
       # signature is invalid.
@@ -917,7 +917,7 @@ def _encrypt(key_data, derived_key_information):
 
   # Encrypt the plaintext and get the associated ciphertext.
   # Do we need to check for any exceptions?
-  ciphertext = encryptor.update(key_data) + encryptor.finalize()
+  ciphertext = encryptor.update(key_data.encode('utf-8')) + encryptor.finalize()
   
   # Generate the hmac of the ciphertext to ensure it has not been modified.
   # The decryption routine may verify a ciphertext without having to perform
@@ -942,7 +942,7 @@ def _encrypt(key_data, derived_key_information):
   # of the fields it is separating.
   return binascii.hexlify(salt).decode() + _ENCRYPTION_DELIMITER + \
          str(iterations) + _ENCRYPTION_DELIMITER + \
-         hmac_value + _ENCRYPTION_DELIMITER + \
+         hmac_value.decode() + _ENCRYPTION_DELIMITER + \
          binascii.hexlify(iv).decode() + _ENCRYPTION_DELIMITER + \
          binascii.hexlify(ciphertext).decode()
   
@@ -993,7 +993,7 @@ def _decrypt(file_contents, password):
   generated_hmac = binascii.hexlify(generated_hmac_object.finalize())
 
 
-  if not tuf.util.digests_are_equal(generated_hmac, hmac):
+  if not tuf.util.digests_are_equal(generated_hmac.decode(), hmac):
     raise tuf.CryptoError('Decryption failed.')
     
   # Construct a Cipher object, with the key and iv.

tuf/pycrypto_keys.py

@@ -294,7 +294,7 @@ def create_rsa_signature(private_key, data):
     # If the passphrase is incorrect, PyCrypto returns: "RSA key format is not
     # supported".
     try:
-      sha256_object = Crypto.Hash.SHA256.new(data.encode('utf-8'))
+      sha256_object = Crypto.Hash.SHA256.new(data)
       rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key)
     
     except (ValueError, IndexError, TypeError) as e:
@@ -396,7 +396,7 @@ def verify_rsa_signature(signature, signature_method, public_key, data):
     try:
       rsa_key_object = Crypto.PublicKey.RSA.importKey(public_key)
       pkcs1_pss_verifier = Crypto.Signature.PKCS1_PSS.new(rsa_key_object)
-      sha256_object = Crypto.Hash.SHA256.new(data.encode('utf'))
+      sha256_object = Crypto.Hash.SHA256.new(data)
       valid_signature = pkcs1_pss_verifier.verify(sha256_object, signature)
     
     except (ValueError, IndexError, TypeError) as e: