Merge pull request #123 from theupdateframework/log

log.py updates
log.py changes reviewed by both Trishank and Monzur.
This commit is contained in:
Vladimir Diaz 2013-10-14 06:24:14 -07:00
commit f8b781b19b
13 changed files with 180 additions and 91 deletions

View file

@ -182,6 +182,3 @@ def test_endless_data_attack(using_tuf=False, TIMESTAMP=False):
print(str(error))
else:
print('Endless data attack did not work on download with TUF!')

View file

@ -84,7 +84,6 @@ def test_extraneous_dependency_attack(using_tuf=False):
ERROR_MSG = 'Extraneous Dependency Attack was Successful!'
try:
# Setup.
root_repo, url, server_proc, keyids = util_test_tools.init_repo(using_tuf)

View file

@ -198,8 +198,3 @@ def test_replay_attack(using_tuf=False):
print('Download without TUF failed due to: '+str(exception))
else:
print('Download without TUF did NOT fail due to replayed metadata attack!')

View file

@ -107,7 +107,6 @@ def test_slow_retrieval_attack(using_tuf=False, mode=None):
url_to_file = url+'reg_repo/'+file_basename
downloaded_file = os.path.join(downloads, file_basename)
if using_tuf:
tuf_repo = os.path.join(root_repo, 'tuf_repo')

View file

@ -36,6 +36,8 @@
# internal function.
json = tuf.util.import_json()
tuf.repo.keystore._PBKDF2_ITERATIONS = 1000
# Creating a directory string in current directory.
_CURRENT_DIR = os.getcwd()
_DIR = os.path.join(_CURRENT_DIR, 'test_keystore')
@ -297,8 +299,10 @@ def test_get_key(self):
def test_internal_encrypt(self):
# Test for valid arguments to '_encrypt()' and a valid return type.
salt = Crypto.Random.new().read(16)
iterations = tuf.repo.keystore._PBKDF2_ITERATIONS
derived_key = Crypto.Protocol.KDF.PBKDF2(PASSWDS[0], salt)
derived_key_information = {'salt': salt, 'derived_key': derived_key}
derived_key_information = {'salt': salt, 'derived_key': derived_key,
'iterations': iterations}
encrypted_key = KEYSTORE._encrypt(json.dumps(RSAKEYS[0]),
derived_key_information)
self.assertEqual(type(encrypted_key), str)
@ -310,8 +314,11 @@ def test_internal_decrypt(self):
tuf.formats.KEY_SCHEMA.check_match(RSAKEYS[0])
salt = Crypto.Random.new().read(16)
salt, derived_key = tuf.repo.keystore._generate_derived_key(PASSWDS[0], salt)
derived_key_information = {'salt': salt, 'derived_key': derived_key}
salt, iterations, derived_key = \
tuf.repo.keystore._generate_derived_key(PASSWDS[0], salt)
derived_key_information = {'salt': salt,
'iterations': iterations,
'derived_key': derived_key}
# Getting a valid encrypted key using '_encrypt()'.
encrypted_key = KEYSTORE._encrypt(json.dumps(RSAKEYS[0]),

View file

@ -72,7 +72,7 @@ def test_2_build_repository(self):
proj_files = self.make_temp_directory_with_data_files()
proj_dir = os.path.join(proj_files[0], 'targets')
input_dict = {'expiration':'12/12/2013',
input_dict = {'expiration':'12/12/2020',
'root':{'threshold':1, 'password':'pass'},
'targets':{'threshold':1, 'password':'pass'},
'release':{'threshold':1, 'password':'pass'},
@ -128,7 +128,7 @@ def _remove_repository_directories(repo_dir, keystore_dir, client_dir):
_remove_repository_directories(repo_dir, keystore_dir, client_dir)
# Restore expiration.
input_dict['expiration'] = '10/10/2013'
input_dict['expiration'] = '10/10/2020'
# Supplying bogus 'root' threshold. Doing this for all roles slows
# the test significantly.

View file

@ -176,6 +176,7 @@ def __str__(self):
class UnknownMethodError(CryptoError):
"""Indicate that a user-specified cryptograpthic method is unknown."""
pass
@ -311,8 +312,3 @@ def __str__(self):
all_errors += '\n '+str(mirror_netloc)+': '+str(mirror_error)
return all_errors

View file

@ -54,4 +54,14 @@
# The time (in seconds) we ignore a server with a slow initial retrieval speed.
SLOW_START_GRACE_PERIOD = 30 #seconds
# The current "good enough" number of PBKDF2 passphrase iterations.
# We recommend that important keys, such as root, be kept offline.
# 'tuf.conf.PBKDF2_ITERATIONS' should increase as CPU speeds increase, set here
# at 100,000 iterations by default (in 2013). The repository maintainer may opt
# to modify the default setting according to their security needs and
# computational restrictions. A strong user password is still important.
# Modifying the number of iterations will result in a new derived key+PBDKF2
# combination if the key is loaded and re-saved, overriding any previous
# iteration setting used by the old '<keyid>.key'.
# https://en.wikipedia.org/wiki/PBKDF2
PBKDF2_ITERATIONS = 100000

View file

@ -121,6 +121,10 @@
# An integer representing length. Must be 0, or greater.
LENGTH_SCHEMA = SCHEMA.Integer(lo=0)
# An integer representing logger levels, such as logging.CRITICAL (=50).
# Must be between 0 and 50.
LOGLEVEL_SCHEMA = SCHEMA.Integer(lo=0, hi=50)
# A string representing a named object.
NAME_SCHEMA = SCHEMA.AnyString()

View file

@ -48,6 +48,13 @@
logging.DEBUG 10
logging.NOTSET 0
The logging module is thread-safe. Logging to a single file from
multiple threads in a single process is also thread-safe. The logging
module is NOT thread-safe when logging to a single file across multiple
processes:
http://docs.python.org/2/library/logging.html#thread-safety
http://docs.python.org/2/howto/logging-cookbook.html
"""
@ -72,9 +79,14 @@
_FORMAT_STRING = '[%(asctime)s UTC] [%(name)s] [%(levelname)s]'+\
'[%(funcName)s:%(lineno)s@%(filename)s] %(message)s'
# Ask all Formatter instances to talk GMT.
# http://docs.python.org/2/library/logging.html#logging.Formatter.formatException
# Ask all Formatter instances to talk GMT. Set the 'converter' attribute of
# 'logging.Formatter' so that all formatters use Greenwich Mean Time.
# http://docs.python.org/2/library/logging.html#logging.Formatter.formatTime
# The 2nd paragraph in the link above contains the relevant information.
# GMT = UTC (Coordinated Universal Time). TUF metadata stores timestamps in UTC.
# We previously displayed the local time but this lead to confusion when
# visually comparing logger events and metadata information. Unix time stamps
# are fine but they may be less human-readable than UTC.
logging.Formatter.converter = time.gmtime
formatter = logging.Formatter(_FORMAT_STRING)
@ -144,6 +156,7 @@ def filter(self, record):
# original exception traceback. The exc_info is explained here:
# http://docs.python.org/2/library/sys.html#sys.exc_info
exc_type, exc_value, exc_traceback = record.exc_info
# Simply set the class name as the exception text.
record.exc_text = exc_type.__name__
@ -177,7 +190,7 @@ def set_log_level(log_level=_DEFAULT_LOG_LEVEL):
# Does 'log_level' have the correct format?
# Raise 'tuf.FormatError' if there is a mismatch.
tuf.formats.LENGTH_SCHEMA.check_match(log_level)
tuf.formats.LOGLEVEL_SCHEMA.check_match(log_level)
logger.setLevel(log_level)
@ -208,7 +221,7 @@ def set_filehandler_log_level(log_level=_DEFAULT_FILE_LOG_LEVEL):
# Does 'log_level' have the correct format?
# Raise 'tuf.FormatError' if there is a mismatch.
tuf.formats.LENGTH_SCHEMA.check_match(log_level)
tuf.formats.LOGLEVEL_SCHEMA.check_match(log_level)
file_handler.setLevel(log_level)
@ -240,7 +253,10 @@ def set_console_log_level(log_level=_DEFAULT_CONSOLE_LOG_LEVEL):
# Does 'log_level' have the correct format?
# Raise 'tuf.FormatError' if there is a mismatch.
tuf.formats.LENGTH_SCHEMA.check_match(log_level)
tuf.formats.LOGLEVEL_SCHEMA.check_match(log_level)
# Assign to the global console_handler object.
global console_handler
if console_handler is not None:
console_handler.setLevel(log_level)
@ -250,6 +266,8 @@ def set_console_log_level(log_level=_DEFAULT_CONSOLE_LOG_LEVEL):
def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL):
"""
<Purpose>
@ -271,17 +289,17 @@ def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL):
None.
"""
# Assign to the global console_handler object.
global console_handler
# Does 'log_level' have the correct format?
# Raise 'tuf.FormatError' if there is a mismatch.
tuf.formats.LENGTH_SCHEMA.check_match(log_level)
tuf.formats.LOGLEVEL_SCHEMA.check_match(log_level)
# Assign to the global console_handler object.
global console_handler
if not console_handler:
# Set the console handler for the logger. The built-in console handler will
# log messages to 'sys.stderr' and capture 'log_level' messages.
# NOTE: This is not thread-safe.
console_handler = logging.StreamHandler()
# Get our filter for the console handler.
console_filter = ConsoleFilter()
@ -299,18 +317,31 @@ def add_console_handler(log_level=_DEFAULT_CONSOLE_LOG_LEVEL):
def remove_console_handler():
# Assign to the global console_handler object.
"""
<Purpose>
Remove the console handler from the logger in 'log.py', if previously added.
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
A handler belonging to the console is removed from the 'log.py' logger
and the console handler is marked as unset.
<Returns>
None.
"""
# Assign to the global 'console_handler' object.
global console_handler
if console_handler:
logger.removeHandler(console_handler)
# NOTE: This is not thread-safe.
console_handler = None
logger.debug('Removed a console handler.')
else:
logger.warn('We do not have a console handler.')

View file

@ -61,14 +61,16 @@
import Crypto.Random
# The mode of operation is presently set to CTR (CounTeR Mode) for symmetric
# block encryption (AES-256). PyCrypto provides a callable stateful block
# counter that can update successive blocks when needed. The initial random
# block (IV) can be set to begin the process of incrementing the 128-bit blocks
# and allowing the AES algorithm to perform cipher block operations on them.
# block encryption (AES-256, where the symmetric key is 256 bits). PyCrypto
# provides a callable stateful block counter that can update successive blocks
# when needed. The initial random block, or initialization vector (IV), can
# be set to begin the process of incrementing the 128-bit blocks and allowing
# the AES algorithm to perform cipher block operations on them.
import Crypto.Util.Counter
import tuf.rsa_key
import tuf.util
import tuf.conf
# See 'log.py' to learn how logging is handled in TUF.
@ -77,7 +79,7 @@
json = tuf.util.import_json()
# The delimiter symbol used to separate the different sections
# of encrypted files (i.e., salt, IV, ciphertext, passphrase).
# of encrypted files (i.e., salt, iterations, hmac, IV, ciphertext).
# This delimiter is arbitrarily chosen and should not occur in
# the hexadecimal representations of the fields it is separating.
_ENCRYPTION_DELIMITER = '@@@@'
@ -86,23 +88,35 @@
_AES_KEY_SIZE = 32
# Default salt size, in bytes. A 128-bit salt (i.e., a random sequence of data
# to protect against dictionary attacks) is generated for PBKDF2.
# to protect against attacks that use precomputed rainbow tables to crack
# password hashes) is generated for PBKDF2.
_SALT_SIZE = 16
# Default PBKDF2 passphrase iterations. The current (2013) "good enough" number
# Default PBKDF2 passphrase iterations. The current "good enough" number
# of passphrase iterations. We recommend that important keys, such as root,
# be kept offline. Are we going overboard with respect to our use case?
# http://security.stackexchange.com/questions/3959/recommended-of-iterations-when-using-pkbdf2-sha256
_PBKDF2_ITERATIONS = 100000
# be kept offline. 'tuf.conf.PBKDF2_ITERATIONS' should increase as CPU
# speeds increase, set here at 100,000 iterations by default (in 2013).
# Repository maintainers may opt to modify the default setting according to
# their security needs and computational restrictions. A strong user password
# is still important. Modifying the number of iterations will result in a new
# derived key+PBDKF2 combination if the key is loaded and re-saved, overriding
# any previous iteration setting used by the old '<keyid>.key'.
# https://en.wikipedia.org/wiki/PBKDF2
_PBKDF2_ITERATIONS = tuf.conf.PBKDF2_ITERATIONS
# A user password is read and a derived key generated. The derived key and
# salt returned by the key derivation function (PBKDF2) is saved in
# '_derived_keys', which has the form:
# {keyid: {'salt': ..., 'derived_key': ...}}
# A user password is read and a derived key generated. The derived key returned
# by the key derivation function (PBKDF2) is saved in '_derived_keys', along
# with the salt and iterations used, which has the form:
# {keyid: {'salt': '\x9b\x90\xf1g\xb9li\x04\x8d\x10h\xb5T\xaa\xc1',
# 'iterations': 10000,
# 'derived_key': '\xda\xed\xf2\xe0\x8f\x03\xeb\xde!\xc4RJ'},
# keyid2: ...}
_derived_keys = {}
# The keystore database, which has the form:
# {keyid: key, keyid2: key2, ...}
# {keyid: key,
# keyid2: key2,
# ...}
_keystore = {}
@ -171,8 +185,8 @@ def add_rsakey(rsakey_dict, password, keyid=None):
'Expected: '+repr(keyid)
raise tuf.Error(message)
# Check if the keyid belonging to 'rsakey_dict' is not already
# available in the key database.
# Check if the keyid belonging to 'rsakey_dict' is not already available in
# the key database.
keyid = rsakey_dict['keyid']
if keyid in _keystore:
message = 'Keyid: '+repr(keyid)+' already exists.'
@ -181,8 +195,10 @@ def add_rsakey(rsakey_dict, password, keyid=None):
# The '_derived_keys' dictionary does not store the user's password. A key
# derivation function is applied to 'password' prior to storing it in
# _derived_key and may then be used as a symmetric key.
salt, derived_key = _generate_derived_key(password)
_derived_keys[keyid] = {'salt': salt, 'derived_key': derived_key}
salt, iterations, derived_key = _generate_derived_key(password)
_derived_keys[keyid] = {'salt': salt,
'derived_key': derived_key,
'iterations': iterations}
_keystore[keyid] = rsakey_dict
@ -249,15 +265,15 @@ def load_keystore_from_keyfiles(directory_name, keyids, passwords):
keyfilename = keyid+'.key'
full_filepath = os.path.join(directory_name, keyfilename)
raw_contents = open(full_filepath, 'rb').read()
except:
logger.warn('Could not find key '+repr(full_filepath)+'.')
except (OSError, IOError), e:
logger.warn('Could not load key file: '+repr(full_filepath)+'.')
else:
# Try to decrypt the file using one of the passwords in 'passwords'.
for password in passwords:
try:
json_data = _decrypt(raw_contents, password)
except:
logger.warn(repr(full_filepath)+' contains an invalid key.')
except tuf.CryptoError, e:
logger.warn(repr(full_filepath)+' could not be decrypted.')
continue
try:
@ -447,16 +463,22 @@ def change_password(keyid, old_password, new_password):
# stores derived keys instead of user passwords, according to the
# key derivation function used by _generate_derived_key().
salt = _derived_keys[keyid]['salt']
junk, old_derived_key = _generate_derived_key(old_password, salt)
iterations = _derived_keys[keyid]['iterations']
# Discard the old "salt" and "iterations" values, as we only need the old
# derived key.
junk_old_salt, junk_old_iterations, old_derived_key = \
_generate_derived_key(old_password, salt, iterations)
if _derived_keys[keyid]['derived_key'] != old_derived_key:
message = 'Old password invalid.'
raise tuf.BadPasswordError(message)
# Update '_derived_keys[keyid]' with the new derived key and salt.
salt, new_derived_key = _generate_derived_key(new_password)
salt, iterations, new_derived_key = _generate_derived_key(new_password)
_derived_keys[keyid] = {}
_derived_keys[keyid]['salt'] = salt
_derived_keys[keyid]['derived_key'] = new_derived_key
_derived_keys[keyid]['iterations'] = iterations
@ -502,7 +524,7 @@ def get_key(keyid):
def _generate_derived_key(password, salt=None):
def _generate_derived_key(password, salt=None, iterations=None):
"""
Generate a derived key by feeding 'password' to the Password-Based Key
Derivation Function (PBKDF2). PyCrypto's PBKDF2 implementation is
@ -514,6 +536,9 @@ def _generate_derived_key(password, salt=None):
if salt is None:
salt = Crypto.Random.new().read(_SALT_SIZE)
if iterations is None:
iterations = _PBKDF2_ITERATIONS
def pseudorandom_function(password, salt):
"""
@ -530,10 +555,10 @@ def pseudorandom_function(password, salt):
# must be callable.
derived_key = Crypto.Protocol.KDF.PBKDF2(password, salt,
dkLen=_AES_KEY_SIZE,
count=_PBKDF2_ITERATIONS,
count=iterations,
prf=pseudorandom_function)
return salt, derived_key
return salt, iterations, derived_key
@ -554,8 +579,9 @@ def _encrypt(key_data, derived_key_information):
'private': '-----BEGIN RSA PRIVATE KEY----- ...'}}
'derived_key_information' is a dictionary of the form:
{'salt': '...'
'derived_key': '...'}
{'salt': '...',
'derived_key': '...',
'iterations': '...'}
'tuf.CryptoError' raised if the encryption fails.
@ -563,7 +589,7 @@ def _encrypt(key_data, derived_key_information):
# Generate a random initialization vector (IV). The 'iv' is treated as the
# initial counter block to a stateful counter block function (i.e.,
# PyCrypto's 'Crypto.Util.Counter'. The AES block cipher operates on 128-bit
# PyCrypto's 'Crypto.Util.Counter'). The AES block cipher operates on 128-bit
# blocks, so generate a random 16-byte initialization block. PyCrypto expects
# the initial value of the stateful counter to be an integer.
# Follow the provably secure encrypt-then-MAC approach, which affords the
@ -583,7 +609,11 @@ def _encrypt(key_data, derived_key_information):
# repetitions are performed by AES, 14 cycles for 256-bit keys.
try:
ciphertext = aes_cipher.encrypt(key_data)
except:
# Raise generic exception message to avoid revealing sensitive information,
# such as invalid passwords, encryption keys, etc., that an attacker can use
# to his advantage.
except Exception, e:
message = 'The key data could not be encrypted.'
raise tuf.CryptoError(message)
@ -591,15 +621,22 @@ def _encrypt(key_data, derived_key_information):
# The decryption routine may verify a ciphertext without having to perform
# a decryption operation.
salt = derived_key_information['salt']
derived_key = derived_key_information['derived_key']
hmac_object = Crypto.Hash.HMAC.new(derived_key, ciphertext, Crypto.Hash.SHA256)
hmac_object = Crypto.Hash.HMAC.new(symmetric_key, ciphertext,
Crypto.Hash.SHA256)
hmac = hmac_object.hexdigest()
# Return the hmac, initialization vector, and ciphertext as a single string.
# These three values are delimited by '_ENCRYPTION_DELIMITER' to make
# extraction easier. This delimiter is arbitrarily chosen and should not
# occur in the hexadecimal representations of the fields it is separating.
# Store the number of PBKDF2 iterations used to derive the symmetric key so
# that the decryption routine can regenerate the symmetric key successfully.
# The pbkdf2 iterations are allowed to vary for the keys loaded and saved.
iterations = derived_key_information['iterations']
# Return the salt, iterations, hmac, initialization vector, and ciphertext
# as a single string. These five values are delimited by
# '_ENCRYPTION_DELIMITER' to make extraction easier. This delimiter is
# arbitrarily chosen and should not occur in the hexadecimal representations
# of the fields it is separating.
return binascii.hexlify(salt) + _ENCRYPTION_DELIMITER + \
binascii.hexlify(str(iterations)) + _ENCRYPTION_DELIMITER + \
binascii.hexlify(hmac) + _ENCRYPTION_DELIMITER + \
binascii.hexlify(iv) + _ENCRYPTION_DELIMITER + \
binascii.hexlify(ciphertext)
@ -616,21 +653,25 @@ def _decrypt(file_contents, password):
"""
# Extract the salt, hmac, initialization vector, and ciphertext from
# 'file_contents'. These three values are delimited by '_ENCRYPTION_DELIMITER'.
# This delimiter is arbitrarily chosen and should not occur in the
# hexadecimal representations of the fields it is separating.
salt, hmac, iv, ciphertext = file_contents.split(_ENCRYPTION_DELIMITER)
# Extract the salt, iterations, hmac, initialization vector, and ciphertext
# from 'file_contents'. These five values are delimited by
# '_ENCRYPTION_DELIMITER'. This delimiter is arbitrarily chosen and should
# not occur in the hexadecimal representations of the fields it is separating.
salt, iterations, hmac, iv, ciphertext = \
file_contents.split(_ENCRYPTION_DELIMITER)
# Ensure we have the expected raw data for the delimited cryptographic data.
salt = binascii.unhexlify(salt)
salt = binascii.unhexlify(salt)
iterations = int(binascii.unhexlify(iterations))
hmac = binascii.unhexlify(hmac)
iv = binascii.unhexlify(iv)
ciphertext = binascii.unhexlify(ciphertext)
# Generate derived key from 'password'. The salt is specified so that
# the expected derived key is regenerated correctly.
junk, derived_key = _generate_derived_key(password, salt)
# Generate derived key from 'password'. The salt and iterations are specified
# so that the expected derived key is regenerated correctly. Discard the old
# "salt" and "iterations" values, as we only need the old derived key.
junk_old_salt, junk_old_iterations, derived_key = \
_generate_derived_key(password, salt, iterations)
# Verify the hmac to ensure the ciphertext is valid and has not been altered.
# See the encryption routine for why we use the encrypt-then-MAC approach.
@ -650,7 +691,11 @@ def _decrypt(file_contents, password):
counter=stateful_counter_128bit_blocks)
try:
key_plaintext = aes_cipher.decrypt(ciphertext)
except:
# Raise generic exception message to avoid revealing sensitive information,
# such as invalid passwords, encryption keys, etc., that an attacker can
# use to his advantage.
except Exception, e:
raise tuf.CryptoError('Decryption failed.')
return key_plaintext

View file

@ -378,9 +378,14 @@ def create_signature(rsakey_dict, data):
keyid = rsakey_dict['keyid']
method = 'PyCrypto-PKCS#1 PSS'
sig = None
if private_key:
# Take
# Verify the signature, but only if the private key has been set. The private
# key is a NULL string if unset. Although it may be clearer to explicit check
# that 'private_key' is not '', we can/should check for a value and not
# compare identities with the 'is' keyword.
if len(private_key):
# Calculate the SHA256 hash of 'data' and generate the hash's PKCS1-PSS
# signature.
try:
rsa_key_object = Crypto.PublicKey.RSA.importKey(private_key)
sha256_object = Crypto.Hash.SHA256.new(data)

View file

@ -390,9 +390,10 @@ def generate_rsakey():
Modified_TestCase.rsa_keyids.append(keyid)
password = Modified_TestCase.random_string()
Modified_TestCase.rsa_passwords[keyid] = password
salt, derived_key = keystore._generate_derived_key(password)
salt, iterations, derived_key = keystore._generate_derived_key(password)
Modified_TestCase.rsa_derived_keys[keyid] = {'salt': salt,
'derived_key': derived_key}
'derived_key': derived_key,
'iterations': iterations}
Modified_TestCase.rsa_keystore[keyid] = rsakey
return keyid