diff --git a/src/tuf/pushtools/__init__.py b/src/tuf/pushtools/__init__.py old mode 100644 new mode 100755 diff --git a/src/tuf/pushtools/push.cfg.sample b/src/tuf/pushtools/push.cfg.sample index 59730bdc..bd0e6efa 100755 --- a/src/tuf/pushtools/push.cfg.sample +++ b/src/tuf/pushtools/push.cfg.sample @@ -1,10 +1,10 @@ [general] transfer_module = scp -metadata_path = targets.txt +metadata_path = /var/tuf/test-repo/metadata/targets.txt +targets_directory = /var/tuf/test-repo/targets [scp] -host = -user = -identity_file = -# remote_dir must be similarly configured on the repository side. -remote_dir = ~/test/pushes +host = localhost +user = user +identity_file = ~/.ssh/id_rsa +remote_directory = ~/pushes diff --git a/src/tuf/pushtools/push.py b/src/tuf/pushtools/push.py index 426fd043..072c26ae 100755 --- a/src/tuf/pushtools/push.py +++ b/src/tuf/pushtools/push.py @@ -1,104 +1,184 @@ -#!/usr/bin/env python -# Copyright 2010 The Update Framework. See LICENSE for licensing information. """ -This script provides a way for developers to push a signed targets metadata -file and the referenced targets to a repository. The repository adds these -files to the repository by running the receivetools/receive.py script. + + push.py -Usage: - ./push.py COMMAND COMMAND_ARGS + + Vladimir Diaz + + + August 2012. Based on a previous version by Geremy Condra. + + + See LICENSE for licensing information. + + + This script provides a way for developers to push a signed targets metadata + file (i.e., 'targets.txt') and the referenced targets to a repository. The + repository adds these files to the repository by running the + 'tuf/pushtools/receivetools/receive.py' script. + + 'push.py' is not a required module of The Update Framework, but is provided + to allow developers to remotely update the target files served by a + repository. The actual file transfers are completed by a separate command + available on the client machine. The 'SCP' (secure copy) command is currently + supported. This script may be viewed as a front-end to the python transfer + modules (e.g., 'tuf.pushtools.transfer.scp'). A configuration file can be + specified allowing the user to customize the transfer and supply the locations + of targets and metadata. + + Usage: + $ python push.py --config - Known commands: - push + Example: + $ python push.py --config ./push.cfg -Example: - ./push.py push push.cfg targets.txt targetfile1 targetfile2 + Details of the 'push.py' script: -Details of 'push' command: - -The developer provides the path to a configuration file that lists: + The developer provides the path to a configuration file that lists: * The path to the targets metadata file. * The name of the transfer module to use for transferring the - files to the repository (e.g. 'scp'). + files to the repository (e.g., 'scp'). * Configuration information that is specific to the transfer module. -See the push.cfg.sample file for an example configuration file. + See the 'push.cfg.sample' file for an example configuration file. -The transfer module needs the following functionality: + The transfer module needs the following functionality: * A way to transfer target files and the new metadata file to the - repository. + repository. The 'scp' transfer modules is currently supported. -The transfer module may also include the following functionality: + The transfer module may also include the following functionality: * A way to determine whether the repository has rejected the push and, if so, the reason for the rejection. + """ -import ConfigParser +import os import sys +import optparse -import tuf +import tuf.formats +import tuf.pushtools.pushtoolslib +import tuf.pushtools.transfer.scp -def _read_config_file(filename): - """Return a dictionary where the keys are section names and the values - are dictionaries of keys/values in that section. - """ - config = ConfigParser.RawConfigParser() - config.read(filename) - configdict = {} - for section in config.sections(): - configdict[section] = {} - for key, value in config.items(section): - if key in ['seconds', 'minutes', 'days', 'hours']: - value = int(value) - elif key in ['keyids']: - value = value.split(',') - if key in configdict[section]: - configdict[section][key] = [] - else: - configdict[section][key] = value - return configdict + +def push(config_filepath): + """ + + Perform a push/transfer of target files to a host. The configuration file + 'config_filepath' provides the required settings needed by the transfer + command. In the case of an 'scp' configuration file, the configuration + file would contain 'host', 'user', 'identity file', and 'remote directory' + entries. + + + config_filepath: + The push configuration file (i.e., 'push.cfg'). + + + tuf.FormatError, if any of the arguments are incorrectly formatted. + + tuf.Error, if there was an error while processing the push. + + + The 'config_filepath' file is read and its contents stored, the files + in the targets directory (specified in the config file) are copied, + and the copied targets transfered to a specified host. + + + None. + + """ + + # Do the arguments have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(config_filepath) + + # Is the path to the configuration file valid? + if not os.path.isfile(config_filepath): + message = 'The configuration file path is invalid.' + raise tuf.Error(message) + config_filepath = os.path.abspath(config_filepath) + + # Retrieve the push configuration settings required by the transfer + # modules. Raise ('tuf.FormatError', 'tuf.Error') if a valid + # configuration file cannot be retrieved. + config_dict = tuf.pushtools.pushtoolslib.read_config_file(config_filepath, 'push') + + # Extract the transfer module identified in the configuration file. + transfer_module = config_dict['general']['transfer_module'] + + # 'scp' is the only transfer module currently supported. Perform + # an scp-transfer of the targets located in the targets directory as + # listed in the configuration file. + if transfer_module == 'scp': + tuf.pushtools.transfer.scp.transfer(config_dict) + else: + message = 'Cannot perform a transfer using '+repr(transfer_module) + raise tuf.Error(message) -def _get_transfer_module(modulename): - __import__("transfer.%s" % modulename) - return sys.modules["transfer.%s" % modulename] -def push(args): - config = _read_config_file(args[0]) - targets = args[1:] - transfermod = _get_transfer_module(config['general']['transfer_module']) - context = transfermod.TransferContext(config['scp']) - context.transfer(targets, config['general']['metadata_path']) - context.finalize() +def parse_options(): + """ + + Parse the command-line options. 'push.py' expects the '--config' + option to be set by the user. + + Example: + $ python push.py --config ./push.cfg + + The '--config' option accepts a path argument to the push configuration + file (i.e., 'push.cfg'). If the required option is unset, a parser error + is printed and the script exits. + + + None. + + + None. + + + None. + + + The options object returned by the parser's parse_args() method. + + """ + + usage = 'usage: %prog --config ' + option_parser = optparse.OptionParser(usage=usage) + + # Add the options supported by 'push.py' to the option parser. + option_parser.add_option('--config', action='store', type='string', + help='Specify the "push.cfg" configuration file.') + + (options, remaining_arguments) = option_parser.parse_args() + + # Ensure the '--config' option is set. If the required option is unset, + # option_parser.error() will print an error message and exit. + if options.config is None: + message = '"--config" must be set on the command-line.' + option_parser.error(message) + + return options -def getstatus(): - raise NotImplementedError -def usage(): - print "Known commands:" - print " push config_file target [target ...]" - sys.exit(1) - - -def main(): - if len(sys.argv) < 2: - usage() - cmd = sys.argv[1] - args = sys.argv[2:] - if cmd in ["push", "getstatus"]: - try: - globals()[cmd](args) - except tuf.BadPasswordError: - print >> sys.stderr, "Password incorrect." - else: - usage() - if __name__ == '__main__': - main() + options = parse_options() + + # Perform a 'push' of the target files specified in the configuration file. + try: + push(options.config) + except (tuf.FormatError, tuf.Error), e: + sys.stderr.write('Error: '+str(e)+'\n') + sys.exit(1) + + # The 'push' and command-line options were processed successfully. + sys.exit(0) diff --git a/src/tuf/pushtools/pushtoolslib.py b/src/tuf/pushtools/pushtoolslib.py new file mode 100755 index 00000000..055de894 --- /dev/null +++ b/src/tuf/pushtools/pushtoolslib.py @@ -0,0 +1,150 @@ +""" + + pushtoolslib.py + + + Vladimir Diaz + + + September 2012. + + + See LICENSE for licensing information. + + + Provide a central location for functions and data useful to multiple + 'tuf.pushtools' modules. A 'read_config_file' function is currently + provided that returns correctly formatted configuration dictionaries + needed by the 'push.py' and 'receive.py' scripts. + +""" + +import ConfigParser +import os + +import tuf.formats + +PUSH_CONFIG = 'push.cfg' +RECEIVE_CONFIG = 'receive.cfg' +TRANSFER_MODULES = ['scp'] +CONFIG_TYPES = ['push', 'receive'] + + +def read_config_file(filename, config_type): + """ + + Return a dictionary where the keys are section names and the values + dictionaries of the keys/values in that section. The returned + dict should be correctly formatted, contain the required data + according to its config type, and be valid (i.e., a correctly named + file and available). + + Example config: + + config_dict = {'general': {'transfer_module': 'scp', ...}, + 'scp': {'host': 'localhost', 'user': 'McFly', ...}} + + + filename: + The filepath to the configuration file. + + config_type: + A string identifying the type of config file expected. Supported + config types: 'push' and 'receive'. + + + tuf.FormatError, if the arguments are improperly formatted. + + tuf.Error, if there is an error processing the config contents. + + + The contents of 'filename' are read and stored. + + + A dictionary containing the data loaded from the configuration file. + + """ + + # Do the arguments have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(filename) + tuf.formats.NAME_SCHEMA.check_match(config_type) + + # RawConfigParser is used because unlike ConfigParser, + # it does not provide magical interpolation/expansion + # of variables (e.g., '%(option)s' would be ignored). + config = ConfigParser.RawConfigParser() + config.read(filename) + if config.sections() is None: + raise tuf.Error('Could not read '+repr(filename)) + + # Extract the relevant information from the config and build the + # 'config_dict' dictionary. + config_dict = {} + for section in config.sections(): + config_dict[section] = {} + for key, value in config.items(section): + # Split comma-separated entries and store them in a list. + # 'pushroots' is the only entry that currently accepts + # multiple values. + if key in ['pushroots']: + value = value.split(',') + config_dict[section][key] = value + + # Before returning a 'push' config dict, check the config is properly + # formatted, valid, and contains the required data. + if config_type == 'push': + # Ensure 'filename' is an appropriately named push config file. + if os.path.basename(filename) != PUSH_CONFIG: + message = repr(filename)+' is not a valid push config file.'+\ + ' The push config file should be named: '+repr(PUSH_CONFIG) + raise tuf.Error(message) + + # Retrieve the transfer module from the push config. The caller + # expects a valid config dict containing the required keys/values. + try: + transfer_module = config_dict['general']['transfer_module'] + except KeyError, e: + message = 'The push config file did not contain the required '+\ + '"transfer_module" entry under "[general]".' + raise tuf.Error(message) + + # Determine the transfer module and ensure the config file is properly + # formatted for an "scp configuration file". Raise 'tuf.FormatError' + # if there is mismatch. + if transfer_module == 'scp': + try: + tuf.formats.SCPCONFIG_SCHEMA.check_match(config_dict) + except tuf.FormatError, e: + message = repr(PUSH_CONFIG)+' rejected. '+str(e) + raise tuf.FormatError(message) + # A supported transfer module was not found. Raise 'tuf.Error'. + else: + message = 'The config file contains an invalid "transfer_module" entry '+\ + 'Supported transfer modules: '+repr(TRANSFER_MODULES) + raise tuf.Error(message) + + # Before returning a 'receive' config dict, check the config is properly + # formatted, valid, and contains the required data. + elif config_type == 'receive': + # Ensure 'filename' is an appropriately named receive config file. + if os.path.basename(filename) != RECEIVE_CONFIG: + message = repr(filename)+' is not a valid receive config file.'+\ + ' The receive config file should be named: '+repr(RECEIVE_CONFIG) + raise tuf.Error(message) + + # Determine if the config file is properly formatted for a "receive + # configuration file". Raise 'tuf.FormatError' if there is a + # mismatch. + try: + tuf.formats.RECEIVECONFIG_SCHEMA.check_match(config_dict) + except tuf.FormatError, e: + message = repr(RECEIVE_CONFIG)+' rejected. '+str(e) + raise tuf.FormatError(message) + + # Invalid 'config_type' requested. + else: + message = 'Invalid "config_type" argument. Supported: '+repr(CONFIG_TYPES) + raise tuf.Error(message) + + return config_dict diff --git a/src/tuf/pushtools/receivetools/receive.cfg.sample b/src/tuf/pushtools/receivetools/receive.cfg.sample new file mode 100755 index 00000000..c8d8b1f8 --- /dev/null +++ b/src/tuf/pushtools/receivetools/receive.cfg.sample @@ -0,0 +1,6 @@ +[general] +pushroots = /home/user/pushes,/home/user2/pushes +repository_directory = /var/tuf/test-repo +metadata_directory = /var/tuf/test-repo/metadata +targets_directory = /var/tuf/test-repo/targets +backup_directory = /var/tuf/test-repo/replaced diff --git a/src/tuf/pushtools/receivetools/receive.py b/src/tuf/pushtools/receivetools/receive.py index 9d720ae6..da7057f0 100755 --- a/src/tuf/pushtools/receivetools/receive.py +++ b/src/tuf/pushtools/receivetools/receive.py @@ -1,28 +1,45 @@ -#!/usr/bin/env python -# Copyright 2010 The Update Framework. See LICENSE for licensing information. """ -This script can be run on a repository to import new targets metadata and -target files into the repository. This is intended to work with the -developer push tools. When this script finds a new directory pushed -by a developer, it checks the metadata and target files and, if everything -is good, adds the files to the repository. + + receive.py -Usage: - ./receive.py + + Vladimir Diaz -Arguments: - None + + September 2012. Based on a previous version by Geremy Condra. -Details: + + See LICENSE for licensing information. -The script looks in a set of pre-defined push locations that one or more -developers may have uploaded files to using the push tools. If it finds -a valid push, it moves the push directory to a 'processing' directory and -also copies the pushed files to a temporary directory (these files are the -ones used by this script). + + This script can be run on a repository to import new targets metadata and + target files into the repository. This is intended to work with the + developer 'push.py' tool. When this script finds a new directory pushed + by a developer, it checks the metadata and target files and, if everything + is correct, adds the files to the repository. -Once the repository has received and copied a set of target files and the -corresponding targets metadata file, it performs the following checks that: + Like the 'push.py' script, 'receive.py' is provided as an optional tool for + maintainers who wish to support the remote updating of target files. The + target files are provided by an outside developer. The developer generates a + correctly signed 'targets.txt' metatada file, along with the target files + specified in it, and uploads them to his/her developer directory on the + repository with 'push.py'. A repository maintainer would then run this script + to ensure a valid targets metadata file is provided and the target files match + to what is listed. Lastly, the maintainer manually generates the new + 'release.txt' and 'timestamp.txt' metadata files so that clients may download + the newly added target files. + + + Details: + + The script looks in a set of pre-defined push locations that one or more + developers may have uploaded files to using the push tool. If it finds + a valid push, it moves the push directory to a 'processing' directory and + also copies the pushed files to a temporary directory (these files are the + ones used by this script). + + Once the repository has received and copied a set of target files and the + corresponding targets metadata file, it performs the following checks: * The metadata file is newer than the last metadata file of that type. * The metadata has not expired. @@ -31,60 +48,99 @@ * The target files described in the metadata are the same target files as were provided. -Once the verification is completed, the script backs up the files to be replaced -or obsoleted and then adds the new files to the repository. The script then -moves the push directory from the pushroot's 'processing' directory to its -'processed' directory and write a 'received.result' file to the push directory -that contains either the word SUCCESS or FAILURE. There may also be a -received.log file written, as well. The client can check these files to determine -whether the push was accepted and, if not, what the problem was. + Once the verification is complete, the script backs up the files to be + replaced or obsoleted, and then adds the new files to the repository. The + script then moves the push directory from the pushroot's 'processing' + directory to its 'processed' directory and writes a 'received.result' file + to the push directory. The 'received.result' file contains either the word + SUCCESS or FAILURE. There may also be a 'received.log' file written. The + client can check these files to determine whether the push was accepted and, + if not, what the problem was. -This script does not generate a new release.txt file or timestamp.txt file. -That needs to be done after this script runs if any pushes have been received. -In some cases, it may make sense to have this script operate on a non-live -copy of the repository and then rsync the files after all changes have been -made. + This script does not generate a new 'release.txt' file or 'timestamp.txt' file. + That needs to be done after this script runs if any pushes have been received. + In some cases, it may make sense to have this script operate on a non-live + copy of the repository and then rsync the files after all changes have been + made. -This script does not handle delegated targets metadata. When the time comes to -implement that here, care needs to be taken to ensure that a delegated targets -metadata file can't replace a target it shouldn't. Such untrusted files would -not trick clients, but they would prevent clients from obtaining updates. It -may be the case that making this script general enough to handle delegated -targets metadata may not be worth it. Such situations may be better suited to -customization per-project because the script could then leverage knowledge -about how the delegation is supposed to be done. + This script does not handle delegated targets metadata. When the time comes to + implement that here, care needs to be taken to ensure that a delegated targets + metadata file can't replace a target it shouldn't. Such untrusted files would + not trick clients, but they would prevent clients from obtaining updates. It + may be the case that making this script general enough to handle delegated + targets metadata may not be worth it. Such situations may be better suited to + customization per-project because the script could then leverage knowledge + about how the delegation is supposed to be done. -Example output of this script: + Usage: + + $ python receive.py --config + + Options: + + --config -$ python receivetools/receive.py -[2010-05-12 15:54:55,683] [tuf] [DEBUG] Looking for pushes in pushroot /tmp/tuf/test/pushes -[2010-05-12 15:54:55,684] [tuf] [INFO] Processing /tmp/tuf/test/pushes/1273704893.55 -[2010-05-12 15:54:55,684] [tuf] [DEBUG] Moving push directory to - /tmp/tuf/test/pushes/processing/1273704893.55 -[2010-05-12 15:54:55,693] [tuf] [DEBUG] Metadata timestamp is 2010-05-06 00:13:46 - (replacing metdata with timestamp 2010-05-06 00:13:46) -[2010-05-12 15:54:55,693] [tuf] [DEBUG] Metadata will expire at 2011-05-06 00:13:46 -[2010-05-12 15:54:55,693] [tuf] [DEBUG] Signatures: threshold: 1 / good: - [u'50792c6713637cf09e1aeb3805fc6d18f80d0a4f4ab7895f4a7cdf1abd7f5b0a'] / bad [] / - unrecognized: [] / unauthorized: [] / unknown method: [] -[2010-05-12 15:54:55,693] [tuf] [INFO] Number of targets specified: 1 -[2010-05-12 15:54:55,694] [tuf] [DEBUG] Size of target - /tmp/tuf/test/pushes/processing/1273704893.55/targets/test.txt is correct (5 bytes). -[2010-05-12 15:54:55,694] [tuf] [DEBUG] 1 hashes to check. -[2010-05-12 15:54:55,694] [tuf] [DEBUG] sha256 hash of target - /tmp/tuf/test/pushes/processing/1273704893.55/targets/test.txt is correct - (f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2). -[2010-05-12 15:54:55,694] [tuf] [INFO] Backing up target - /var/tuf/repo/targets/test.txt to /var/tuf/replaced/1273704893.55Tsr9QJ/targets/test.txt -[2010-05-12 15:54:55,695] [tuf] [INFO] Backing up old metadata - /var/tuf/repo/meta/targets.txt to /var/tuf/replaced/1273704893.55Tsr9QJ/targets.txt -[2010-05-12 15:54:55,695] [tuf] [INFO] Adding target to repo: /var/tuf/repo/targets/test.txt -[2010-05-12 15:54:55,695] [tuf] [INFO] Adding new targets metadata to repo: - /var/tuf/repo/meta/targets.txt -[2010-05-12 15:54:55,696] [tuf] [DEBUG] Moving push directory to - /tmp/tuf/test/pushes/processed/1273704893.55 -[2010-05-12 15:54:55,696] [tuf] [INFO] Completed processing of all push roots. - Push successes = 1, failures = 0. + --verbose <1-5> + + Example output of this script: + + $ python receive.py --verbose 1 --config ./receive.cfg + + + [2012-09-23 21:25:35,822] [tuf.receive] [DEBUG] Looking for pushes in pushroot + /home/user/pushes + [2012-09-23 21:25:35,822] [tuf.receive] [INFO] Processing /home/user/pushes/ + 1348449811.39 + [2012-09-23 21:25:35,822] [tuf.receive] [DEBUG] Moving push directory to + /home/user/pushes/processing/1348449811.39 + [2012-09-23 21:25:35,828] [tuf.receive] [DEBUG] New metadata timestamp is + 2012-09-23 23:24:17. Replacing old metadata with timestamp 2012-09-23 23:14:19 + [2012-09-23 21:25:35,829] [tuf.receive] [DEBUG] Metadata will expire at + 2013-09-23 23:24:17 + [2012-09-23 21:25:35,834] [tuf.receive] [DEBUG] {'unknown_method_sigs': [], + 'untrusted_sigs': [], 'bad_sigs': [], 'threshold': 1, 'good_sigs': + [u'efed647da99d1759637a80d225fc18e1d2a778812dd753f2d98b0311f19f26a1'], + 'unknown_sigs': []} + [2012-09-23 21:25:35,834] [tuf.receive] [INFO] Number of targets specified: 3 + [2012-09-23 21:25:35,835] [tuf.receive] [DEBUG] Size of target + /tmp/tmpQr4P_j/push/targets/helloworld.py is correct (19 bytes). + [2012-09-23 21:25:35,835] [tuf.receive] [DEBUG] 1 hash(es) to check. + [2012-09-23 21:25:35,835] [tuf.receive] [DEBUG] sha256 hash of target + /tmp/tmpQr4P_j/push/targets/helloworld.py is correct (9df93f8cd91e085db74d88c + 788ed00c9b865370fd484884c8db077f979788376). + [2012-09-23 21:25:35,835] [tuf.receive] [DEBUG] Size of target /tmp/tmpQr4P_j + /push/targets/LICENSE is correct (12 bytes). + [2012-09-23 21:25:35,836] [tuf.receive] [DEBUG] 1 hash(es) to check. + [2012-09-23 21:25:35,836] [tuf.receive] [DEBUG] sha256 hash of target /tmp/tmp + Qr4P_j/push/targets/LICENSE is correct (f9f661288421a20acf49017975e51dd09a662b + 8e6b3ca5f676d9d1feb153986c). + [2012-09-23 21:25:35,836] [tuf.receive] [DEBUG] Size of target /tmp/tmpQr4P_j/ + push/targets/new_file.txt is correct (10 bytes). + [2012-09-23 21:25:35,836] [tuf.receive] [DEBUG] 1 hash(es) to check. + [2012-09-23 21:25:35,836] [tuf.receive] [DEBUG] sha256 hash of target /tmp/tmp + Qr4P_j/push/targets/new_file.txt is correct (f1fc221623f24cc1a31d972ddba368481 + dd03b8bb124632fef78544342797215). + [2012-09-23 21:25:35,837] [tuf.receive] [INFO] Backing up target /var/tuf/test + -repo/targets/helloworld.py to /var/tuf/test-repo/replaced/1348449811.39aicLFk + /targets/helloworld.py + [2012-09-23 21:25:35,837] [tuf.receive] [INFO] Backing up target /var/tuf/test- + repo/targets/LICENSE to /var/tuf/test-repo/replaced/1348449811.39aicLFk/target + s/LICENSE + [2012-09-23 21:25:35,837] [tuf.receive] [INFO] Backing up old metadata /var/tuf + /src/tuf/test-repo/metadata/targets.txt to /var/tuf/test-repo/replaced/13484498 + 11.39aicLFk/targets.txt + [2012-09-23 21:25:35,838] [tuf.receive] [INFO] Adding target to repository: /va + r/tuf/test-repo/targets/helloworld.py + [2012-09-23 21:25:35,838] [tuf.receive] [INFO] Adding target to repository: /va + r/tuf/test-repo/targets/LICENSE + [2012-09-23 21:25:35,839] [tuf.receive] [INFO] Adding target to repository: /va + r/tuf/test-repo/targets/new_file.txt + [2012-09-23 21:25:35,839] [tuf.receive] [INFO] Adding new targets metadata to + repository: /var/tuf/test-repo/metadata/targets.txt + [2012-09-23 21:25:35,840] [tuf.receive] [DEBUG] Moving push directory to /home + /user/pushes/processed/1348449811.39 + [2012-09-23 21:25:35,840] [tuf.receive] [INFO] Completed processing of all pus + hes. Push successes = 1, failures = 0. """ @@ -94,307 +150,547 @@ import sys import tempfile import time +import logging +import optparse import tuf.formats -import tuf.hash import tuf.keydb -import tuf.log +import tuf.roledb import tuf.sig +import tuf.hash +import tuf.util +import tuf.log +import tuf.pushtools.pushtoolslib -logger = tuf.log.get_logger() +# See 'log.py' to learn how logging is handled in TUF. +logger = logging.getLogger('tuf.receive') -# These are the locations where developers may push files to using the -# push tools. Each push will be in its own directory with the -# developer's push root. Each developer's push either must have the -# directories 'processed' and 'processing' which are writable by this -# script. -PUSHROOTS = ['/home/SOMEUSER/pushes'] +def receive(config_filepath): + """ + + Locate and process the pushes found in any of the pushroots directories. + The pushroots are specified in the 'receive.cfg' configuration file. -# This is the directory where the repository resides. As far as this -# script is concerned, this is the live repository. Changes will be made -# directory to this repository. -REPODIR = '/var/tuf/repo' + pushroot + | + =============================================== + | | | | + processed processing 12345(push1) 54321(push2) -# This is the metadata directory within the repository. -METADIR = os.path.join(REPODIR, 'meta') + + config_filepath: + The receive configuration file (i.e., 'receive.cfg'). -# This is the targets directory within the repository. -TARGETSDIR = os.path.join(REPODIR, 'targets') + + tuf.FormatError, if any of the arguments are incorrectly formatted. -# Where replaced files will be stored. This will be used globally rather -# than a separate backup/replaced files directory for each pushroot. -BACKUPDIR = '/var/tuf/replaced' + tuf.Error, if there was error processing the receive. + + If a push is processed successfully, the repository specified in the + configuration file is updated with new target files and a 'targets.txt' + metadata file. -def run(): - """Look for and process pushes found in any PUSHROOTS.""" - successcount = 0 - failurecount = 0 - for pushroot in PUSHROOTS: - if not os.path.exists(pushroot): - logger.error("The pushroot %s does not exist. Skipping." % pushroot) - continue - logger.debug('Looking for pushes in pushroot %s' % pushroot) - if not os.path.exists(os.path.join(pushroot, 'processed')): - os.mkdir(os.path.join(pushroot), 'processed') - if not os.path.exists(os.path.join(pushroot, 'processing')): - os.mkdir(os.path.join(pushroot), 'processing') - # TODO: use only the newest push and move the others to the 'processed' - # directory, adding an appropriate log file. - for name in os.listdir(pushroot): - pushpath = os.path.join(pushroot, name) - if name == 'processed' or name == 'processing': - continue - if os.path.isdir(pushpath): - if not os.path.exists(os.path.join(pushpath, 'info')): - logger.warn("Skipping incomplete push %s (no info file)." - % pushpath) - continue - success = process_new_push(pushroot, name) - if success: - successcount += 1 - else: - failurecount += 1 - logger.info("Completed processing of all push roots. " - "Push successes = %s, failures = %s." % - (successcount, failurecount)) + + None. + """ -def process_old_push(pushroot, pushname): - """When there are multiple pushes, only the newest is used. All of the - older ones are ignored. This function makes the appropriate logs for - an old push and moves it into the 'processed' directory.""" - raise NotImplementedError + # Do the arguments have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.PATH_SCHEMA.check_match(config_filepath) + # Save a reference to the 'tuf.pushtools.pushtoolslib' module + # to avoid long lines of code. 'pushtoolslib' is needed here + # to read the 'receive.cfg' configuration file. + pushtoolslib = tuf.pushtools.pushtoolslib -def append_to_receive_log(pushpath, msg): - """Appends msg to [pushpath]/receive.log""" - try: - fp = open(os.path.join(pushpath, 'receive.log'), 'a') - except IOError, e: - raise tuf.Error('Unable to open receive log file: %s' % e) - try: - fp.write(msg) - fp.write('\n') - finally: - fp.close() + # Is the path to the configuration file valid? + if not os.path.isfile(config_filepath): + message = 'The configuration file path is invalid.' + raise tuf.Error(message) + config_filepath = os.path.abspath(config_filepath) + # Retrieve the configuration settings required by 'receive'. + # Raise ('tuf.FormatError', 'tuf.Error') if a valid configuration file + # cannot be retrieved. + config_dict = pushtoolslib.read_config_file(config_filepath, 'receive') + + # These are the locations where developers may push files using the + # push tools. Each push will be in its own directory with the + # developer's push root. Each developer's push must have the directories + # 'processed' and 'processing' writable by this script. + pushroots = config_dict['general']['pushroots'] -def record_receive_result(pushpath, success): - """Writes the [pushpath]/receive.result file that indicates SUCCESS or - FAILURE.""" - try: - fp = open(os.path.join(pushpath, 'receive.result'), 'w') - except IOError, e: - raise tuf.Error('Unable to open receive result file: %s' % e) - try: + # This is the directory where the repository resides. As far as this + # script is concerned, this is the live repository. Changes will be made + # to this repository directory. + repository_directory = config_dict['general']['repository_directory'] + repository_directory = os.path.expanduser(repository_directory) + + # This is the metadata directory within the repository. + # The successfully processed 'targets.txt' metadata file is saved here. + metadata_directory = config_dict['general']['metadata_directory'] + metadata_directory = os.path.expanduser(metadata_directory) + + # This is the targets directory within the repository. + # The successfully processed target files are saved here. + targets_directory = config_dict['general']['targets_directory'] + targets_directory = os.path.expanduser(targets_directory) + + # Where replaced files will be stored. This will be used globally rather + # than a separate backup/replaced files directory for each pushroot. + backup_directory = config_dict['general']['backup_directory'] + backup_directory = os.path.expanduser(backup_directory) + + # Check that the various defined directories exist. + # We don't check the 'pushroots' here because we consider it non-fatal + # if those are missing. A log message is issued if any of those are + # missing. + directories_to_check = {'repository': repository_directory, + 'metadata': metadata_directory, + 'targets': targets_directory, + 'backup': backup_directory} + + for directory_name, path in directories_to_check.items(): + if not os.path.exists(path): + message = directory_name+' directory does not exist: '+str(path) + logger.error(message) + raise tuf.Error(message) + + # Keep track of the number of pushes that were successfully processed, + # or that failed. These values are used to log/print detailed results + # after a pushroot is processed. + success_count = 0 + failure_count = 0 + + # Process all the pushes for each of the pushroots. + for pushroot in pushroots: + if not os.path.exists(pushroot): + logger.error('The pushroot '+str(pushroot)+' does not exist. Skipping.') + continue + + # Add the 'processed' and 'processing' directories if not present. + # These directories must exist so that we can properly process + # a push. + logger.debug('Looking for pushes in pushroot '+str(pushroot)) + if not os.path.exists(os.path.join(pushroot, 'processed')): + os.mkdir(os.path.join(pushroot, 'processed')) + if not os.path.exists(os.path.join(pushroot, 'processing')): + os.mkdir(os.path.join(pushroot, 'processing')) + + # Locate all the pushed directories and process them. 'pushname' + # should be a directory with a timestamp as its directory name. + # TODO: Use only the newest push and move the others to the 'processed' + # directory, adding an appropriate log file. + for pushname in os.listdir(pushroot): + # Skip over the 'processed' and 'processing' directories. + if pushname == 'processed' or pushname == 'processing': + continue + + # Found a directory we can potentially process. + pushpath = os.path.join(pushroot, pushname) + if os.path.isdir(pushpath): + # Ensure the 'info' file exists. A successful push operation creates + # and saves this 'info' file to the push directory. + if not os.path.exists(os.path.join(pushpath, 'info')): + message = 'Skipping incomplete push '+str(pushpath)+' (no info file).' + logger.warn(message) + continue + + # Process the new push and record if it was processed successfully. + # Raise 'tuf.Error' if a push processing error cannot be logged + # to a file properly. + success = _process_new_push(pushroot, pushname, metadata_directory, + targets_directory, backup_directory) if success: - fp.write("SUCCESS") + success_count += 1 else: - fp.write("FAILURE") - fp.write('\n') - finally: - fp.close() + failure_count += 1 + + # Done. Log the result of processing the pushes for 'pushroot'. + message = 'Completed processing of all pushes. Push successes = '+\ + str(success_count)+', failures = '+str(failure_count)+'.' + logger.info(message) -def process_new_push(pushroot, pushname): - """Process a push. + + + +def _process_new_push(pushroot, pushname, metadata_directory, + targets_directory, backup_directory): + """ + + Process a push. This will check the validity of targets metadata in the push (including whether the signatures are trusted) and, if valid, will copy the targets metadata and target files to the repository. + + + pushroot: + The root directory containing the developer's pushes. This root is one + of multiple directories listed under the 'pushroots' entry in the + 'receive.cfg' configuration file. + + pushname: + The name of the directory (i.e., '1348449811.39') containing the pushed + files. - Args: - pushroot: - pushname: - """ - logger.info("Processing %s/%s" % (pushroot, pushname)) + metadata_directory: + The directory where the repository's metadata files (e.g., 'targets.txt', + 'root.txt') are stored. - pushpath = os.path.join(pushroot, 'processing', pushname) - logger.debug("Moving push directory to %s" % pushpath) - os.rename(os.path.join(pushroot, pushname), pushpath) + targets_directory: + The directory where the repository's target files are stored. - # Copy the contents of pushpath to a temp directory. We don't want the - # user to be able to modify the files we work with. - tempdir = tempfile.mkdtemp() - pushtempdir = os.path.join(tempdir, 'push') - shutil.copytree(pushpath, pushtempdir) + backup_directory: + The directory where the pushed directories are saved after a + successful 'receive'. + + + tuf.Error, if a push processing error cannot be written to + 'receive.result'. + + Directories are created, the repository updated, and log files + added. + + + Boolean. True on success, False on failure. + + """ + + logger.info('Processing '+str(pushroot)+'/'+str(pushname)) + + # Move the pushed directory to the 'processing' directory. + pushpath = os.path.join(pushroot, 'processing', pushname) + logger.debug('Moving push directory to '+str(pushpath)) + if os.path.isdir(pushpath) or os.path.isfile(pushpath): + os.remove(pushpath) + os.rename(os.path.join(pushroot, pushname), pushpath) + + # Process 'pushpath' and log the appropriate results. + try: try: - try: - _process_copied_push(pushpath) - record_receive_result(pushpath, True) - return True - except (tuf.Error, OSError), e: - record_receive_result(pushpath, False) - append_to_receive_log(pushpath, str(e)) - logger.exception("Processing failed for push: %s/%s" % - (pushroot, pushname)) - return False - finally: - processedpath = os.path.join(pushroot, 'processed', pushname) - logger.debug("Moving push directory to %s" % processedpath) - os.rename(pushpath, processedpath) + # Raise 'tuf.Error' if the copied push cannot be properly processed. + _process_copied_push(pushpath, metadata_directory, + targets_directory, backup_directory) + # Write the '{pushpath}/receive.result' file that indicates SUCCESS. + # The developer may later read this file to quickly determine if + # the push was successfully processed. + try: + file_object = open(os.path.join(pushpath, 'receive.result'), 'w') + except IOError, e: + raise tuf.Error('Unable to open "receive.result" file: '+str(e)) + try: + file_object.write('SUCCESS') + file_object.write('\n') + finally: + file_object.close() + return True + + except tuf.Error, e: + # Write the '{pushpath}/receive.result' file that indicates FAILURE. + try: + file_object = open(os.path.join(pushpath, 'receive.result'), 'w') + except IOError, e: + raise tuf.Error('Unable to open "receive.result" file: '+str(e)) + try: + file_object.write("FAILURE") + file_object.write('\n') + finally: + file_object.close() + + # Log the error message to {pushpath}/receive.log + # The developer may later search this log file for specific + # error messages on failed push attempts. + try: + file_object = open(os.path.join(pushpath, 'receive.log'), 'a') + except IOError, e: + raise tuf.Error('Unable to open receive log file: '+str(e)) + try: + file_object.write(str(e)) + file_object.write('\n') + finally: + file_object.close() + + message = 'Could not process: '+str(pushroot)+'/'+str(pushname) + logger.exception(message) + return False + + # On success or failure, move 'pushpath' to the processed directory. + finally: + processedpath = os.path.join(pushroot, 'processed', pushname) + logger.debug('Moving push directory to '+str(processedpath)) + if os.path.isdir(processedpath) or os.path.isfile(processedpath): + os.remove(processedpath) + os.rename(pushpath, processedpath) -def _process_copied_push(pushpath): - """Helper function for process_new_push. + + + +def _process_copied_push(pushpath, metadata_directory, + targets_directory, backup_directory): + """ + + Helper function for _process_new_push(). This does the actual work of copying pushpath to a temp directory, checking the metadata and targets, and copying the files to the repository on success. The push is valid and successfully processed if no exception is raised. + + + pushpath: + The push directory currently being processed (i.e., the 'processing' + directory on the developer's pushroot) - Raises: - OSError or tuf.Error. - """ - pushname = os.path.basename(pushpath) + metadata_directory: + The directory where the repository's metadata files (e.g., 'targets.txt', + 'root.txt') are stored. - # Read the metadata of the current repository. - rootmetapath = os.path.join(METADIR, 'root.txt') - root_json = tuf.util.load_json_file(rootmetapath) - root_meta = root_json['signed'] - root_obj = tuf.formats.RootFile.from_meta(root_meta) - keydb = tuf.keydb.KeyDB.create_from_root(root_obj) + targets_directory: + The directory where the repository's target files are stored. - # Determine the name of the targets metadata file that was pushed. - targetsmetafile = None - try: - fp = open(os.path.join(pushpath, 'info'), 'r') - except IOError, e: - raise tuf.Error('Unable to open push info file: %s' % e) - try: - for line in fp: - parts = line.strip().split('=') - if parts[0] == 'metadata': - if parts[1] != 'targets.txt': - raise NotImplementedError('No support yet for pushing ' + - 'delegated targets metadata.') - else: - targetsmetafile = parts[1] - break + backup_directory: + The directory where the pushed directories are saved after a + successful 'receive'. + + + tuf.Error, if there is an error processing the push. + + + The repository is updated if the push is successful. + + + None. + + """ + + # The push's timestamp directory name (e.g., '1348449811.39') + pushname = os.path.basename(pushpath) + + # Copy the contents of pushpath to a temp directory. We don't want the + # user modifying the files we work with. The temp directory is only + # accessible by the calling process. + temporary_directory = tempfile.mkdtemp() + push_temporary_directory = os.path.join(temporary_directory, 'push') + shutil.copytree(pushpath, push_temporary_directory) + + # Read the 'root' metadata of the current repository. 'root.txt' + # is needed to authorize the 'targets' metadata file. + root_metadatapath = os.path.join(metadata_directory, 'root.txt') + root_signable = tuf.util.load_json_file(root_metadatapath) + + # Ensure 'root_signable' is properly formatted. + try: + tuf.formats.check_signable_object_format(root_signable) + except tuf.FormatError, e: + raise tuf.Error('The repository contains an invalid "root.txt".') + + # Extract the metadata object and load the key and role databases. + # The keys and roles are needed to verify the signatures of the + # metadata files. + root_metadata = root_signable['signed'] + tuf.keydb.create_keydb_from_root_metadata(root_metadata) + tuf.roledb.create_roledb_from_root_metadata(root_metadata) + + # Determine the name of the targets metadata file that was pushed. + # The required 'info' file should list the metadata file that was + # pushed by the developer. Only 'targets.txt' currently supported + # (i.e., no delegated roles are accepted). + new_targets_metadata_file = None + try: + file_object = open(os.path.join(push_temporary_directory, 'info'), 'r') + except IOError, e: + raise tuf.Error('Unable to open push "info" file: '+str(e)) + try: + # Inspect each line of the 'info' file, searching for the line that + # specifies the targets metadata file. Raise an exception if all + # the lines are processed without finding the 'metadata=' line. + for line in file_object: + # Search 'info' for a 'metadata=.../targets.txt' line. + parts = line.strip().split('=') + if parts[0] == 'metadata': + metadata_basename = os.path.basename(parts[1]) + if metadata_basename != 'targets.txt': + message = 'No support yet for pushing delegated targets metadata.' + raise tuf.Error(message) else: - raise tuf.Error('No metadata= line in push info file.') - finally: - fp.close() - - # Read the new metadata that was pushed. - targetsmetapath = os.path.join(pushpath, targetsmetafile) - targets_json = tuf.util.load_json_file(targetsmetapath) - - # Read the existing metadata from the repository. - repotargetsmetapath = os.path.join(METADIR, targetsmetafile) - - # Check the metadata. This is mostly to make sure we don't replace good - # metadata with bad metadata as clients do their own security checking. - # This is what we check: - # * it is newer than the last metadata. - # * it has not expired. - # * all signatures valid. - # * a threshold of trusted signatures. only check the delegating - # role rather than the trust hierachy all the way up. - # * all of the files listed in the metadata were provided and have - # the sizes and hashes listed in the metadata. - - # Check that the new metadata is newer than the existing metadata. - if os.path.exists(repotargetsmetapath): - repo_targets_json = tuf.util.load_json_file(targetsmetapath) - cur_timestamp_string = repo_targets_json['signed']['ts'] - cur_meta_timestamp = tuf.formats.parse_time(cur_timestamp_string) - - new_timestamp_string = targets_json['signed']['ts'] - new_meta_timestamp = tuf.formats.parse_time(new_timestamp_string) - - # Allowing equality makes testing/development easier. - if cur_meta_timestamp > new_meta_timestamp: - raise tuf.Error("Existing metadata timestamp (%s) is newer than " - "the new metadata's timestamp (%s)" % - (cur_timestamp_string, new_timestamp_string)) - else: - logger.debug('Metadata timestamp is %s (replacing metdata with ' - 'timestamp %s)' % - (new_timestamp_string, cur_timestamp_string)) - + new_targets_metadata_file = parts[1] + break else: - logger.warn("The old targets metadata file %s doesn't exist in " - "the repo. Skipping timestamp check." % - repotargetsmetapath) + raise tuf.Error('No "metadata=" line in push info file.') + finally: + file_object.close() - # Ensure the metadata is not expired. - expiration_string = repo_targets_json['signed']['expires'] - expiration_timestamp = tuf.formats.parse_time(expiration_string) - if expiration_timestamp <= time.time(): - raise tuf.Error("Pushed metadata expired at %s" % expiration_string) + # Read the new targets metadata that was pushed. + new_targets_metadatapath = os.path.join(push_temporary_directory, + new_targets_metadata_file) + new_targets_signable = tuf.util.load_json_file(new_targets_metadatapath) + + # Ensure 'new_targets_signable' is properly formatted. + try: + tuf.formats.check_signable_object_format(new_targets_signable) + except tuf.FormatError, e: + raise tuf.Error('The pushed targets metadata file is invalid.') + + # Read the existing targets metadata from the repository. + targets_metadatapath = os.path.join(metadata_directory, 'targets.txt') + + # Check the metadata. This is mostly to make sure we don't replace good + # metadata with bad metadata as clients do their own security checking. + # This is what we check: + # * it is newer than the last metadata. + # * it has not expired. + # * all signatures valid. + # * a threshold of trusted signatures. only check the delegating + # role rather than the trust hierachy all the way up. + # * all of the files listed in the metadata were provided and have + # the sizes and hashes listed in the metadata. + + # Check that the new metadata file is newer than the existing metadata. + if os.path.exists(targets_metadatapath): + targets_signable = tuf.util.load_json_file(targets_metadatapath) + + # Ensure 'targets_signable' is properly formatted. + try: + tuf.formats.check_signable_object_format(targets_signable) + except tuf.FormatError, e: + raise tuf.Error('The repository\'s targets metadata file is invalid.') + + # Extract the timestamp value of the current targets metadata. + # This value is used to determine if the new metadata is newer. + timestamp = targets_signable['signed']['ts'] + formatted_timestamp = tuf.formats.parse_time(timestamp) + + # Extract the timestamp of the new targets metadata. + new_timestamp = new_targets_signable['signed']['ts'] + new_formatted_timestamp = tuf.formats.parse_time(new_timestamp) + + # Allowing equality makes testing/development easier. + if formatted_timestamp > new_formatted_timestamp: + message = 'Existing metadata timestamp '+str(timestamp)+' is newer '+\ + 'than the new metadata\'s timestamp '+str(new_timestamp) + raise tuf.Error(message) else: - logger.debug('Metadata will expire at %s' % expiration_string) + message = 'New metadata timestamp is '+str(new_timestamp)+'. '+\ + ' Replacing old metadata with timestamp '+str(timestamp) + logger.debug(message) - # This raises tuf.BadSignature if the check fails. - status = tuf.sig.check_signatures(targets_json, keydb, role='targets') - logger.debug('Signatures: %s' % status) + # There appears to be no 'targets.txt' metadata file on the repository. + else: + message = 'The old targets metadata file '+str(targets_metadatapath)+'. '+\ + 'doesn\'t exist in the repo. Skipping the timestamp check.' + logger.warn(message) - logger.info("Number of targets specified: %s" % - len(targets_json['signed']['targets'].keys())) + # Ensure the new metadata is not expired. + expiration = new_targets_signable['signed']['expires'] + formatted_expiration = tuf.formats.parse_time(expiration) + + if formatted_expiration <= time.time(): + message = 'Pushed metadata expired at '+str(expiration) + raise tuf.Error(message) + else: + message = 'Metadata will expire at '+str(expiration) + logger.debug(message) - for targetrelpath, targetinfo in targets_json['signed']['targets'].items(): - targetpath = os.path.join(pushpath, 'targets', targetrelpath) + # Verify the signatures of the new targets metadata. + if not tuf.sig.verify(new_targets_signable, 'targets'): + message = 'The pushed targets metadata file does not '+\ + 'have the required number of good signatures.' + raise tuf.Error(message) + # Log the status of the signatures. For example, the number of good, + # bad, untrusted, unknown, signatures. + status = tuf.sig.get_signature_status(new_targets_signable, 'targets') + logger.debug(str(status)) - # Check that the target was provided. - if not os.path.exists(targetpath): - raise tuf.Error('The specified target file was not provided: %s', - targetrelpath) + # Log the number of targets specified in the new targets metadata file. + targets_count = len(new_targets_signable['signed']['targets'].keys()) + message = 'Number of targets specified: '+str(targets_count) + logger.info(message) - # Check size. - actualsize = os.path.getsize(targetpath) - if actualsize != targetinfo['length']: - raise tuf.Error('The size of target file %s is incorrect: ' + - 'was %s, expected %s' % (targetrelpath, actualsize, - targetinfo['length'])) - else: - logger.debug('Size of target %s is correct (%s bytes).' % - (targetpath, actualsize)) + # Verify the files of the new targets metadata file. + new_targets_dict = new_targets_signable['signed']['targets'] + for target_relativepath, target_info in new_targets_dict.items(): + targets_basename = os.path.basename(targets_directory) + targetpath = os.path.join(push_temporary_directory, targets_basename, + target_relativepath) + + # Check that the target was provided. + if not os.path.exists(targetpath): + message = 'The specified target file was not provided: '+\ + str(target_relativepath) + raise tuf.Error(message) - # Check hashes. - hashcount = len(targetinfo['hashes'].items()) - if hashcount == 0: - raise tuf.Error('Empty hashes dictionary.') - else: - logger.debug('%s hashes to check.' % hashcount) - for hashalg, hashval in targetinfo['hashes'].items(): - d_obj = tuf.hash.Digest(hashalg) - d_obj.update_filename(targetpath) - if d_obj.format() != hashval: - raise tuf.Error('%s hash does not match: was %s, expected %s' % - (hashalg, d_obj.format(), hashval)) - else: - logger.debug('%s hash of target %s is correct (%s).' % - (hashalg, targetpath, hashval)) + # Check the target's size. A valid size is required of target files. + target_size = os.path.getsize(targetpath) + if target_size != target_info['length']: + message = 'The size of target file '+str(target_relativepath)+\ + ' is incorrect: was '+str(target_size)+', expected '+\ + str(target_info['length']) + raise tuf.Error(message) + else: + message = 'Size of target '+str(targetpath)+' is correct '+\ + '('+str(target_size)+' bytes).' + logger.debug(message) - # At this point, the targets metadata and all specified files have been - # verified. + # Check hashes. Valid target files is required. + hash_count = len(target_info['hashes'].items()) + if hash_count == 0: + message = str(targetpath)+' contains an empty hashes dictionary.' + raise tuf.Error(message) + else: + logger.debug(str(hash_count)+' hash(es) to check.') + + for algorithm, digest in target_info['hashes'].items(): + digest_object = tuf.hash.digest_filename(targetpath, algorithm=algorithm) + if digest_object.hexdigest() != digest: + message = str(algorithm)+' hash does not match: '+\ + ' was '+str(digest_object.hexdigest())+', expected '+\ + str(digest) + raise tuf.Error(message) + else: + message = str(algorithm)+' hash of target '+str(targetpath)+\ + ' is correct ('+str(digest)+').' + logger.debug(message) - # Remove the files referenced by the old targets metadata as well as the - # old targets metadata itself. - _remove_old_files(repotargetsmetapath, pushname) + # At this point, the targets metadata and all specified files have been + # verified. Remove the files referenced by the old targets metadata as + # well as the old targets metadata itself. + # Raise 'tuf.Error' if there is an error backing up the old targets. + _remove_old_files(targets_metadatapath, pushname, + targets_directory, backup_directory) - # Copy the new target files into place on the repository. - for targetrelpath in targets_json['signed']['targets'].keys(): - srcpath = os.path.join(pushpath, 'targets', targetrelpath) - destpath = os.path.join(TARGETSDIR, targetrelpath) - logger.info("Adding target to repo: %s" % destpath) - destdir = os.path.dirname(destpath) - if not os.path.exists(destdir): - os.mkdir(destdir) - shutil.copy(srcpath, destpath) + # Copy the new target files into place on the repository. + for target_relativepath in new_targets_signable['signed']['targets'].keys(): + targets_basename = os.path.basename(targets_directory) + source_path = os.path.join(push_temporary_directory, targets_basename, + target_relativepath) + destination_path = os.path.join(targets_directory, target_relativepath) + logger.info('Adding target to repository: '+str(destination_path)) + destination_directory = os.path.dirname(destination_path) + if not os.path.exists(destination_directory): + os.mkdir(destination_directory) + shutil.copy(source_path, destination_path) - # Copy the targets metadata into place on the repository. - logger.info("Adding new targets metadata to repo: %s" % - repotargetsmetapath) - shutil.copy(targetsmetapath, repotargetsmetapath) + # Copy the new targets metadata file into place on the repository. + message = 'Adding new targets metadata to repository: '+str(targets_metadatapath) + logger.info(message) + shutil.copy(new_targets_metadatapath, targets_metadatapath) -def _remove_old_files(oldtargetsfile, pushname): - """Remove metadata and target files that will be replaced. + + + +def _remove_old_files(targets_metadatapath, pushname, + targets_directory, backup_directory): + """ + + Remove metadata and target files that will be replaced. This does not take into account any targets that are the same between the old and new metadata. For simplicity, all old targets are removed @@ -402,62 +698,161 @@ def _remove_old_files(oldtargetsfile, pushname): into place after this has been called. This function currently assumes that the the metadata file is the - top-level targets.txt file rather than a delegated metadata file. + top-level 'targets.txt' file rather than a delegated metadata file + and that the arguments have been validated (i.e., exist, correct, etc). - Args: - oldtargetsfile: The old targets metadata file that is to be - replaced, along with all of its referenced targets. - """ - if not os.path.exists(oldtargetsfile): - logger.warn("The old targets metadata file %s doesn't exist in " - "the repo. Skipping file backup." % oldtargetsfile) - return + + targets_metadatapath: + The old targets metadata file to be replaced, along with all of + its referenced targets. - backupdestdir = tempfile.mktemp(prefix=pushname, dir=BACKUPDIR) - os.mkdir(backupdestdir) - backuptargetsdir = os.path.join(backupdestdir, 'targets') - os.mkdir(backuptargetsdir) + pushname: + The name of the directory (i.e., timestamp name) containing the pushed + files. + + targets_directory: + The directory where the repository's target files are stored. - targets_json = tuf.util.load_json_file(oldtargetsfile) - for targetrelpath in targets_json['signed']['targets'].keys(): - curtargetpath = os.path.join(TARGETSDIR, targetrelpath) - baktargetpath = os.path.join(backuptargetsdir, targetrelpath) - logger.info("Backing up target %s to %s" % (curtargetpath, baktargetpath)) - if os.path.exists(curtargetpath): - mkdir_p(os.path.dirname(baktargetpath)) - os.rename(curtargetpath, baktargetpath) + backup_directory: + The directory where the pushed directories are saved to after a + successful 'receive'. + + + tuf.Error, if there is an error backing up the old targets. + + + Replaces the old 'targets.txt' metadata file and removes all of the old + target files. + + + None. + + """ + + # Create the backup destination directories. The old target files + # and target metadata are backed up to these directories. + backup_destdirectory = os.path.join(backup_directory, pushname) + os.mkdir(backup_destdirectory) + targets_basename = os.path.basename(targets_directory) + backup_targetsdirectory = os.path.join(backup_destdirectory, targets_basename) + os.mkdir(backup_targetsdirectory) + + # Load the old 'targets.txt' file and determine all the targets to be replaced. + # Need to ensure we only remove target files specified by 'targets.txt'. + targets_signable = tuf.util.load_json_file(targets_metadatapath) + for target_relativepath in targets_signable['signed']['targets'].keys(): + targetpath = os.path.join(targets_directory, target_relativepath) + backup_targetpath = os.path.join(backup_targetsdirectory, target_relativepath) + message = 'Backing up target '+str(targetpath)+' to '+str(backup_targetpath) + logger.info(message) + + # Move the old target file to the backup directory. Create any + # directories along the way. + if os.path.exists(targetpath): + try: + os.makedirs(os.path.dirname(backup_targetpath)) + except OSError, e: + if e.errno == errno.EEXIST: + pass else: - logger.warn("The old target %s doesn't exist in the repo." % - curtargetpath) + raise tuf.Error(str(e)) + os.rename(targetpath, backup_targetpath) + else: + message = 'The old target '+str(targetpath)+' doesn\'t exist in the repo.' + logger.warn(message) - baktargetsmetafile = os.path.join(backupdestdir, 'targets.txt') - logger.info("Backing up old metadata %s to %s" % (oldtargetsfile, - baktargetsmetafile)) - os.rename(oldtargetsfile, baktargetsmetafile) + # Backup the old 'targets.txt' metadata file. + backup_targets_metadatafile = os.path.join(backup_destdirectory, 'targets.txt') + message = 'Backing up old metadata '+str(targets_metadatapath)+\ + ' to '+str(backup_targets_metadatafile) + logger.info(message) + if os.path.isfile(backup_targets_metadatafile): + os.remove(backup_targets_metadatafile) + os.rename(targets_metadatapath, backup_targets_metadatafile) -def mkdir_p(path): - try: - os.makedirs(path) - except OSError, err: - if err.errno == errno.EEXIST: - pass - else: raise -def _check_directories_exist(): - """Check that the various defined directories exist.""" - # We don't check the PUSHROOTS here because we consider it non-fatal - # if those are missing. A log message is issued if any of those are - # missing. - dirs_to_check = {'REPODIR':REPODIR, 'METADIR':METADIR, - 'TARGETSDIR':TARGETSDIR, 'BACKUPDIR':BACKUPDIR} - for name, path in dirs_to_check.items(): - if not os.path.exists(path): - logger.error("%s directory does not exist: %s" % (name, path)) - sys.exit(1) + +def parse_options(): + """ + + Parse the command-line options. 'receive.py' expects the '--config' + option to be set by the user. + + Example: + $ python receive.py --config ./receive.cfg + + The '--config' option accepts a path argument to the receive configuration + file (i.e., 'receive.cfg'). If the required option is unset, a parser error + is printed and the script exits. + + The '--verbose' option sets the verbosity level of the TUF logger. Accepts + values 1-5. + + + None. + + + None. + + + Sets the logging level of the TUF logger. + + + The options object returned by the parser's parse_args() method. + + """ + + usage = 'usage: %prog --config ' + option_parser = optparse.OptionParser(usage=usage) + + # Add the options supported by 'receive' to the option parser. + option_parser.add_option('--config', action='store', type='string', + help='Specify the "receive.cfg" configuration file.') + + option_parser.add_option('--verbose', dest='VERBOSE', type=int, default=2, + help='Set the verbosity level (1-5) of logging ' + 'messages. The lower the setting, the greater the ' + 'verbosity.') + + (options, remaining_arguments) = option_parser.parse_args() + + # Ensure the '--config' option is set. If the required option is unset, + # option_parser.error() will print an error message and exit. + if options.config is None: + message = '"--config" must be set on the command-line.' + option_parser.error(message) + + # Set the logging level. + if options.VERBOSE == 5: + tuf.log.set_log_level(logging.CRITICAL) + elif options.VERBOSE == 4: + tuf.log.set_log_level(logging.ERROR) + elif options.VERBOSE == 3: + tuf.log.set_log_level(logging.WARNING) + elif options.VERBOSE == 2: + tuf.log.set_log_level(logging.INFO) + elif options.VERBOSE == 1: + tuf.log.set_log_level(logging.DEBUG) + else: + tuf.log.set_log_level(logging.NOTSET) + + return options -if __name__ == "__main__": - _check_directories_exist() - run() + + + +if __name__ == '__main__': + options = parse_options() + + # Perform a 'receive' of the pushroots specified in the configuration file. + try: + receive(options.config) + except (tuf.FormatError, tuf.Error), e: + sys.stderr.write('Error: '+str(e)+'\n') + sys.exit(1) + + # The 'receive' and command-line options were processed successfully. + sys.exit(0) diff --git a/src/tuf/pushtools/transfer/scp.py b/src/tuf/pushtools/transfer/scp.py index fea0c0b3..c844f4e9 100755 --- a/src/tuf/pushtools/transfer/scp.py +++ b/src/tuf/pushtools/transfer/scp.py @@ -1,35 +1,51 @@ -# Copyright 2010 The Update Framework. See LICENSE for licensing information. """ -SCP transfer module for the developer push mechanism. + + scp.py -This will use scp to upload a push directory to the repository. The directory -will be named with the current timestamp in the format XXXXXXXXXX.XX. The -directory will contain a file named 'info' that provides information about -the push, the signed metadata file, and a 'targets' directory that contains -the targets specified in the metadata. + + Vladimir Diaz -Use of this module requires the following section to be present in the push -configuration file provided to push.py: + + August 2012. Based on a previous version of this module by Geremy Condra. -[scp] -host = somehost -user = someuser -identity_file = optional_path_to_ssh_key -remote_dir = ~/pushes + + See LICENSE for licensing information. -The remote_dir should correspond to a pushroot configured in the repository's -receive.py script. + + SCP (secure copy) transfer module for the developer push mechanism. -This transfer module will output to stdout the commands it runs and the output -of those commands. + This will use scp to upload a push directory to the repository. The directory + will be named with the current timestamp in the format XXXXXXXXXX.XX. The + directory will contain a file named 'info' that provides information about + the push, the signed metadata file, and a 'targets' directory that contains + the targets specified in the metadata. -Example: + Use of this module requires the following section to be present in the push + configuration file provided to 'push.py': + + [scp] + host = host + user = user + identity_file = optional_path_to_ssh_key + remote_directory = ~/pushes + + The 'remote_directory' should correspond to a pushroot configured in the + repository's 'receive.py' configuration file. + + This transfer module will output to stdout the commands it runs and the output + of those commands. + + Example: + + $ python pushtools/push.py --config ./push.cfg + + Running command: scp -r /tmp/tmpXi0GZH user@host:~/pushes/1348352878.31 + + helloworld.py 100% 13 0.0KB/s 00:00 + LICENSE 100% 12 0.0KB/s 00:00 + targets.txt 100% 7 0.0KB/s 00:00 + info 100% 32 0.0KB/s 00:00 -$ python pushtools/push.py push push.cfg test.txt -Running command: scp -r /tmp/tmpc8PiXo somehost:~/test/pushes/1273704893.55 -info 100% 21 0.0KB/s 00:00 -targets.txt 100% 771 0.8KB/s 00:00 -test.txt 100% 5 0.0KB/s 00:00 """ import os @@ -38,75 +54,118 @@ import tempfile import time +import tuf.formats -class TransferContext(object): - def __init__(self, config): - self.host = config['host'] - self.user = config.get('user') - self.identity_file = config.get('identity_file') - self.remote_dir = config.get('remote_dir', '.') +def transfer(scp_config_dict): + """ + + Create a local temporary directory with an added 'info' file used to + communicate additional information to the repository. This directory + will be transferred to the repository. + + + scp_config_dict: + The dict containing the options to use with the SCP command. - def transfer(self, target_paths, metadata_path): - """ - Create a local temporary directory with an additional file used to - communicate additional information to the repository. This directory - will be transferred to the repository. - """ + + tuf.FormatError, if the arguments are improperly formatted. - basecommand = ['scp'] - if self.identity_file: - basecommand.extend(['-i', self.identity_file]) + tuf.Error, if the transfer failed. - timestamp = time.time() - dest = "" - if self.user: - dest += "%s@" - dest += "%s:%s/%s" % (self.host, self.remote_dir, timestamp) + + Files specified in 'push.cfg' will be transfered to a host using + 'scp'. + + + None. + + """ - tempdir = tempfile.mkdtemp() - try: - # Make sure the temp directory is world-readable as the permissions - # get carried over in the scp'ing. - os.chmod(tempdir, 0755) + # Do the arguments have the correct format? + # Raise 'tuf.FormatError' if there is a mismatch. + tuf.formats.SCPCONFIG_SCHEMA.check_match(scp_config_dict) + + # Extract the required 'scp' entries. If an entry contains + # a path argument, Tilde Expansions or user home symbols + # are converted. + host = scp_config_dict['scp']['host'] + user = scp_config_dict['scp']['user'] + + # The SCP command accepts an optional path to an SSH private key file. + identity_file = scp_config_dict['scp']['identity_file'] + identity_file = os.path.expanduser(identity_file) + + # The directory on the host the target files will be pushed to. + remote_directory = scp_config_dict['scp'].get('remote_directory', '.') + remote_directory = os.path.expanduser(remote_directory) + + # The 'targets.txt' metadata file to be pushed to the host. + metadata_path = scp_config_dict['general']['metadata_path'] + metadata_path = os.path.expanduser(metadata_path) + + # The local targets directory containing the target to be pushed. + targets_directory = scp_config_dict['general']['targets_directory'] + targets_directory = os.path.expanduser(targets_directory) + + basecommand = ['scp'] + if identity_file: + basecommand.extend(['-i', identity_file]) - # Create a file that tells the repository the name of the targets - # metadata file. For delegation, this will be the only way the - # the repository knows the full role name. - fp = open(os.path.join(tempdir, 'info'), 'w') - fp.write("metadata=%s\n" % metadata_path) - fp.close() + # Build the destination. + # Example: 'user@localhost:~/pushes/1273704893.55' + timestamp = time.time() + destination = '' + if user: + destination = destination+user+'@' + destination = destination+host+':'+remote_directory+'/'+str(timestamp) - # Copy the metadata. - basename = os.path.basename(metadata_path) - shutil.copy(metadata_path, os.path.join(tempdir, basename)) + temporary_directory = tempfile.mkdtemp() + try: + # Make sure the temp directory is world-readable, as the permissions + # get carried over in the scp'ing. + os.chmod(temporary_directory, 0755) - # Create a directory that all target files will be put in before - # being transferred. - targetsdir = os.path.join(tempdir, 'targets') - os.mkdir(targetsdir) + # Create a file that tells the repository the name of the targets + # metadata file. For delegation, this will be the only way the + # the repository knows the full role name. + file_object = open(os.path.join(temporary_directory, 'info'), 'w') + file_object.write('metadata='+metadata_path+'\n') + file_object.close() - # This is quite inefficient for large files, but just copy all - # targets into the correct directory structure. - for path in target_paths: - dirname = os.path.dirname(path) - basename = os.path.basename(path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - shutil.copy(path, os.path.join(targetsdir, basename)) + # Copy the targets metadata. + basename = os.path.basename(metadata_path) + shutil.copy(metadata_path, os.path.join(temporary_directory, basename)) - # This will create the 'timestamp' directory on the remote host and - # it will contain the info file and an empty targets directory. - command = basecommand[:] - command.append('-r') # recursive - command.append(tempdir) - command.append(dest) - print "Running command: %s" % ' '.join(command) - # Raises subprocess.CalledProcessError on failure. - subprocess.check_call(command) + # Create a directory that all target files will be put in before + # being transferred. + temporary_targets_directory = os.path.join(temporary_directory, 'targets') - finally: - shutil.rmtree(tempdir) + # Copy all the targets into the correct directory structure. + shutil.copytree(targets_directory, temporary_targets_directory) - def finalize(self): - pass + # This will create the 'timestamp' directory on the remote host. The + # 'timestamp' directory will contain the 'info' file, targets metadata, + # and the targets directory being pushed. + command = basecommand[:] + # Add the recursive option, which will add the full contents of + # 'temporary_directory' + command.append('-r') + command.append(temporary_directory) + command.append(destination) + # Example 'command': + # ['scp', '-i', '/home/user/.ssh/id_dsa', '-r', '/tmp/tmpmxWxLS', + # 'user@host:~/pushes/1348349228.4'] + print 'Running command: '+' '.join(command) + + # 'subprocess.CalledProcessError' raised on scp command failure. + # Catch the exception and raise 'tuf.Error'. + # For important security information on 'subprocess', + # See http://docs.python.org/library/subprocess.html + try: + subprocess.check_call(command) + except subprocess.CalledProcessError, e: + message = 'scp.transfer failed.' + raise tuf.Error(message) + finally: + shutil.rmtree(temporary_directory)