Elgato_dark/python-tuf
1
0
Fork 0
mirror of https://github.com/theupdateframework/python-tuf synced 2026-05-24 10:08:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adding the refactored pushtools modules.

Browse source
This commit is contained in:
vladdd 2013-02-10 13:35:07 -05:00
parent b4993302a5
commit af8b22dd95
7 changed files with 1211 additions and 521 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
src/tuf/pushtools/__init__.py Normal file → Executable file
View file

12
src/tuf/pushtools/push.cfg.sample
View file

@ -1,10 +1,10 @@
[general]
transfer_module = scp
metadata_path = targets.txt
metadata_path = /var/tuf/test-repo/metadata/targets.txt
targets_directory = /var/tuf/test-repo/targets
[scp]
host =
user =
identity_file =
# remote_dir must be similarly configured on the repository side.
remote_dir = ~/test/pushes
host = localhost
user = user
identity_file = ~/.ssh/id_rsa
remote_directory = ~/pushes

224
src/tuf/pushtools/push.py
View file

@ -1,104 +1,184 @@
#!/usr/bin/env python
# Copyright 2010 The Update Framework. See LICENSE for licensing information.
"""
This script provides a way for developers to push a signed targets metadata
file and the referenced targets to a repository. The repository adds these
files to the repository by running the receivetools/receive.py script.
<Program Name>
push.py
Usage:
./push.py COMMAND COMMAND_ARGS
<Author>
Vladimir Diaz <vladimir.v.diaz@gmail.com>
<Started>
August 2012. Based on a previous version by Geremy Condra.
<Copyright>
See LICENSE for licensing information.
<Purpose>
This script provides a way for developers to push a signed targets metadata
file (i.e., 'targets.txt') and the referenced targets to a repository. The
repository adds these files to the repository by running the
'tuf/pushtools/receivetools/receive.py' script.
'push.py' is not a required module of The Update Framework, but is provided
to allow developers to remotely update the target files served by a
repository. The actual file transfers are completed by a separate command
available on the client machine. The 'SCP' (secure copy) command is currently
supported. This script may be viewed as a front-end to the python transfer
modules (e.g., 'tuf.pushtools.transfer.scp'). A configuration file can be
specified allowing the user to customize the transfer and supply the locations
of targets and metadata.
Usage:
$ python push.py --config <config path>
Known commands:
push
Example:
$ python push.py --config ./push.cfg
Example:
./push.py push push.cfg targets.txt targetfile1 targetfile2
Details of the 'push.py' script:
Details of 'push' command:
The developer provides the path to a configuration file that lists:
The developer provides the path to a configuration file that lists:
* The path to the targets metadata file.
* The name of the transfer module to use for transferring the
files to the repository (e.g. 'scp').
files to the repository (e.g., 'scp').
* Configuration information that is specific to the transfer
module.
See the push.cfg.sample file for an example configuration file.
See the 'push.cfg.sample' file for an example configuration file.
The transfer module needs the following functionality:
The transfer module needs the following functionality:
* A way to transfer target files and the new metadata file to the
repository.
repository. The 'scp' transfer modules is currently supported.
The transfer module may also include the following functionality:
The transfer module may also include the following functionality:
* A way to determine whether the repository has rejected the push and, if
so, the reason for the rejection.
"""
import ConfigParser
import os
import sys
import optparse
import tuf
import tuf.formats
import tuf.pushtools.pushtoolslib
import tuf.pushtools.transfer.scp
def _read_config_file(filename):
"""Return a dictionary where the keys are section names and the values
are dictionaries of keys/values in that section.
"""
config = ConfigParser.RawConfigParser()
config.read(filename)
configdict = {}
for section in config.sections():
configdict[section] = {}
for key, value in config.items(section):
if key in ['seconds', 'minutes', 'days', 'hours']:
value = int(value)
elif key in ['keyids']:
value = value.split(',')
if key in configdict[section]:
configdict[section][key] = []
else:
configdict[section][key] = value
return configdict
def push(config_filepath):
"""
<Purpose>
Perform a push/transfer of target files to a host. The configuration file
'config_filepath' provides the required settings needed by the transfer
command. In the case of an 'scp' configuration file, the configuration
file would contain 'host', 'user', 'identity file', and 'remote directory'
entries.
<Arguments>
config_filepath:
The push configuration file (i.e., 'push.cfg').
<Exceptions>
tuf.FormatError, if any of the arguments are incorrectly formatted.
tuf.Error, if there was an error while processing the push.
<Side Effects>
The 'config_filepath' file is read and its contents stored, the files
in the targets directory (specified in the config file) are copied,
and the copied targets transfered to a specified host.
<Returns>
None.
"""
# Do the arguments have the correct format?
# Raise 'tuf.FormatError' if there is a mismatch.
tuf.formats.PATH_SCHEMA.check_match(config_filepath)
# Is the path to the configuration file valid?
if not os.path.isfile(config_filepath):
message = 'The configuration file path is invalid.'
raise tuf.Error(message)
config_filepath = os.path.abspath(config_filepath)
# Retrieve the push configuration settings required by the transfer
# modules. Raise ('tuf.FormatError', 'tuf.Error') if a valid
# configuration file cannot be retrieved.
config_dict = tuf.pushtools.pushtoolslib.read_config_file(config_filepath, 'push')
# Extract the transfer module identified in the configuration file.
transfer_module = config_dict['general']['transfer_module']
# 'scp' is the only transfer module currently supported. Perform
# an scp-transfer of the targets located in the targets directory as
# listed in the configuration file.
if transfer_module == 'scp':
tuf.pushtools.transfer.scp.transfer(config_dict)
else:
message = 'Cannot perform a transfer using '+repr(transfer_module)
raise tuf.Error(message)
def _get_transfer_module(modulename):
__import__("transfer.%s" % modulename)
return sys.modules["transfer.%s" % modulename]
def push(args):
config = _read_config_file(args[0])
targets = args[1:]
transfermod = _get_transfer_module(config['general']['transfer_module'])
context = transfermod.TransferContext(config['scp'])
context.transfer(targets, config['general']['metadata_path'])
context.finalize()
def parse_options():
"""
<Purpose>
Parse the command-line options. 'push.py' expects the '--config'
option to be set by the user.
Example:
$ python push.py --config ./push.cfg
The '--config' option accepts a path argument to the push configuration
file (i.e., 'push.cfg'). If the required option is unset, a parser error
is printed and the script exits.
<Arguments>
None.
<Exceptions>
None.
<Side Effects>
None.
<Returns>
The options object returned by the parser's parse_args() method.
"""
usage = 'usage: %prog --config <config path>'
option_parser = optparse.OptionParser(usage=usage)
# Add the options supported by 'push.py' to the option parser.
option_parser.add_option('--config', action='store', type='string',
help='Specify the "push.cfg" configuration file.')
(options, remaining_arguments) = option_parser.parse_args()
# Ensure the '--config' option is set. If the required option is unset,
# option_parser.error() will print an error message and exit.
if options.config is None:
message = '"--config" must be set on the command-line.'
option_parser.error(message)
return options
def getstatus():
raise NotImplementedError
def usage():
print "Known commands:"
print " push config_file target [target ...]"
sys.exit(1)
def main():
if len(sys.argv) < 2:
usage()
cmd = sys.argv[1]
args = sys.argv[2:]
if cmd in ["push", "getstatus"]:
try:
globals()[cmd](args)
except tuf.BadPasswordError:
print >> sys.stderr, "Password incorrect."
else:
usage()
if __name__ == '__main__':
main()
options = parse_options()
# Perform a 'push' of the target files specified in the configuration file.
try:
push(options.config)
except (tuf.FormatError, tuf.Error), e:
sys.stderr.write('Error: '+str(e)+'\n')
sys.exit(1)
# The 'push' and command-line options were processed successfully.
sys.exit(0)

150
src/tuf/pushtools/pushtoolslib.py Executable file
View file

@ -0,0 +1,150 @@
"""
<Program Name>
pushtoolslib.py
<Author>
Vladimir Diaz <vladimir.v.diaz@gmail.com>
<Started>
September 2012.
<Copyright>
See LICENSE for licensing information.
<Purpose>
Provide a central location for functions and data useful to multiple
'tuf.pushtools' modules. A 'read_config_file' function is currently
provided that returns correctly formatted configuration dictionaries
needed by the 'push.py' and 'receive.py' scripts.
"""
import ConfigParser
import os
import tuf.formats
PUSH_CONFIG = 'push.cfg'
RECEIVE_CONFIG = 'receive.cfg'
TRANSFER_MODULES = ['scp']
CONFIG_TYPES = ['push', 'receive']
def read_config_file(filename, config_type):
"""
<Purpose>
Return a dictionary where the keys are section names and the values
dictionaries of the keys/values in that section. The returned
dict should be correctly formatted, contain the required data
according to its config type, and be valid (i.e., a correctly named
file and available).
Example config:
config_dict = {'general': {'transfer_module': 'scp', ...},
'scp': {'host': 'localhost', 'user': 'McFly', ...}}
<Arguments>
filename:
The filepath to the configuration file.
config_type:
A string identifying the type of config file expected. Supported
config types: 'push' and 'receive'.
<Exceptions>
tuf.FormatError, if the arguments are improperly formatted.
tuf.Error, if there is an error processing the config contents.
<Side Effects>
The contents of 'filename' are read and stored.
<Returns>
A dictionary containing the data loaded from the configuration file.
"""
# Do the arguments have the correct format?
# Raise 'tuf.FormatError' if there is a mismatch.
tuf.formats.PATH_SCHEMA.check_match(filename)
tuf.formats.NAME_SCHEMA.check_match(config_type)
# RawConfigParser is used because unlike ConfigParser,
# it does not provide magical interpolation/expansion
# of variables (e.g., '%(option)s' would be ignored).
config = ConfigParser.RawConfigParser()
config.read(filename)
if config.sections() is None:
raise tuf.Error('Could not read '+repr(filename))
# Extract the relevant information from the config and build the
# 'config_dict' dictionary.
config_dict = {}
for section in config.sections():
config_dict[section] = {}
for key, value in config.items(section):
# Split comma-separated entries and store them in a list.
# 'pushroots' is the only entry that currently accepts
# multiple values.
if key in ['pushroots']:
value = value.split(',')
config_dict[section][key] = value
# Before returning a 'push' config dict, check the config is properly
# formatted, valid, and contains the required data.
if config_type == 'push':
# Ensure 'filename' is an appropriately named push config file.
if os.path.basename(filename) != PUSH_CONFIG:
message = repr(filename)+' is not a valid push config file.'+\
' The push config file should be named: '+repr(PUSH_CONFIG)
raise tuf.Error(message)
# Retrieve the transfer module from the push config. The caller
# expects a valid config dict containing the required keys/values.
try:
transfer_module = config_dict['general']['transfer_module']
except KeyError, e:
message = 'The push config file did not contain the required '+\
'"transfer_module" entry under "[general]".'
raise tuf.Error(message)
# Determine the transfer module and ensure the config file is properly
# formatted for an "scp configuration file". Raise 'tuf.FormatError'
# if there is mismatch.
if transfer_module == 'scp':
try:
tuf.formats.SCPCONFIG_SCHEMA.check_match(config_dict)
except tuf.FormatError, e:
message = repr(PUSH_CONFIG)+' rejected. '+str(e)
raise tuf.FormatError(message)
# A supported transfer module was not found. Raise 'tuf.Error'.
else:
message = 'The config file contains an invalid "transfer_module" entry '+\
'Supported transfer modules: '+repr(TRANSFER_MODULES)
raise tuf.Error(message)
# Before returning a 'receive' config dict, check the config is properly
# formatted, valid, and contains the required data.
elif config_type == 'receive':
# Ensure 'filename' is an appropriately named receive config file.
if os.path.basename(filename) != RECEIVE_CONFIG:
message = repr(filename)+' is not a valid receive config file.'+\
' The receive config file should be named: '+repr(RECEIVE_CONFIG)
raise tuf.Error(message)
# Determine if the config file is properly formatted for a "receive
# configuration file". Raise 'tuf.FormatError' if there is a
# mismatch.
try:
tuf.formats.RECEIVECONFIG_SCHEMA.check_match(config_dict)
except tuf.FormatError, e:
message = repr(RECEIVE_CONFIG)+' rejected. '+str(e)
raise tuf.FormatError(message)
# Invalid 'config_type' requested.
else:
message = 'Invalid "config_type" argument. Supported: '+repr(CONFIG_TYPES)
raise tuf.Error(message)
return config_dict

6
src/tuf/pushtools/receivetools/receive.cfg.sample Executable file
View file

@ -0,0 +1,6 @@
[general]
pushroots = /home/user/pushes,/home/user2/pushes
repository_directory = /var/tuf/test-repo
metadata_directory = /var/tuf/test-repo/metadata
targets_directory = /var/tuf/test-repo/targets
backup_directory = /var/tuf/test-repo/replaced

1115
src/tuf/pushtools/receivetools/receive.py
View file

File diff suppressed because it is too large Load diff

225
src/tuf/pushtools/transfer/scp.py
View file

@ -1,35 +1,51 @@
# Copyright 2010 The Update Framework. See LICENSE for licensing information.
"""
SCP transfer module for the developer push mechanism.
<Program Name>
scp.py
This will use scp to upload a push directory to the repository. The directory
will be named with the current timestamp in the format XXXXXXXXXX.XX. The
directory will contain a file named 'info' that provides information about
the push, the signed metadata file, and a 'targets' directory that contains
the targets specified in the metadata.
<Author>
Vladimir Diaz <vladimir.v.diaz@gmail.com>
Use of this module requires the following section to be present in the push
configuration file provided to push.py:
<Started>
August 2012. Based on a previous version of this module by Geremy Condra.
[scp]
host = somehost
user = someuser
identity_file = optional_path_to_ssh_key
remote_dir = ~/pushes
<Copyright>
See LICENSE for licensing information.
The remote_dir should correspond to a pushroot configured in the repository's
receive.py script.
<Purpose>
SCP (secure copy) transfer module for the developer push mechanism.
This transfer module will output to stdout the commands it runs and the output
of those commands.
This will use scp to upload a push directory to the repository. The directory
will be named with the current timestamp in the format XXXXXXXXXX.XX. The
directory will contain a file named 'info' that provides information about
the push, the signed metadata file, and a 'targets' directory that contains
the targets specified in the metadata.
Example:
Use of this module requires the following section to be present in the push
configuration file provided to 'push.py':
[scp]
host = host
user = user
identity_file = optional_path_to_ssh_key
remote_directory = ~/pushes
The 'remote_directory' should correspond to a pushroot configured in the
repository's 'receive.py' configuration file.
This transfer module will output to stdout the commands it runs and the output
of those commands.
Example:
$ python pushtools/push.py --config ./push.cfg
Running command: scp -r /tmp/tmpXi0GZH user@host:~/pushes/1348352878.31
helloworld.py 100% 13 0.0KB/s 00:00
LICENSE 100% 12 0.0KB/s 00:00
targets.txt 100% 7 0.0KB/s 00:00
info 100% 32 0.0KB/s 00:00
$ python pushtools/push.py push push.cfg test.txt
Running command: scp -r /tmp/tmpc8PiXo somehost:~/test/pushes/1273704893.55
info 100% 21 0.0KB/s 00:00
targets.txt 100% 771 0.8KB/s 00:00
test.txt 100% 5 0.0KB/s 00:00
"""
import os
@ -38,75 +54,118 @@
import tempfile
import time
import tuf.formats
class TransferContext(object):
def __init__(self, config):
self.host = config['host']
self.user = config.get('user')
self.identity_file = config.get('identity_file')
self.remote_dir = config.get('remote_dir', '.')
def transfer(scp_config_dict):
"""
<Purpose>
Create a local temporary directory with an added 'info' file used to
communicate additional information to the repository. This directory
will be transferred to the repository.
<Arguments>
scp_config_dict:
The dict containing the options to use with the SCP command.
def transfer(self, target_paths, metadata_path):
"""
Create a local temporary directory with an additional file used to
communicate additional information to the repository. This directory
will be transferred to the repository.
"""
<Exceptions>
tuf.FormatError, if the arguments are improperly formatted.
basecommand = ['scp']
if self.identity_file:
basecommand.extend(['-i', self.identity_file])
tuf.Error, if the transfer failed.
timestamp = time.time()
dest = ""
if self.user:
dest += "%s@"
dest += "%s:%s/%s" % (self.host, self.remote_dir, timestamp)
<Side Effects>
Files specified in 'push.cfg' will be transfered to a host using
'scp'.
<Returns>
None.
"""
tempdir = tempfile.mkdtemp()
try:
# Make sure the temp directory is world-readable as the permissions
# get carried over in the scp'ing.
os.chmod(tempdir, 0755)
# Do the arguments have the correct format?
# Raise 'tuf.FormatError' if there is a mismatch.
tuf.formats.SCPCONFIG_SCHEMA.check_match(scp_config_dict)
# Extract the required 'scp' entries. If an entry contains
# a path argument, Tilde Expansions or user home symbols
# are converted.
host = scp_config_dict['scp']['host']
user = scp_config_dict['scp']['user']
# The SCP command accepts an optional path to an SSH private key file.
identity_file = scp_config_dict['scp']['identity_file']
identity_file = os.path.expanduser(identity_file)
# The directory on the host the target files will be pushed to.
remote_directory = scp_config_dict['scp'].get('remote_directory', '.')
remote_directory = os.path.expanduser(remote_directory)
# The 'targets.txt' metadata file to be pushed to the host.
metadata_path = scp_config_dict['general']['metadata_path']
metadata_path = os.path.expanduser(metadata_path)
# The local targets directory containing the target to be pushed.
targets_directory = scp_config_dict['general']['targets_directory']
targets_directory = os.path.expanduser(targets_directory)
basecommand = ['scp']
if identity_file:
basecommand.extend(['-i', identity_file])
# Create a file that tells the repository the name of the targets
# metadata file. For delegation, this will be the only way the
# the repository knows the full role name.
fp = open(os.path.join(tempdir, 'info'), 'w')
fp.write("metadata=%s\n" % metadata_path)
fp.close()
# Build the destination.
# Example: 'user@localhost:~/pushes/1273704893.55'
timestamp = time.time()
destination = ''
if user:
destination = destination+user+'@'
destination = destination+host+':'+remote_directory+'/'+str(timestamp)
# Copy the metadata.
basename = os.path.basename(metadata_path)
shutil.copy(metadata_path, os.path.join(tempdir, basename))
temporary_directory = tempfile.mkdtemp()
try:
# Make sure the temp directory is world-readable, as the permissions
# get carried over in the scp'ing.
os.chmod(temporary_directory, 0755)
# Create a directory that all target files will be put in before
# being transferred.
targetsdir = os.path.join(tempdir, 'targets')
os.mkdir(targetsdir)
# Create a file that tells the repository the name of the targets
# metadata file. For delegation, this will be the only way the
# the repository knows the full role name.
file_object = open(os.path.join(temporary_directory, 'info'), 'w')
file_object.write('metadata='+metadata_path+'\n')
file_object.close()
# This is quite inefficient for large files, but just copy all
# targets into the correct directory structure.
for path in target_paths:
dirname = os.path.dirname(path)
basename = os.path.basename(path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
shutil.copy(path, os.path.join(targetsdir, basename))
# Copy the targets metadata.
basename = os.path.basename(metadata_path)
shutil.copy(metadata_path, os.path.join(temporary_directory, basename))
# This will create the 'timestamp' directory on the remote host and
# it will contain the info file and an empty targets directory.
command = basecommand[:]
command.append('-r') # recursive
command.append(tempdir)
command.append(dest)
print "Running command: %s" % ' '.join(command)
# Raises subprocess.CalledProcessError on failure.
subprocess.check_call(command)
# Create a directory that all target files will be put in before
# being transferred.
temporary_targets_directory = os.path.join(temporary_directory, 'targets')
finally:
shutil.rmtree(tempdir)
# Copy all the targets into the correct directory structure.
shutil.copytree(targets_directory, temporary_targets_directory)
def finalize(self):
pass
# This will create the 'timestamp' directory on the remote host. The
# 'timestamp' directory will contain the 'info' file, targets metadata,
# and the targets directory being pushed.
command = basecommand[:]
# Add the recursive option, which will add the full contents of
# 'temporary_directory'
command.append('-r')
command.append(temporary_directory)
command.append(destination)
# Example 'command':
# ['scp', '-i', '/home/user/.ssh/id_dsa', '-r', '/tmp/tmpmxWxLS',
# 'user@host:~/pushes/1348349228.4']
print 'Running command: '+' '.join(command)
# 'subprocess.CalledProcessError' raised on scp command failure.
# Catch the exception and raise 'tuf.Error'.
# For important security information on 'subprocess',
# See http://docs.python.org/library/subprocess.html
try:
subprocess.check_call(command)
except subprocess.CalledProcessError, e:
message = 'scp.transfer failed.'
raise tuf.Error(message)
finally:
shutil.rmtree(temporary_directory)