From f58113a67323397e5e18049557b4d41934f23abb Mon Sep 17 00:00:00 2001 From: Hardik Darji Date: Thu, 19 Jun 2014 12:24:13 -0400 Subject: [PATCH 01/52] Partial header block added --- tuf/interposition/updater.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index c2d266e2..629c39e9 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -1,3 +1,12 @@ +""" + + updater.py + + + Pankhuri Goyal + +""" + import mimetypes import os.path import re From 31adb150beee6428ad260845450f09790ceb85c3 Mon Sep 17 00:00:00 2001 From: pankh Date: Thu, 10 Jul 2014 11:14:56 -0400 Subject: [PATCH 02/52] Add purpose section and comments to interposition/updater.py --- tuf/interposition/updater.py | 80 +++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 629c39e9..adb0c58c 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -3,7 +3,85 @@ updater.py - Pankhuri Goyal + Pankhuri Goyal + + + June 2014. + + + See LICENSE for licensing information. + + + Interposition is the high-level integration of TUF. 'updater.py' is used to perform high-level integration of TUF to the + software updater. This means that all the processes which are taking place in the low-level integration will be done + automatically. This layer of processes will be transparent to the client. + TODO: Add more description to purpose. + TODO: Add Pros and Cons of using interposition. + + + + To implement interpostion client only need to have two files - + 1. A python file which client will have to run in order to perform interposition. For example - interposition.py. + + # First import the main module called interposition which contains all the required directories and classes. + import tuf.interposition + + # urllib_tuf and urllib2_tuf are TUF's copy of urllib and urllib2 + from tuf.interposition import urllib_tuf as urllib + from tuf.interposition import urllib2_tuf as urllib2 + + # From tuf.interposition, configure() method is called. + # configure() is within __init__.py + # Ways to call this method are as follows : + # First, configure() - By default, the configuration object is expected to be situated in the current working directory + # in the file with the name "tuf.interposition.json". + # Second, configure(filename="/path/to/json") + # Configure() returns a dictionary of configurations + configurations = tuf.interposition.configure() + + url = 'http://example.com/path/to/document' + # This is the standard way of opening and retrieving url in python. + urllib.urlopen(url) + urllib.urlretrieve(url) + urllib2.urlopen(url) + + # Remove TUF interposition for previously read configurations. That is remove the updater object. + tuf.interposition.deconfigure(configurations) + + + 2. A JSON object which tells tuf.interposition which URLs to intercept, how to transform them (if necessary), and where to forward them + (possibly over SSL) for secure responses via TUF. By default, the name of the file is tuf.interposition.json which is as follows - + + # configurations are simply a JSON object which allows you to answer these questions - + # - Which network location get intercepted? + # - Given a network location, which TUF mirrors should we forward requests to? + # - Given a network location, which paths should be intercepted? + # - Given a TUF mirror, how do we verify its SSL certificate? + { + # This is required root object. + "configurations": { + # Which network location should be intercepted? + # Network locations may be specified as "hostname" or "hostname:port". + "seattle.poly.edu": { + # Where do we find the client copy of the TUF server metadata? + "repository_directory": ".", + # Where do we forward the requests to seattle.poly.edu? + "repository_mirrors" : { + "mirror1": { + # In this case, we forward them to http://tuf.seattle.poly.edu + "url_prefix": "http://localhost:8001", + # You do not have to worry about these default parameters. + "metadata_path": "metadata", + "targets_path": "targets", + "confined_target_dirs": [ "" ] + } + } + } + } + } + + # After making these two files on the client side, run interposition.py. This will start the interposition process. It generates a log + # file in the same directory which can be used for a review. """ From a3e63a293c4cfabf96f20a1b5faab25c78360750 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Thu, 10 Jul 2014 11:31:43 -0400 Subject: [PATCH 03/52] Add purpose section and comments to interposition/updater.py --- tuf/interposition/updater.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index adb0c58c..fcc2e63a 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -15,6 +15,7 @@ Interposition is the high-level integration of TUF. 'updater.py' is used to perform high-level integration of TUF to the software updater. This means that all the processes which are taking place in the low-level integration will be done automatically. This layer of processes will be transparent to the client. + Updater.py have two classes named as Updater and UpdaterController. TODO: Add more description to purpose. TODO: Add Pros and Cons of using interposition. From 0cda6d3d22ecc63e2ce7fa3bb8a3fafd0380c898 Mon Sep 17 00:00:00 2001 From: Pankh Date: Thu, 10 Jul 2014 18:56:47 -0400 Subject: [PATCH 04/52] Add purpose and comments to interposition/updater.py --- tuf/interposition/__init__.py | 3 ++ tuf/interposition/updater.py | 89 +++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index 4360016e..be9ef62b 100644 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -188,6 +188,9 @@ def __read_configuration(configuration_handler, parsed_configurations = {} try: + # open() is function in class Updater. It opens the file with given url + # as a temporary file in the binary mode and remains transparent to the + # software updater. with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) configurations = tuf_interpositions.get("configurations", {}) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index fcc2e63a..4f137ad5 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -12,19 +12,23 @@ See LICENSE for licensing information. - Interposition is the high-level integration of TUF. 'updater.py' is used to perform high-level integration of TUF to the - software updater. This means that all the processes which are taking place in the low-level integration will be done - automatically. This layer of processes will be transparent to the client. + Interposition is the high-level integration of TUF. 'updater.py' is used to + perform high-level integration of TUF to the software updater. This means + that all the processes which are taking place in the low-level integration + will be done automatically. This layer of processes will be transparent to + the client. Updater.py have two classes named as Updater and UpdaterController. - TODO: Add more description to purpose. - TODO: Add Pros and Cons of using interposition. + #TODO: Add more description to purpose. + #TODO: Add Pros and Cons of using interposition. To implement interpostion client only need to have two files - - 1. A python file which client will have to run in order to perform interposition. For example - interposition.py. + 1. A python file which client will have to run in order to perform + interposition. For example - interposition.py. - # First import the main module called interposition which contains all the required directories and classes. + # First import the main module called interposition which contains all + # the required directories and classes. import tuf.interposition # urllib_tuf and urllib2_tuf are TUF's copy of urllib and urllib2 @@ -34,8 +38,9 @@ # From tuf.interposition, configure() method is called. # configure() is within __init__.py # Ways to call this method are as follows : - # First, configure() - By default, the configuration object is expected to be situated in the current working directory - # in the file with the name "tuf.interposition.json". + # First, configure() - By default, the configuration object is expected + # to be situated in the current working directory in the file with the + # name "tuf.interposition.json". # Second, configure(filename="/path/to/json") # Configure() returns a dictionary of configurations configurations = tuf.interposition.configure() @@ -46,16 +51,21 @@ urllib.urlretrieve(url) urllib2.urlopen(url) - # Remove TUF interposition for previously read configurations. That is remove the updater object. + # Remove TUF interposition for previously read configurations. That is + # remove the updater object. tuf.interposition.deconfigure(configurations) - 2. A JSON object which tells tuf.interposition which URLs to intercept, how to transform them (if necessary), and where to forward them - (possibly over SSL) for secure responses via TUF. By default, the name of the file is tuf.interposition.json which is as follows - + 2. A JSON object which tells tuf.interposition which URLs to intercept, how + to transform them (if necessary), and where to forward them (possibly over + SSL) for secure responses via TUF. By default, the name of the file is + tuf.interposition.json which is as follows - - # configurations are simply a JSON object which allows you to answer these questions - + # configurations are simply a JSON object which allows you to answer + # these questions - # - Which network location get intercepted? - # - Given a network location, which TUF mirrors should we forward requests to? + # - Given a network location, which TUF mirrors should we forward + # requests to? # - Given a network location, which paths should be intercepted? # - Given a TUF mirror, how do we verify its SSL certificate? { @@ -81,8 +91,9 @@ } } - # After making these two files on the client side, run interposition.py. This will start the interposition process. It generates a log - # file in the same directory which can be used for a review. + # After making these two files on the client side, run interposition.py. This + # will start the interposition process. It generates a log file in the same + # directory which can be used for a review. """ @@ -102,7 +113,7 @@ # We import them directly into our namespace so that there is no name conflict. from configuration import Configuration, InvalidConfiguration from utility import Logger, InterpositionException - +#TODO: Remove utility because the Logger is it two places. @@ -110,9 +121,7 @@ ################################ GLOBAL CLASSES ################################ - - - +#TODO: Put this class in the Exception file of TUF. class URLMatchesNoPattern(InterpositionException): """URL matches no user-specified regular expression pattern.""" pass @@ -188,6 +197,23 @@ def download_target(self, target_filepath): return destination_directory, filename + # TODO: decide prudent course of action in case of failure + def get_target_filepath(self, source_url): + # Locate the fileinfo of 'target_filepath'. updater.target() searches + # Targets metadata in order of trust, according to the currently trusted + # snapshot. To prevent consecutive target file requests from referring to + # different snapshots, top-level metadata is not automatically refreshed. + targets = [self.updater.target(target_filepath)] + + # TODO: targets are always updated if destination directory is new, right? + updated_targets = self.updater.updated_targets(targets, destination_directory) + + for updated_target in updated_targets: + self.updater.download_target(updated_target, destination_directory) + + return destination_directory, filename + + # TODO: decide prudent course of action in case of failure def get_target_filepath(self, source_url): """Given source->target map, figure out what TUF *should* download given a @@ -242,8 +268,11 @@ def open(self, url, data=None): # Windows. # http://docs.python.org/2/tutorial/inputoutput.html#reading-and-writing-files # TODO: like tempfile, ensure file is deleted when closed? + # open() in the line below is a predefined function in python. temporary_file = open(filename, 'rb') + #TODO: addinfourl is not in urllib package anymore. We need to check if + # other option for this is working or not. # Extend temporary_file with info(), getcode(), geturl() # http://docs.python.org/2/library/urllib.html#urllib.urlopen response = urllib.addinfourl(temporary_file, headers, url, code=200) @@ -454,23 +483,3 @@ def remove(self, configuration): assert isinstance(configuration, Configuration) - repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() - - assert configuration.hostname in self.__updaters - assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) - - # Get the updater. - updater = self.__updaters.get(configuration.hostname) - - # If all is well, remove the stored Updater as well as its associated - # repository mirror hostnames. - updater.cleanup() - del self.__updaters[configuration.hostname] - self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames) - - Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration)) - - - - - From 98acdb8bc05c5d7fdc0188dc83063d38f91d8d40 Mon Sep 17 00:00:00 2001 From: Pankh Date: Thu, 10 Jul 2014 20:34:03 -0400 Subject: [PATCH 05/52] Add purpose section and comments to interposition/updater.py --- tuf/interposition/updater.py | 37 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 4f137ad5..25d2d19a 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -113,7 +113,7 @@ # We import them directly into our namespace so that there is no name conflict. from configuration import Configuration, InvalidConfiguration from utility import Logger, InterpositionException -#TODO: Remove utility because the Logger is it two places. +#TODO: Remove utility because the Logger is at two places. @@ -197,23 +197,6 @@ def download_target(self, target_filepath): return destination_directory, filename - # TODO: decide prudent course of action in case of failure - def get_target_filepath(self, source_url): - # Locate the fileinfo of 'target_filepath'. updater.target() searches - # Targets metadata in order of trust, according to the currently trusted - # snapshot. To prevent consecutive target file requests from referring to - # different snapshots, top-level metadata is not automatically refreshed. - targets = [self.updater.target(target_filepath)] - - # TODO: targets are always updated if destination directory is new, right? - updated_targets = self.updater.updated_targets(targets, destination_directory) - - for updated_target in updated_targets: - self.updater.download_target(updated_target, destination_directory) - - return destination_directory, filename - - # TODO: decide prudent course of action in case of failure def get_target_filepath(self, source_url): """Given source->target map, figure out what TUF *should* download given a @@ -383,7 +366,6 @@ def __check_configuration_on_add(self, configuration): return repository_mirror_hostnames - def add(self, configuration): """Add an Updater based on the given Configuration.""" @@ -395,7 +377,6 @@ def add(self, configuration): self.__repository_mirror_hostnames.update(repository_mirror_hostnames) - def refresh(self, configuration): """Refresh the top-level metadata of the given Configuration.""" @@ -419,7 +400,6 @@ def refresh(self, configuration): updater.refresh() - def get(self, url): """Get an Updater, if any, for this URL. @@ -483,3 +463,18 @@ def remove(self, configuration): assert isinstance(configuration, Configuration) + repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + + assert configuration.hostname in self.__updaters + assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) + + # Get the updater. + updater = self.__updaters.get(configuration.hostname) + + # If all is well, remove the stored Updater as well as its associated + # repository mirror hostnames. + updater.cleanup() + del self.__updaters[configuration.hostname] + self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames) + + Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration)) From dc263af83d37345ffe766951a9db9412c1c79d6d Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Thu, 10 Jul 2014 23:37:03 -0400 Subject: [PATCH 06/52] Made changes to the purpose section of interposition/updater.py --- tuf/interposition/updater.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 25d2d19a..1dd17f58 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -9,10 +9,10 @@ June 2014. - See LICENSE for licensing information. + See LICENSE for licensing information. - Interposition is the high-level integration of TUF. 'updater.py' is used to + Interposition is the high-level integration of TUF. 'updater.py' is used to perform high-level integration of TUF to the software updater. This means that all the processes which are taking place in the low-level integration will be done automatically. This layer of processes will be transparent to @@ -37,12 +37,17 @@ # From tuf.interposition, configure() method is called. # configure() is within __init__.py + # It takes 3 arguments, one of which is filename of a JSON file. + # This JSON file contains a set of configurations. To make this file, + # follow the second point below. # Ways to call this method are as follows : # First, configure() - By default, the configuration object is expected # to be situated in the current working directory in the file with the # name "tuf.interposition.json". # Second, configure(filename="/path/to/json") # Configure() returns a dictionary of configurations + # Internally, configure() calls add(configuration) function which is in + # the class UpdaterController. configurations = tuf.interposition.configure() url = 'http://example.com/path/to/document' @@ -53,12 +58,17 @@ # Remove TUF interposition for previously read configurations. That is # remove the updater object. + # Deconfigure() takes only one argument i.e. configurations. + # It calls remove(configuration) function which is in UpdaterController + # class in updater.py. tuf.interposition.deconfigure(configurations) - 2. A JSON object which tells tuf.interposition which URLs to intercept, how - to transform them (if necessary), and where to forward them (possibly over - SSL) for secure responses via TUF. By default, the name of the file is + 2. The filename passed as a parameter in configure function is a JSON file. + It is called as configurations. It is a JSON object which tells + tuf.interposition which URLs to intercept, how to transform them (if + necessary), and where to forward them (possibly over SSL) for secure + responses via TUF. By default, the name of the file is tuf.interposition.json which is as follows - # configurations are simply a JSON object which allows you to answer @@ -73,13 +83,13 @@ "configurations": { # Which network location should be intercepted? # Network locations may be specified as "hostname" or "hostname:port". - "seattle.poly.edu": { + "localhost": { # Where do we find the client copy of the TUF server metadata? "repository_directory": ".", - # Where do we forward the requests to seattle.poly.edu? + # Where do we forward the requests to localhost? "repository_mirrors" : { "mirror1": { - # In this case, we forward them to http://tuf.seattle.poly.edu + # In this case, we forward them to http://localhost:8001 "url_prefix": "http://localhost:8001", # You do not have to worry about these default parameters. "metadata_path": "metadata", From 070574d220fbd01e4f2a3b1b13d4301619b10319 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Mon, 14 Jul 2014 17:14:08 -0400 Subject: [PATCH 07/52] Add the purpose section and comments to all the funtions in the Updater class in tuf.interposition.updater --- tuf/interposition/updater.py | 225 ++++++++++++++++++++++++++++++++--- 1 file changed, 208 insertions(+), 17 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 1dd17f58..05562cb3 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -141,10 +141,90 @@ class URLMatchesNoPattern(InterpositionException): class Updater(object): - """I am an Updater model.""" + """ + + Provide a class that can download target files securely. It performs all + those things which client/updator.py performs. But it performs it in the + background, transparent to the client. + + + refresh(): + This method refresh top-level metadata. It calls the refresh() method of + client/updater. refresh() method of client/updater.py downloads, verifies, + and loads metadata for the top-level roles in a specific order (i.e., + timestamp -> snapshot -> root -> targets). The expiration time for + downloaded metadata is also verified. + + cleanup(): + It will clean up all the temporary directories which were made as a result + of download. It then prints a message of deletion and also mentions the + name of the deleted directory. + + download_target(target_filepath): + It downloads the 'target' and verify it is trusted. This procedure happens + in the background, transparent to the client. This will only store the + file at 'destination_directory' if the downloaded file matches the + description of the file in the trusted metadata. + + get_target_filepath(source_url): + Given source->target map, this method will figure out what TUF should + download when a URL is given. + + open(url, data): + Open the URL url which can either be a string or a request object. + The file is opened in the binary read mode as a temporary file. + + retrieve(url, filename, reporthook, data): + retrieve() method first get the target file path by calling + get_target_filepath(url) which in tuf.interposition.updater and then + calls download_target() method for the above file path. + + switch_context(): + There is an updater object for each network location that is interposed. + Context switching is required because there are multiple + tuf.client.Updater() objects and each one depends on tuf.conf settings + that are shared. + """ def __init__(self, configuration): + """ + + Constructor. Instantiating an updater object causes creation of a temporary + directory. This temporary directory is used for the interposition updater. + After that the updater of client module which performs the low-level + integration is called. + + + configuration: + A dictionary holding information like the following - + - Which network location get intercepted? + - Given a network location, which TUF mirrors should we forward requests + to? + - Given a network location, which paths should be intercepted? + - Given a TUF mirror, how do we verify its SSL certificate? + + This dictionary holds repository mirror information, conformant to + 'tuf.formats.MIRRORDICT_SCHEMA'. Information such as the directory + containing the metadata and target files, the server's URL prefix, and + the target content directories the client should be confined to. + + repository_mirrors = {'mirror1': {'url_prefix': 'http://localhost:8001', + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']}} + + + #TODO: Exceptions + + + The metadata files (e.g., 'root.json', 'targets.json') for the top-level + roles are read from disk and stored in dictionaries. + + + None. + """ + CREATED_TEMPDIR_MESSAGE = "Created temporary directory at {tempdir}" self.configuration = configuration @@ -152,27 +232,51 @@ def __init__(self, configuration): self.tempdir = tempfile.mkdtemp() Logger.debug(CREATED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir)) - # must switch context before instantiating updater - # because updater depends on some module (tuf.conf) variables + # Switching context before instantiating updater because updater depends + # on some module (tuf.conf) variables. self.switch_context() + + # Instantiating an client/updater object causes all the configurations for + # the top-level roles to be read from disk, including the key and role + # information for the delegated targets of 'targets'. The actual metadata + # for delegated roles is not loaded in __init__. The metadata for these + # delegated roles, including nested delegated roles, are loaded, updated, + # and saved to the 'self.metadata' store by the target methods, like + # all_targets() and targets_of_role(). self.updater = tuf.client.updater.Updater(self.configuration.hostname, self.configuration.repository_mirrors) # Update the client's top-level metadata. The download_target() method does # not automatically refresh top-level prior to retrieving target files and - # their associated Targets metadata, so update the top-level - # metadata here. + # their associated Targets metadata, so update the top-level metadata here. Logger.info('Refreshing top-level metadata for interposed '+repr(configuration)) self.updater.refresh() def refresh(self): - """Refresh top-level metadata""" + """ + + This method refresh top-level metadata. It calls the refresh() method of + client/updater. + refresh() method of client/updater.py downloads, verifies, and loads + metadata for the top-level roles in a specific order (i.e., timestamp -> + snapshot -> root -> targets) + The expiration time for downloaded metadata is also verified. + + This refresh() method should be called by the client before any target + requests. Therefore to automate the process, it is called here. + """ + self.updater.refresh() def cleanup(self): - """Clean up after certain side effects, such as temporary directories.""" + """ + + It will clean up all the temporary directories which were made as a + result of download. It then prints a message of deletion and also + mentions the name of the deleted directory. + """ DELETED_TEMPDIR_MESSAGE = "Deleted temporary directory at {tempdir}" shutil.rmtree(self.tempdir) @@ -180,27 +284,62 @@ def cleanup(self): def download_target(self, target_filepath): - """Downloads target with TUF as a side effect.""" + """ + + It downloads the 'target' and verify it is trusted. This procedure + happens in the background, transparent to the client. + + This will only store the file at 'destination_directory' if the downloaded + file matches the description of the file in the trusted metadata. - # download file into a temporary directory shared over runtime + + target_filepath contains the path to the target to be downloaded. + + + #TODO: Exceptions + + + A target file is saved to the local system. + + + It returns destination_directory where the target is been stored and + filename of the target file been stored in the directory. + + """ + + # Download file into a temporary directory shared over runtime destination_directory = self.tempdir + # A new path is generated by joining the destination directory path that is + # our temporary directory path and target file path. # Note: join() discards 'destination_directory' if 'target_filepath' # contains a leading path separator (i.e., is treated as an absolute path). filename = os.path.join(destination_directory, target_filepath.lstrip(os.sep)) - # Switch TUF context. + # Switch TUF context. Switching context before instantiating updater + # because updater depends on some module (tuf.conf) variables. self.switch_context() # Locate the fileinfo of 'target_filepath'. updater.target() searches # Targets metadata in order of trust, according to the currently trusted # snapshot. To prevent consecutive target file requests from referring to # different snapshots, top-level metadata is not automatically refreshed. + # It returns the target information for a specific file identified by its + # file path. This target method also downloads the metadata of updated + # targets. targets = [self.updater.target(target_filepath)] # TODO: targets are always updated if destination directory is new, right? + # After the client has retrieved the target information for those targets + # they are interested in updating, updated_targets() method is called to + # determine which targets have changed from those saved locally on disk. + # All the targets that have changed are returns in a list. From this list, + # a request to download is made by calling 'download_target()'. updated_targets = self.updater.updated_targets(targets, destination_directory) + # The download_target() method in client/updater.py performs the actual + # download of the specified target. The file is saved to the + # 'destination_directory' argument. for updated_target in updated_targets: self.updater.download_target(updated_target, destination_directory) @@ -209,8 +348,18 @@ def download_target(self, target_filepath): # TODO: decide prudent course of action in case of failure def get_target_filepath(self, source_url): - """Given source->target map, figure out what TUF *should* download given a - URL.""" + """ + + Given source->target map, this method will figure out what TUF should + download when a URL is given. + + + source_url is passed while calling the function. This is the url which + we want to retrieve. For this url, get_target_filepath() method is called. + + + It returns target_filepath. This is the target which TUF should download. + """ WARNING_MESSAGE = "Possibly invalid target_paths for " + \ "{network_location}! No TUF interposition for {url}" @@ -223,6 +372,7 @@ def get_target_filepath(self, source_url): # how to map the source URL to a target URL understood by TUF? for target_path in self.configuration.target_paths: + #TODO: What these two lines are doing? # target_path: { "regex_with_groups", "target_with_group_captures" } # e.g. { ".*(/some/directory)/$", "{0}/index.html" } source_path_pattern, target_path_pattern = target_path.items()[0] @@ -253,6 +403,20 @@ def get_target_filepath(self, source_url): # TODO: distinguish between urllib and urllib2 contracts def open(self, url, data=None): + """ + + Open the URL url which can either be a string or a request object. + The file is opened in the binary read mode as a temporary file. + + + url, the one which is to be opened. + + data must be a bytes object specifying additional data to be sent to the + server or None, if no such data needed. + + + + """ filename, headers = self.retrieve(url, data=data) # TUF should always open files in binary mode and remain transparent to the @@ -268,6 +432,7 @@ def open(self, url, data=None): # other option for this is working or not. # Extend temporary_file with info(), getcode(), geturl() # http://docs.python.org/2/library/urllib.html#urllib.urlopen + # addinfourl() works as a context manager. response = urllib.addinfourl(temporary_file, headers, url, code=200) return response @@ -275,6 +440,22 @@ def open(self, url, data=None): # TODO: distinguish between urllib and urllib2 contracts def retrieve(self, url, filename=None, reporthook=None, data=None): + """ + + retrieve() method first get the target file path by calling + get_target_filepath(url) which in tuf.interposition.updater and then + calls download_target() method for the above file path. + + + url, which is to be retrieved. + + filename, if the is given then everywhere the given filename is used. + If the filename is none, then temporary file is used. + + + It returns the filename and the headers of the file just retrieved. + + """ INTERPOSITION_MESSAGE = "Interposing for {url}" Logger.info(INTERPOSITION_MESSAGE.format(url=url)) @@ -311,11 +492,22 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): # TODO: thread-safety, perhaps with a context manager def switch_context(self): - # Set the local repository directory containing the metadata files. - tuf.conf.repository_directory = self.configuration.repository_directory + """ + + There is an updater object for each network location that is interposed. + Context switching is required because there are multiple + tuf.client.Updater() objects and each one depends on tuf.conf settings + that are shared. - # Set the local SSL certificates PEM file. - tuf.conf.ssl_certificates = self.configuration.ssl_certificates + For this, two settings are required - + 1. Setting local repository directory + 2. Setting the local SSL certificate PEM file + """ + # Set the local repository directory containing the metadata files. + tuf.conf.repository_directory = self.configuration.repository_directory + + # Set the local SSL certificates PEM file. + tuf.conf.ssl_certificates = self.configuration.ssl_certificates @@ -465,7 +657,6 @@ def get(self, url): return updater - def remove(self, configuration): """Remove an Updater matching the given Configuration.""" From cb0ce7ee2724209f5d6d04a54fc32374bb6e614d Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Wed, 30 Jul 2014 14:18:36 -0400 Subject: [PATCH 08/52] Removed tuf.interposition.utility file and added exceptions and Looger in interposition.updater --- tuf/interposition/updater.py | 387 ++++++++++++++++++++++++++--------- tuf/interposition/utility.py | 43 ---- 2 files changed, 291 insertions(+), 139 deletions(-) delete mode 100644 tuf/interposition/utility.py diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 05562cb3..3b8eab85 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """ updater.py @@ -21,11 +23,14 @@ #TODO: Add more description to purpose. #TODO: Add Pros and Cons of using interposition. - + - To implement interpostion client only need to have two files - - 1. A python file which client will have to run in order to perform - interposition. For example - interposition.py. + To implement interpostion client only need to have- + First, a client module which is modified to include interposition library and + code and second, a JSON configuration file is created, each of which is + explained below - + 1. "interposition.py" is an example client updater module that is integrating + TUF with interposition. # First import the main module called interposition which contains all # the required directories and classes. @@ -47,7 +52,7 @@ # Second, configure(filename="/path/to/json") # Configure() returns a dictionary of configurations # Internally, configure() calls add(configuration) function which is in - # the class UpdaterController. + # the tuf.interposition.UpdaterController. configurations = tuf.interposition.configure() url = 'http://example.com/path/to/document' @@ -59,8 +64,8 @@ # Remove TUF interposition for previously read configurations. That is # remove the updater object. # Deconfigure() takes only one argument i.e. configurations. - # It calls remove(configuration) function which is in UpdaterController - # class in updater.py. + # It calls remove(configuration) function which is in + # tuf.interposition.UpdaterController. tuf.interposition.deconfigure(configurations) @@ -88,7 +93,7 @@ "repository_directory": ".", # Where do we forward the requests to localhost? "repository_mirrors" : { - "mirror1": { + gg "mirror1": { # In this case, we forward them to http://localhost:8001 "url_prefix": "http://localhost:8001", # You do not have to worry about these default parameters. @@ -102,11 +107,18 @@ } # After making these two files on the client side, run interposition.py. This - # will start the interposition process. It generates a log file in the same - # directory which can be used for a review. - + # will start the interposition process. It generates a log file named tuf.log + # in the same directory, which can be used for a review. """ +# Help with Python 3 compatibility where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import mimetypes import os.path import re @@ -114,30 +126,17 @@ import tempfile import urllib import urlparse - +import logging import tuf.client.updater import tuf.conf - +import tuf.log # We import them directly into our namespace so that there is no name conflict. from configuration import Configuration, InvalidConfiguration -from utility import Logger, InterpositionException -#TODO: Remove utility because the Logger is at two places. - - - - -################################ GLOBAL CLASSES ################################ - - -#TODO: Put this class in the Exception file of TUF. -class URLMatchesNoPattern(InterpositionException): - """URL matches no user-specified regular expression pattern.""" - pass - +Logger = logging.getLogger('tuf.interposition.updater') class Updater(object): @@ -150,9 +149,9 @@ class Updater(object): refresh(): This method refresh top-level metadata. It calls the refresh() method of - client/updater. refresh() method of client/updater.py downloads, verifies, - and loads metadata for the top-level roles in a specific order (i.e., - timestamp -> snapshot -> root -> targets). The expiration time for + tuf.client.updater. refresh() method of tuf.client.updater downloads, + verifies, and loads metadata for the top-level roles in a specific order + (i.e., timestamp -> snapshot -> root -> targets). The expiration time for downloaded metadata is also verified. cleanup(): @@ -161,14 +160,12 @@ class Updater(object): name of the deleted directory. download_target(target_filepath): - It downloads the 'target' and verify it is trusted. This procedure happens - in the background, transparent to the client. This will only store the - file at 'destination_directory' if the downloaded file matches the - description of the file in the trusted metadata. + It downloads the target from the target_filepath. It also downloads the + metadata of the updated targets. get_target_filepath(source_url): - Given source->target map, this method will figure out what TUF should - download when a URL is given. + source_url is the url of the file to be updated. This method will find the + updated target for this file. open(url, data): Open the URL url which can either be a string or a request object. @@ -182,21 +179,21 @@ class Updater(object): switch_context(): There is an updater object for each network location that is interposed. Context switching is required because there are multiple - tuf.client.Updater() objects and each one depends on tuf.conf settings + tuf.client.updater objects and each one depends on tuf.conf settings that are shared. """ def __init__(self, configuration): - """ - - Constructor. Instantiating an updater object causes creation of a temporary - directory. This temporary directory is used for the interposition updater. - After that the updater of client module which performs the low-level - integration is called. + """ + + Constructor. Instantiating an updater object causes creation of a + temporary directory. This temporary directory is used for the + tuf.interposition.updater. After that the tuf.client.updater module which + performs the low-level integration is called. - - configuration: + + configuration: A dictionary holding information like the following - - Which network location get intercepted? - Given a network location, which TUF mirrors should we forward requests @@ -210,20 +207,20 @@ def __init__(self, configuration): the target content directories the client should be confined to. repository_mirrors = {'mirror1': {'url_prefix': 'http://localhost:8001', - 'metadata_path': 'metadata', - 'targets_path': 'targets', - 'confined_target_dirs': ['']}} + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']}} - - #TODO: Exceptions + + #TODO: Exceptions - - The metadata files (e.g., 'root.json', 'targets.json') for the top-level - roles are read from disk and stored in dictionaries. + + The metadata files (e.g., 'root.json', 'targets.json') for the top-level + roles are read from disk and stored in dictionaries. - - None. - """ + + None. + """ CREATED_TEMPDIR_MESSAGE = "Created temporary directory at {tempdir}" @@ -236,8 +233,8 @@ def __init__(self, configuration): # on some module (tuf.conf) variables. self.switch_context() - # Instantiating an client/updater object causes all the configurations for - # the top-level roles to be read from disk, including the key and role + # Instantiating an tuf.client.updater object causes all the configurations + # for the top-level roles to be read from disk, including the key and role # information for the delegated targets of 'targets'. The actual metadata # for delegated roles is not loaded in __init__. The metadata for these # delegated roles, including nested delegated roles, are loaded, updated, @@ -257,14 +254,31 @@ def refresh(self): """ This method refresh top-level metadata. It calls the refresh() method of - client/updater. - refresh() method of client/updater.py downloads, verifies, and loads + tuf.client.updater. + refresh() method of tuf.client.updater.py downloads, verifies, and loads metadata for the top-level roles in a specific order (i.e., timestamp -> snapshot -> root -> targets) The expiration time for downloaded metadata is also verified. This refresh() method should be called by the client before any target - requests. Therefore to automate the process, it is called here. + requests. Therefore to automate the process, it is called here. + + + None + + + tuf.NoWorkingMirrorError: + If the metadata for any of the top-level roles cannot be updated. + + tuf.ExpiredMetadataError: + if any metadata has expired. + + + Updates the metadata files of the top-level roles with the latest + information + + + None """ self.updater.refresh() @@ -286,14 +300,24 @@ def cleanup(self): def download_target(self, target_filepath): """ - It downloads the 'target' and verify it is trusted. This procedure - happens in the background, transparent to the client. + It downloads the target files from the path provided named + target_filepath. + Everything here is performed in a temporary directory. + It identifies the target information for target_filepath by calling + target() method of tuf.client.updater. This method also downloads the + metadata of the updated targets. By doing this, the client retrieves the + target information for the targets they want to update. When client + retrieves all the information, the updated_targets() method of + tuf.client.updater is called to determine the list of targets which have + been changed from those saved locally on disk. + tuf.client.upater.download_target() downloads all the targets in the list + in the destination directory which is our temporary directory. This will only store the file at 'destination_directory' if the downloaded file matches the description of the file in the trusted metadata. - target_filepath contains the path to the target to be downloaded. + 'target_filepath' is the target's relative path on the remote repository. #TODO: Exceptions @@ -321,7 +345,7 @@ def download_target(self, target_filepath): self.switch_context() # Locate the fileinfo of 'target_filepath'. updater.target() searches - # Targets metadata in order of trust, according to the currently trusted + # targets metadata in order of trust, according to the currently trusted # snapshot. To prevent consecutive target file requests from referring to # different snapshots, top-level metadata is not automatically refreshed. # It returns the target information for a specific file identified by its @@ -333,11 +357,11 @@ def download_target(self, target_filepath): # After the client has retrieved the target information for those targets # they are interested in updating, updated_targets() method is called to # determine which targets have changed from those saved locally on disk. - # All the targets that have changed are returns in a list. From this list, + # All the targets that have changed are returned in a list. From this list, # a request to download is made by calling 'download_target()'. updated_targets = self.updater.updated_targets(targets, destination_directory) - # The download_target() method in client/updater.py performs the actual + # The download_target() method in tuf.client.updater performs the actual # download of the specified target. The file is saved to the # 'destination_directory' argument. for updated_target in updated_targets: @@ -359,7 +383,8 @@ def get_target_filepath(self, source_url): It returns target_filepath. This is the target which TUF should download. - """ + + """ WARNING_MESSAGE = "Possibly invalid target_paths for " + \ "{network_location}! No TUF interposition for {url}" @@ -390,7 +415,7 @@ def get_target_filepath(self, source_url): # If source_url does not match any regular expression... if target_filepath is None: # ...then we raise a predictable exception. - raise URLMatchesNoPattern(source_url) + raise tuf.URLMatchesNoPattern(source_url) except: Logger.exception(WARNING_MESSAGE.format( @@ -407,15 +432,13 @@ def open(self, url, data=None): Open the URL url which can either be a string or a request object. The file is opened in the binary read mode as a temporary file. - + url, the one which is to be opened. data must be a bytes object specifying additional data to be sent to the server or None, if no such data needed. - - - + """ filename, headers = self.retrieve(url, data=data) @@ -496,7 +519,7 @@ def switch_context(self): There is an updater object for each network location that is interposed. Context switching is required because there are multiple - tuf.client.Updater() objects and each one depends on tuf.conf settings + tuf.client.updater objects and each one depends on tuf.conf settings that are shared. For this, two settings are required - @@ -515,11 +538,55 @@ def switch_context(self): class UpdaterController(object): """ - I am a controller of Updaters; given a Configuration, I will build and - store an Updater which you can get and use later. + + tuf.interposition.UpdaterController is a controller of the Updaters. + Given a configuration, it can build and store an Updater, which can be + used later with the help of get() method. + + + + __init__(): + It creates and initializes an empty private map of updaters and an empty + private set of repository mirror hostnames. This is used to store the + updaters added by TUF and later on TUF can get these updater to reutilize. + + __check_configuration_on_add(configuration): + It checks if the given configuration is valid or not. + + add(configuration): + This method adds the updater by adding an object of + tuf.interposition.Updater in the __updater map and by adding repository + mirror's hostname in the empty set initialized when the object of + tuf.interposition.UpdaterController is created. + + get(url): + refresh(configuration): + remove(configuration): + """ def __init__(self): + """ + + To initalize a private map of updaters and a private set of repository + mirror hostnames once the object of tuf.interposition.UpdaterController + is created. This empty map and set is later used to add, get and remove + updaters and their mirrors. + + + None + + + None + + + An empty map called '__updaters' and an empty set called + '__repository_mirror_hostnames' is created. + + + None + """ + # A private map of Updaters (network_location: str -> updater: Updater) self.__updaters = {} @@ -529,24 +596,58 @@ def __init__(self): def __check_configuration_on_add(self, configuration): """ - If the given Configuration is invalid, I raise an exception. - Otherwise, I return some information about the Configuration, - such as repository mirror hostnames. + + If the given Configuration is invalid, an exception is raised. + Otherwise, repository mirror hostnames are returned. + + + 'configuration' contains hostname, port number, repository mirrors which + are to be checked if they are valid or not. + + + tuf.FormatError + This exception is raised if the configuration is invalid i.e. if the + hostname is not unique or configuration.hostname is same as + repository_mirror_hostname. + + + It logs the error message. + + + 'repository_mirror_hostnames' + In order to prove that everything worked well, a part of configuration + is returned which is the list of repository mirrors. """ INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!" # Updater has a "global" view of configurations, so it performs - # additional checks after Configuration's own local checks. - assert isinstance(configuration, Configuration) + # additional checks after Configuration's own local checks. This will + # check if everything in tuf.interposition.ConfigurationParser worked + # or not. + + # According to __read_configuration() method in + # tuf.interposition.__init__.py, + # configuration is an instance of tuf.interposition.Configuration because + # in this method - + # configuration = configuration_parser.parse() + # configuration_parser is an instance of + # tuf.interposition.ConfigurationParser + # The configuration_parser.parse() returns tuf.interposition.Configuration + # as an object which makes configuration an instance of + # tuf.interposition.Configuration + if not isinstance(configuration, Configuration): + raise tuf.FormatError("Invalid Configuration") # Restrict each (incoming, outgoing) hostname pair to be unique across # configurations; this prevents interposition cycles, amongst other # things. # GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ... # BAD: A -> { B }, B -> { C }, C -> { A }, ... - assert configuration.hostname not in self.__updaters - assert configuration.hostname not in self.__repository_mirror_hostnames + if configuration.hostname in self.__updaters: + raise tuf.FormatError("Hostname Not Unique") + if configuration.hostname in self.__repository_mirror_hostnames: + raise tuf.FormatError("Hostname Already Exists") # Check for redundancy in server repository mirrors. repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() @@ -556,8 +657,10 @@ def __check_configuration_on_add(self, configuration): # Restrict each hostname in every (incoming, outgoing) pair to be # unique across configurations; this prevents interposition cycles, # amongst other things. - assert mirror_hostname not in self.__updaters - assert mirror_hostname not in self.__repository_mirror_hostnames + if mirror_hostname in self.__updaters: + raise tuf.FormatError("Mirror Hostname Not Unique") + if mirror_hostname in self.__repository_mirror_hostnames: + raise tuf.FormatError("Mirror Hostname Already Exists") except: error_message = \ @@ -569,25 +672,85 @@ def __check_configuration_on_add(self, configuration): def add(self, configuration): - """Add an Updater based on the given Configuration.""" + """ + + Add an Updater based on the given Configuration. Tuf keeps the track of + the updaters so that it can be fetched for later use. + + + 'configuration' is an object and on the basis of this configuration, an + updater will be added. + + + tuf.FormatError + This exception is raised if the hostname which tuf is trying to add + is not unique. + + + The object of tuf.interposition.Updater is added in the list of updaters. + Also, the mirrors of this updater are added into a + repository_mirror_hostname are added. + + + None + """ repository_mirror_hostnames = self.__check_configuration_on_add(configuration) # If all is well, build and store an Updater, and remember hostnames. Logger.info('Adding updater for interposed '+repr(configuration)) + # Adding an object of the tuf.interposition.updater with the given + # configuration. self.__updaters[configuration.hostname] = Updater(configuration) + # Adding the new the repository mirror hostnames to the list. self.__repository_mirror_hostnames.update(repository_mirror_hostnames) def refresh(self, configuration): - """Refresh the top-level metadata of the given Configuration.""" + """ + + To refresh the top-level metadata of the given 'configuration'. + It updates the latest copies of the metadata for the top-level roles. - assert isinstance(configuration, Configuration) + + 'configuration' is the object containing the configurations of the updater + to be refreshed. + + tuf.FormatError: + If there is anything wrong with the Format of the configuration, this + exception is raised. + + tuf.NotFound: + If the updater to be refreshed is not found in the list of updaters or + mirrors, then tuf.NotFound exception is raised. + + tuf.NoWorkingMirrorError: + If the metadata for any of the top-level roles cannot be updated. + + tuf.ExpiredMetadataError: + If any metadata has expired. + + + It refreshes the updater and indicate this in the log file. + + + None + """ + + # Check if the configuration is valid else raise an exception. + if not isinstance(configuration, Configuration): + raise tuf.FormatError("Invalid Configuration") + + # Get the repository mirrors of the given configuration. repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() - assert configuration.hostname in self.__updaters - assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) + # Check if the configuration.hostname is available in the updater or mirror + # list. + if not configuration.hostname in self.__updaters: + raise tuf.NotFound("Hostname Not Found") + if not repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames): + raise tuf.NotFound("Hostname Not Found") # Get the updater and refresh its top-level metadata. In the majority of # integrations, a software updater integrating TUF with interposition will @@ -598,15 +761,37 @@ def refresh(self, configuration): # updaters that require an explicit refresh of top-level metadata, this # method is provided. Logger.info('Refreshing top-level metadata for '+ repr(configuration)) + + # If everything is good then fetch the updater from __updaters with the + # given configurations. updater = self.__updaters.get(configuration.hostname) + + # Refresh the fetched updater. updater.refresh() def get(self, url): - """Get an Updater, if any, for this URL. + """ + + This method is to get the updater if it already exists. It takes the url + and parse it. Then it utilizes hostname and port of that url to check if + it already exists or not. If the updater exists, then it calls the + get_target_filepath() method which returns a target file path to be + downloaded. - Assumptions: - - @url is a string.""" + + url, for which tuf is trying to get an updater. Assumption that url is a + string. + + + + + + + + + + """ GENERIC_WARNING_MESSAGE = "No updater or interposition for url={url}" DIFFERENT_NETLOC_MESSAGE = "We have an updater for netloc={netloc1} but not for netlocs={netloc2}" @@ -622,8 +807,9 @@ def get(self, url): netloc = parsed_url.netloc network_location = "{hostname}:{port}".format(hostname=hostname, port=port) - # Sometimes parsed_url.netloc does not have a port (e.g. 80), - # so we do a double check. + # There can be a case when parsed_url.netloc does not have a port (e.g. + # 80). To avoid errors because of this case, tuf.interposition again set + # the parameters. network_locations = set((netloc, network_location)) updater = self.__updaters.get(hostname) @@ -658,7 +844,16 @@ def get(self, url): return updater def remove(self, configuration): - """Remove an Updater matching the given Configuration.""" + """ + + Remove an Updater matching the given Configuration. + + + + + + + """ UPDATER_REMOVED_MESSAGE = "Updater removed for interposed {configuration}." diff --git a/tuf/interposition/utility.py b/tuf/interposition/utility.py deleted file mode 100644 index 8e1c4cd9..00000000 --- a/tuf/interposition/utility.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging - - -# Import our standard logger for its side effects. -import tuf.log - - - - - -class InterpositionException(Exception): - """Base exception class.""" - pass - - - - - -class Logger(object): - """A static logging object for tuf.interposition.""" - - - __logger = logging.getLogger("tuf.interposition") - - - @staticmethod - def debug(message): - Logger.__logger.debug(message) - - - @staticmethod - def exception(message): - Logger.__logger.exception(message) - - - @staticmethod - def info(message): - Logger.__logger.info(message) - - - @staticmethod - def warn(message): - Logger.__logger.warn(message) From ca81096c8c00708d34651e4386a1bf4f6ff990af Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Wed, 30 Jul 2014 14:48:46 -0400 Subject: [PATCH 09/52] Remove Logger from configuration.py and add Python 3 compatibility --- tuf/interposition/configuration.py | 35 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tuf/interposition/configuration.py b/tuf/interposition/configuration.py index e165a073..d4bfeeb1 100644 --- a/tuf/interposition/configuration.py +++ b/tuf/interposition/configuration.py @@ -1,27 +1,25 @@ +#!/usr/bin/env python + +""" + +""" + +# Help with Python 3 compatibility where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + import os.path import types import urlparse - -# We import them directly into our namespace so that there is no name conflict. -from utility import Logger, InterpositionException - - - - - -################################ GLOBAL CLASSES ################################ - - - - - -class InvalidConfiguration(InterpositionException): - """User configuration is invalid.""" - pass - +import tuf.log +Logger = logging.getLogger('tuf.interposition.configuration') class Configuration(object): @@ -59,6 +57,7 @@ def get_repository_mirror_hostnames(self): for repository_mirror in repository_mirrors: mirror_configuration = repository_mirrors[repository_mirror] + url_prefix = mirror_configuration["url_prefix"] parsed_url = urlparse.urlparse(url_prefix) mirror_hostname = parsed_url.hostname From 572e0c6079a7967707291c1e07d92dc4d6bce42d Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Wed, 30 Jul 2014 15:31:35 -0400 Subject: [PATCH 10/52] Add interposition exception error classes in tuf.__init__ --- tuf/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tuf/__init__.py b/tuf/__init__.py index 4b9d3b89..8342bbe1 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -345,3 +345,13 @@ def __str__(self): all_errors += '\n ' + repr(mirror_netloc) + ': ' + repr(mirror_error) return all_errors + + +class NotFound(Error): + """If any configuration hostname or repository mirror hostname are not found""" + pass + + +class URLMatchesNoPattern(Error): + """URL matches no user-specified regular expression pattern""" + pass From 03164345e632b82d277dee5a7f3ca43a78638d07 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Thu, 31 Jul 2014 18:14:48 -0400 Subject: [PATCH 11/52] Add an InvalidConfiguration exception class in tuf.__init__ --- tuf/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tuf/__init__.py b/tuf/__init__.py index 8342bbe1..ca65f988 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -355,3 +355,8 @@ class NotFound(Error): class URLMatchesNoPattern(Error): """URL matches no user-specified regular expression pattern""" pass + + +class InvalidConfiguration(Error): + """If the configuration do not match a particular pattern then InvalidConfiguration Exception is raised""" + pass From da7a8faa49303328ad49d25b1a12fb04e336bb90 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Thu, 31 Jul 2014 18:24:44 -0400 Subject: [PATCH 12/52] Few changes in tuf.interposition.__init__ Made few changes after adding InvalidConfiguration Exception into tuf.__init__ Also made changes in comments --- tuf/interposition/__init__.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index be9ef62b..f9903e4b 100644 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -4,13 +4,15 @@ import socket import urllib import urllib2 +import logging +import tuf.log # We import them directly into our namespace so that there is no name conflict. -from configuration import ConfigurationParser, InvalidConfiguration -from utility import Logger -from updater import UpdaterController +from tuf.interposition.configuration import ConfigurationParser +from tuf.interposition.updater import UpdaterController +Logger = logging.getLogger('tuf.interposition.__init__') # Export nothing when: from tuf.interposition import * __all__ = [] @@ -196,7 +198,7 @@ def __read_configuration(configuration_handler, configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: - raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) + raise tuf.InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) else: for network_location, configuration in configurations.iteritems(): @@ -204,7 +206,11 @@ def __read_configuration(configuration_handler, configuration_parser = ConfigurationParser(network_location, configuration, parent_repository_directory=parent_repository_directory, parent_ssl_certificates_directory=parent_ssl_certificates_directory) - + + # configuration_parser.parse() returns + # tuf.interposition.Configuration which makes configuration an + # object. The integration of interposition is done on the basis + # of 'configuration' which is an object. configuration = configuration_parser.parse() configuration_handler(configuration) parsed_configurations[configuration.hostname] = configuration @@ -221,9 +227,6 @@ def __read_configuration(configuration_handler, return parsed_configurations - - - # TODO: Is parent_repository_directory a security risk? For example, would it # allow the user to overwrite another TUF repository metadata on the filesystem? # On the other hand, it is beyond TUF's scope to handle filesystem permissions. @@ -237,6 +240,9 @@ def configure(filename="tuf.interposition.json", containing parent directory of the "repository_directory" specified in a configuration for *all* network locations, because sometimes the absolute location of the "repository_directory" is only known at runtime. If you + containing parent directory of the "repository_directory" specified in a + configuration for *all* network locations, because sometimes the absolute + location of the "repository_directory" is only known at runtime. If you need to specify a different parent_repository_directory for other network locations, simply call this method again with different parameters. @@ -368,6 +374,3 @@ def wrapper(self, *args, **kwargs): __monkey_patch() - - - From 97d88d29c22823f72c5c8824a0e3fbca5f20a9eb Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Thu, 31 Jul 2014 18:32:02 -0400 Subject: [PATCH 13/52] Removed the repeated sentences in comments in tuf.interposition.__init__ --- tuf/interposition/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index f9903e4b..8871ae72 100644 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -240,9 +240,6 @@ def configure(filename="tuf.interposition.json", containing parent directory of the "repository_directory" specified in a configuration for *all* network locations, because sometimes the absolute location of the "repository_directory" is only known at runtime. If you - containing parent directory of the "repository_directory" specified in a - configuration for *all* network locations, because sometimes the absolute - location of the "repository_directory" is only known at runtime. If you need to specify a different parent_repository_directory for other network locations, simply call this method again with different parameters. From 0514e66658e4336a49715f009c6bf8d1ac4dc5c6 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Thu, 31 Jul 2014 18:37:56 -0400 Subject: [PATCH 14/52] Made following changes in tuf.interposition.updater Added comments in tuf.interposition.UpdaterController Changed the code depending on configuration.hostname to configuration.network_location --- tuf/interposition/updater.py | 171 +++++++++++++++++++++-------------- 1 file changed, 104 insertions(+), 67 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 3b8eab85..cc3a0c3d 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -133,7 +133,7 @@ import tuf.log # We import them directly into our namespace so that there is no name conflict. -from configuration import Configuration, InvalidConfiguration +from tuf.interposition.configuration import Configuration Logger = logging.getLogger('tuf.interposition.updater') @@ -547,8 +547,9 @@ class UpdaterController(object): __init__(): It creates and initializes an empty private map of updaters and an empty - private set of repository mirror hostnames. This is used to store the - updaters added by TUF and later on TUF can get these updater to reutilize. + private set of repository mirror network locations (hostname:port). This + is used to store the updaters added by TUF and later on TUF can get these + updater to reutilize. __check_configuration_on_add(configuration): It checks if the given configuration is valid or not. @@ -556,7 +557,7 @@ class UpdaterController(object): add(configuration): This method adds the updater by adding an object of tuf.interposition.Updater in the __updater map and by adding repository - mirror's hostname in the empty set initialized when the object of + mirror's network location in the empty set initialized when the object of tuf.interposition.UpdaterController is created. get(url): @@ -569,9 +570,9 @@ def __init__(self): """ To initalize a private map of updaters and a private set of repository - mirror hostnames once the object of tuf.interposition.UpdaterController - is created. This empty map and set is later used to add, get and remove - updaters and their mirrors. + mirror network locations (hostname:port) once the object of + tuf.interposition.UpdaterController is created. This empty map and set is + later used to add, get and remove updaters and their mirrors. None @@ -581,7 +582,7 @@ def __init__(self): An empty map called '__updaters' and an empty set called - '__repository_mirror_hostnames' is created. + '__repository_mirror_network_locations' is created. None @@ -590,31 +591,34 @@ def __init__(self): # A private map of Updaters (network_location: str -> updater: Updater) self.__updaters = {} - # A private set of repository mirror hostnames - self.__repository_mirror_hostnames = set() + # A private set of repository mirror network locations + self.__repository_mirror_network_locations = set() def __check_configuration_on_add(self, configuration): """ If the given Configuration is invalid, an exception is raised. - Otherwise, repository mirror hostnames are returned. + Otherwise, repository mirror network locations are returned. 'configuration' contains hostname, port number, repository mirrors which are to be checked if they are valid or not. - tuf.FormatError - This exception is raised if the configuration is invalid i.e. if the - hostname is not unique or configuration.hostname is same as - repository_mirror_hostname. + tuf.InvalidConfiguration: + If the configuration is invalid. For example - wrong hostname, invalid + port number, wrong mirror format. + + tuf.FormatError: + If the network_location is not unique or configuration.network_location + is same as repository_mirror_network_locations. It logs the error message. - 'repository_mirror_hostnames' + 'repository_mirror_network_locations' In order to prove that everything worked well, a part of configuration is returned which is the list of repository mirrors. """ @@ -637,38 +641,38 @@ def __check_configuration_on_add(self, configuration): # as an object which makes configuration an instance of # tuf.interposition.Configuration if not isinstance(configuration, Configuration): - raise tuf.FormatError("Invalid Configuration") + raise tuf.InvalidConfiguration("Invalid Configuration") - # Restrict each (incoming, outgoing) hostname pair to be unique across + # Restrict each (incoming, outgoing) network location pair to be unique across # configurations; this prevents interposition cycles, amongst other # things. # GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ... # BAD: A -> { B }, B -> { C }, C -> { A }, ... - if configuration.hostname in self.__updaters: - raise tuf.FormatError("Hostname Not Unique") - if configuration.hostname in self.__repository_mirror_hostnames: - raise tuf.FormatError("Hostname Already Exists") + if configuration.network_location in self.__updaters: + raise tuf.FormatError("Updater with "+repr(configuration.network_location)+" Already Exists as an updater") + if configuration.network_location in self.__repository_mirror_network_locations: + raise tuf.FormatError("Updater with "+repr(configuration.network_location)+" Already Exists as a mirror") # Check for redundancy in server repository mirrors. - repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() - for mirror_hostname in repository_mirror_hostnames: + for mirror_network_location in repository_mirror_network_locations: try: - # Restrict each hostname in every (incoming, outgoing) pair to be + # Restrict each network location in every (incoming, outgoing) pair to be # unique across configurations; this prevents interposition cycles, # amongst other things. - if mirror_hostname in self.__updaters: - raise tuf.FormatError("Mirror Hostname Not Unique") - if mirror_hostname in self.__repository_mirror_hostnames: - raise tuf.FormatError("Mirror Hostname Already Exists") + if mirror_network_location in self.__updaters: + raise tuf.FormatError("Mirror with "+repr(mirror_network_location)+" Already Exists as an updater") + if mirror_network_location in self.__repository_mirror_network_locations: + raise tuf.FormatError("Mirror with "+repr(mirror_network_location)+" Already Exists as a mirror") - except: + except (tuf.FormatError) as e: error_message = \ - INVALID_REPOSITORY_MIRROR.format(repository_mirror=mirror_hostname) + INVALID_REPOSITORY_MIRROR.format(repository_mirror=mirror_network_location) Logger.exception(error_message) - raise InvalidConfiguration(error_message) + raise - return repository_mirror_hostnames + return repository_mirror_network_locations def add(self, configuration): @@ -683,27 +687,27 @@ def add(self, configuration): tuf.FormatError - This exception is raised if the hostname which tuf is trying to add - is not unique. + This exception is raised if the network location which tuf is trying to + add is not unique. The object of tuf.interposition.Updater is added in the list of updaters. Also, the mirrors of this updater are added into a - repository_mirror_hostname are added. + repository_mirror_network_locations are added. None """ - repository_mirror_hostnames = self.__check_configuration_on_add(configuration) + repository_mirror_network_locations = self.__check_configuration_on_add(configuration) - # If all is well, build and store an Updater, and remember hostnames. + # If all is well, build and store an Updater, and remember network locations. Logger.info('Adding updater for interposed '+repr(configuration)) # Adding an object of the tuf.interposition.updater with the given # configuration. - self.__updaters[configuration.hostname] = Updater(configuration) - # Adding the new the repository mirror hostnames to the list. - self.__repository_mirror_hostnames.update(repository_mirror_hostnames) + self.__updaters[configuration.network_location] = Updater(configuration) + # Adding the new the repository mirror network locations to the list. + self.__repository_mirror_network_locations.update(repository_mirror_network_locations) def refresh(self, configuration): @@ -717,7 +721,7 @@ def refresh(self, configuration): to be refreshed. - tuf.FormatError: + tuf.InvalidConfiguration: If there is anything wrong with the Format of the configuration, this exception is raised. @@ -740,17 +744,17 @@ def refresh(self, configuration): # Check if the configuration is valid else raise an exception. if not isinstance(configuration, Configuration): - raise tuf.FormatError("Invalid Configuration") + raise tuf.InvalidConfiguration("Invalid Configuration") # Get the repository mirrors of the given configuration. - repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() - # Check if the configuration.hostname is available in the updater or mirror + # Check if the configuration.network_location is available in the updater or mirror # list. - if not configuration.hostname in self.__updaters: - raise tuf.NotFound("Hostname Not Found") - if not repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames): - raise tuf.NotFound("Hostname Not Found") + if not configuration.network_location in self.__updaters: + raise tuf.NotFound("Network Location Not Found") + if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): + raise tuf.NotFound("Network Location Not Found") # Get the updater and refresh its top-level metadata. In the majority of # integrations, a software updater integrating TUF with interposition will @@ -764,7 +768,7 @@ def refresh(self, configuration): # If everything is good then fetch the updater from __updaters with the # given configurations. - updater = self.__updaters.get(configuration.hostname) + updater = self.__updaters.get(configuration.network_location) # Refresh the fetched updater. updater.refresh() @@ -784,13 +788,15 @@ def get(self, url): string. - + None - + This method logs the messages in a log file if updater is not found or + not for the given url. - + The get() method returns the updater with the given configuration. If + updater does not exists, it returns None. """ GENERIC_WARNING_MESSAGE = "No updater or interposition for url={url}" @@ -801,10 +807,14 @@ def get(self, url): updater = None try: + # Parse the given url to access individual parts of it. parsed_url = urlparse.urlparse(url) hostname = parsed_url.hostname port = parsed_url.port or 80 netloc = parsed_url.netloc + + # Combine the hostname and port number and assign it to network_location. + # The combination of hostname and port is used to identify an updater. network_location = "{hostname}:{port}".format(hostname=hostname, port=port) # There can be a case when parsed_url.netloc does not have a port (e.g. @@ -812,7 +822,7 @@ def get(self, url): # the parameters. network_locations = set((netloc, network_location)) - updater = self.__updaters.get(hostname) + updater = self.__updaters.get(network_location) if updater is None: Logger.warn(HOSTNAME_NOT_FOUND_MESSAGE.format(hostname=hostname)) @@ -843,34 +853,61 @@ def get(self, url): return updater + def remove(self, configuration): """ - Remove an Updater matching the given Configuration. + Remove an Updater matching the given Configuration as well as its + associated mirrors. - - - + 'configuration' is the configuration object of the updater to be removed. + + tuf.InvalidConfiguration: + If there is anything wrong with the configuration for example invalid + hostname, invalid port number etc, tuf.InvalidConfiguration is raised. + + tuf.NotFound: + If the updater with the given configuration does not exists, + tuf.NotFound exception is raised. + + + Removes the stored updater and the mirrors associated with that updater. + Then tuf logs this information in a log file. + + + None """ UPDATER_REMOVED_MESSAGE = "Updater removed for interposed {configuration}." - assert isinstance(configuration, Configuration) + # Check if the given configuration is valid or not. + if not isinstance(configuration, Configuration): + raise tuf.InvalidConfiguration('Invalid Configuration') + + # If the configuration is valid, get the repository mirrors associated with + # it. + repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() - repository_mirror_hostnames = configuration.get_repository_mirror_hostnames() + # Check if network location of the given configuration exists or not. + if configuration.network_location not in self.__updaters: + raise tuf.NotFound('Network Location Not Found') - assert configuration.hostname in self.__updaters - assert repository_mirror_hostnames.issubset(self.__repository_mirror_hostnames) + # Check if the associated mirrors exists or not. + if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): + raise tuf.NotFound('Repository Mirror Does Not Exists') # Get the updater. - updater = self.__updaters.get(configuration.hostname) + updater = self.__updaters.get(configuration.network_location) - # If all is well, remove the stored Updater as well as its associated - # repository mirror hostnames. + # If everything works well, remove the stored Updater as well as its + # associated repository mirror network locations. updater.cleanup() - del self.__updaters[configuration.hostname] - self.__repository_mirror_hostnames.difference_update(repository_mirror_hostnames) + # Delete the updater from the list of updaters. + del self.__updaters[configuration.network_location] + # Remove the associated mirrors from the repository mirror set. + self.__repository_mirror_network_locations.difference_update(repository_mirror_network_locations) + # Log the message that the given updater is removed. Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration)) From 513e051296fd0cf30e006dda31d0dee70c49af55 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Thu, 31 Jul 2014 18:43:15 -0400 Subject: [PATCH 15/52] Changes in tuf.interposition.configuration Instead of making a set of repositiory mirror hostname, a set of repository mirror network location is created. --- tuf/interposition/configuration.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tuf/interposition/configuration.py b/tuf/interposition/configuration.py index d4bfeeb1..6461c220 100644 --- a/tuf/interposition/configuration.py +++ b/tuf/interposition/configuration.py @@ -15,6 +15,7 @@ import os.path import types import urlparse +import logging import tuf.log @@ -61,7 +62,10 @@ def get_repository_mirror_hostnames(self): url_prefix = mirror_configuration["url_prefix"] parsed_url = urlparse.urlparse(url_prefix) mirror_hostname = parsed_url.hostname - repository_mirror_hostnames.add(mirror_hostname) + mirror_port = parsed_url.port + mirror_network_location = \ + "{hostname}:{port}".format(hostname=mirror_hostname, port = mirror_port) + repository_mirror_hostnames.add(mirror_network_location) return repository_mirror_hostnames @@ -96,7 +100,7 @@ def get_network_location(self): if len(network_location_tokens) > 1: port = int(network_location_tokens[1], 10) if port <= 0 or port >= 2**16: - raise InvalidConfiguration(INVALID_NETWORK_LOCATION.format( + raise tuf.InvalidConfiguration(INVALID_NETWORK_LOCATION.format( network_location=self.network_location)) return hostname, port @@ -120,7 +124,7 @@ def get_repository_directory(self): # TODO: assert os.path.isdir(repository_directory) else: - raise InvalidConfiguration(INVALID_PARENT_REPOSITORY_DIRECTORY.format( + raise tuf.InvalidConfiguration(INVALID_PARENT_REPOSITORY_DIRECTORY.format( network_location=self.network_location)) return repository_directory @@ -146,11 +150,11 @@ def get_ssl_certificates(self): ssl_certificates) if not os.path.isfile(ssl_certificates): - raise InvalidConfiguration(INVALID_SSL_CERTIFICATES.format( + raise tuf.InvalidConfiguration(INVALID_SSL_CERTIFICATES.format( network_location=self.network_location)) else: - raise InvalidConfiguration( + raise tuf.InvalidConfiguration( INVALID_PARENT_SSL_CERTIFICATES_DIRECTORY.format( network_location=self.network_location)) @@ -199,7 +203,7 @@ def get_repository_mirrors(self, hostname, port, ssl_certificates): error_message = \ INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror) Logger.exception(error_message) - raise InvalidConfiguration(error_message) + raise tuf.InvalidConfiguration(error_message) return repository_mirrors @@ -233,7 +237,7 @@ def get_target_paths(self): error_message = \ INVALID_TARGET_PATH.format(network_location=self.network_location) Logger.exception(error_message) - raise InvalidConfiguration(error_message) + raise tuf.InvalidConfiguration(error_message) return target_paths From b1636a062a94304d3f1194396555982eb7041ec4 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Fri, 1 Aug 2014 12:05:15 -0400 Subject: [PATCH 16/52] Change tuf.interposition.UpdaterController to tuf.interposition.updater.UpdaterController --- tuf/interposition/updater.py | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index cc3a0c3d..172777b6 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -52,7 +52,7 @@ # Second, configure(filename="/path/to/json") # Configure() returns a dictionary of configurations # Internally, configure() calls add(configuration) function which is in - # the tuf.interposition.UpdaterController. + # the tuf.interposition.updater.UpdaterController. configurations = tuf.interposition.configure() url = 'http://example.com/path/to/document' @@ -65,7 +65,7 @@ # remove the updater object. # Deconfigure() takes only one argument i.e. configurations. # It calls remove(configuration) function which is in - # tuf.interposition.UpdaterController. + # tuf.interposition.updater.UpdaterController. tuf.interposition.deconfigure(configurations) @@ -173,8 +173,8 @@ class Updater(object): retrieve(url, filename, reporthook, data): retrieve() method first get the target file path by calling - get_target_filepath(url) which in tuf.interposition.updater and then - calls download_target() method for the above file path. + get_target_filepath(url) which in tuf.interposition.updater.Updater and + then calls download_target() method for the above file path. switch_context(): There is an updater object for each network location that is interposed. @@ -189,7 +189,7 @@ def __init__(self, configuration): Constructor. Instantiating an updater object causes creation of a temporary directory. This temporary directory is used for the - tuf.interposition.updater. After that the tuf.client.updater module which + tuf.interposition.updater.Updater. After that the tuf.client.updater module which performs the low-level integration is called. @@ -466,8 +466,8 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): """ retrieve() method first get the target file path by calling - get_target_filepath(url) which in tuf.interposition.updater and then - calls download_target() method for the above file path. + get_target_filepath(url) which is in tuf.interposition.updater.Updater + and then calls download_target() method for the above file path. url, which is to be retrieved. @@ -539,7 +539,7 @@ def switch_context(self): class UpdaterController(object): """ - tuf.interposition.UpdaterController is a controller of the Updaters. + tuf.interposition.updater.UpdaterController is a controller of the Updaters. Given a configuration, it can build and store an Updater, which can be used later with the help of get() method. @@ -556,9 +556,9 @@ class UpdaterController(object): add(configuration): This method adds the updater by adding an object of - tuf.interposition.Updater in the __updater map and by adding repository + tuf.interposition.updater.Updater in the __updater map and by adding repository mirror's network location in the empty set initialized when the object of - tuf.interposition.UpdaterController is created. + tuf.interposition.updater.UpdaterController is created. get(url): refresh(configuration): @@ -571,7 +571,7 @@ def __init__(self): To initalize a private map of updaters and a private set of repository mirror network locations (hostname:port) once the object of - tuf.interposition.UpdaterController is created. This empty map and set is + tuf.interposition.updater.UpdaterController is created. This empty map and set is later used to add, get and remove updaters and their mirrors. @@ -627,19 +627,19 @@ def __check_configuration_on_add(self, configuration): # Updater has a "global" view of configurations, so it performs # additional checks after Configuration's own local checks. This will - # check if everything in tuf.interposition.ConfigurationParser worked - # or not. + # check if everything in tuf.interposition.configuration.ConfigurationParser + # worked or not. # According to __read_configuration() method in - # tuf.interposition.__init__.py, - # configuration is an instance of tuf.interposition.Configuration because - # in this method - + # tuf.interposition.__init__, + # configuration is an instance of + # tuf.interposition.configuration.Configuration because in this method - # configuration = configuration_parser.parse() # configuration_parser is an instance of - # tuf.interposition.ConfigurationParser - # The configuration_parser.parse() returns tuf.interposition.Configuration - # as an object which makes configuration an instance of - # tuf.interposition.Configuration + # tuf.interposition.configuration.ConfigurationParser + # The configuration_parser.parse() returns + # tuf.interposition.configuration.Configuration as an object which makes + # configuration an instance of tuf.interposition.configuration.Configuration if not isinstance(configuration, Configuration): raise tuf.InvalidConfiguration("Invalid Configuration") @@ -691,7 +691,7 @@ def add(self, configuration): add is not unique. - The object of tuf.interposition.Updater is added in the list of updaters. + The object of tuf.interposition.updater.Updater is added in the list of updaters. Also, the mirrors of this updater are added into a repository_mirror_network_locations are added. @@ -703,7 +703,7 @@ def add(self, configuration): # If all is well, build and store an Updater, and remember network locations. Logger.info('Adding updater for interposed '+repr(configuration)) - # Adding an object of the tuf.interposition.updater with the given + # Adding an object of the tuf.interposition.updater.Updater with the given # configuration. self.__updaters[configuration.network_location] = Updater(configuration) # Adding the new the repository mirror network locations to the list. From 9aacb66d412fcad513886db4b65d06645cb8d9ce Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Fri, 1 Aug 2014 13:32:14 -0400 Subject: [PATCH 17/52] Changes in error name in tuf.interposition.updater --- tuf/interposition/updater.py | 48 +++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 172777b6..ce0a0aad 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -93,7 +93,7 @@ "repository_directory": ".", # Where do we forward the requests to localhost? "repository_mirrors" : { - gg "mirror1": { + "mirror1": { # In this case, we forward them to http://localhost:8001 "url_prefix": "http://localhost:8001", # You do not have to worry about these default parameters. @@ -415,7 +415,7 @@ def get_target_filepath(self, source_url): # If source_url does not match any regular expression... if target_filepath is None: # ...then we raise a predictable exception. - raise tuf.URLMatchesNoPattern(source_url) + raise tuf.URLMatchesNoPatternError(source_url) except: Logger.exception(WARNING_MESSAGE.format( @@ -606,7 +606,7 @@ def __check_configuration_on_add(self, configuration): are to be checked if they are valid or not. - tuf.InvalidConfiguration: + tuf.InvalidConfigurationError: If the configuration is invalid. For example - wrong hostname, invalid port number, wrong mirror format. @@ -641,7 +641,7 @@ def __check_configuration_on_add(self, configuration): # tuf.interposition.configuration.Configuration as an object which makes # configuration an instance of tuf.interposition.configuration.Configuration if not isinstance(configuration, Configuration): - raise tuf.InvalidConfiguration("Invalid Configuration") + raise tuf.InvalidConfigurationError("Invalid configuration") # Restrict each (incoming, outgoing) network location pair to be unique across # configurations; this prevents interposition cycles, amongst other @@ -649,9 +649,10 @@ def __check_configuration_on_add(self, configuration): # GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ... # BAD: A -> { B }, B -> { C }, C -> { A }, ... if configuration.network_location in self.__updaters: - raise tuf.FormatError("Updater with "+repr(configuration.network_location)+" Already Exists as an updater") + raise tuf.FormatError("Updater with "+repr(configuration.network_location)+" already exists as an updater") + if configuration.network_location in self.__repository_mirror_network_locations: - raise tuf.FormatError("Updater with "+repr(configuration.network_location)+" Already Exists as a mirror") + raise tuf.FormatError("Updater with "+repr(configuration.network_location)+" already exists as a mirror") # Check for redundancy in server repository mirrors. repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() @@ -662,9 +663,9 @@ def __check_configuration_on_add(self, configuration): # unique across configurations; this prevents interposition cycles, # amongst other things. if mirror_network_location in self.__updaters: - raise tuf.FormatError("Mirror with "+repr(mirror_network_location)+" Already Exists as an updater") + raise tuf.FormatError("Mirror with "+repr(mirror_network_location)+" already exists as an updater") if mirror_network_location in self.__repository_mirror_network_locations: - raise tuf.FormatError("Mirror with "+repr(mirror_network_location)+" Already Exists as a mirror") + raise tuf.FormatError("Mirror with "+repr(mirror_network_location)+" already exists as a mirror") except (tuf.FormatError) as e: error_message = \ @@ -721,13 +722,13 @@ def refresh(self, configuration): to be refreshed. - tuf.InvalidConfiguration: + tuf.InvalidConfigurationError: If there is anything wrong with the Format of the configuration, this exception is raised. - tuf.NotFound: - If the updater to be refreshed is not found in the list of updaters or - mirrors, then tuf.NotFound exception is raised. + tuf.NotFoundError: + If the updater to be refreshed is not found in the list of updaters or + mirrors, then tuf.NotFoundError exception is raised. tuf.NoWorkingMirrorError: If the metadata for any of the top-level roles cannot be updated. @@ -744,17 +745,17 @@ def refresh(self, configuration): # Check if the configuration is valid else raise an exception. if not isinstance(configuration, Configuration): - raise tuf.InvalidConfiguration("Invalid Configuration") + raise tuf.InvalidConfigurationError("Invalid configuration") # Get the repository mirrors of the given configuration. repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() # Check if the configuration.network_location is available in the updater or mirror # list. - if not configuration.network_location in self.__updaters: - raise tuf.NotFound("Network Location Not Found") + if configuration.network_location not in self.__updaters: + raise tuf.NotFoundError("Network location not found") if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): - raise tuf.NotFound("Network Location Not Found") + raise tuf.NotFoundError("Network location not found") # Get the updater and refresh its top-level metadata. In the majority of # integrations, a software updater integrating TUF with interposition will @@ -864,13 +865,14 @@ def remove(self, configuration): 'configuration' is the configuration object of the updater to be removed. - tuf.InvalidConfiguration: + tuf.InvalidConfigurationError: If there is anything wrong with the configuration for example invalid - hostname, invalid port number etc, tuf.InvalidConfiguration is raised. + hostname, invalid port number etc, tuf.InvalidConfigurationError is + raised. - tuf.NotFound: + tuf.NotFoundError: If the updater with the given configuration does not exists, - tuf.NotFound exception is raised. + tuf.NotFoundError exception is raised. Removes the stored updater and the mirrors associated with that updater. @@ -884,7 +886,7 @@ def remove(self, configuration): # Check if the given configuration is valid or not. if not isinstance(configuration, Configuration): - raise tuf.InvalidConfiguration('Invalid Configuration') + raise tuf.InvalidConfigurationError('Invalid configuration') # If the configuration is valid, get the repository mirrors associated with # it. @@ -892,11 +894,11 @@ def remove(self, configuration): # Check if network location of the given configuration exists or not. if configuration.network_location not in self.__updaters: - raise tuf.NotFound('Network Location Not Found') + raise tuf.NotFoundError('Network location not found') # Check if the associated mirrors exists or not. if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): - raise tuf.NotFound('Repository Mirror Does Not Exists') + raise tuf.NotFoundError('Repository mirror does not exists') # Get the updater. updater = self.__updaters.get(configuration.network_location) From 750b9b3beafe4049086d753644bea959bc59fd49 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Fri, 1 Aug 2014 13:40:08 -0400 Subject: [PATCH 18/52] Changes in error name in tuf.interposition.__init__ --- tuf/interposition/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index 8871ae72..d7844eee 100644 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -198,7 +198,7 @@ def __read_configuration(configuration_handler, configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: - raise tuf.InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) + raise tuf.InvalidConfigurationError(NO_CONFIGURATIONS.format(filename=filename)) else: for network_location, configuration in configurations.iteritems(): From d2bb3cb136b1d8491343f65c991fac5f092b884a Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Fri, 1 Aug 2014 13:41:48 -0400 Subject: [PATCH 19/52] Changes in error name in tuf.interposition.configuration --- tuf/interposition/configuration.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tuf/interposition/configuration.py b/tuf/interposition/configuration.py index 6461c220..bb36491e 100644 --- a/tuf/interposition/configuration.py +++ b/tuf/interposition/configuration.py @@ -100,7 +100,7 @@ def get_network_location(self): if len(network_location_tokens) > 1: port = int(network_location_tokens[1], 10) if port <= 0 or port >= 2**16: - raise tuf.InvalidConfiguration(INVALID_NETWORK_LOCATION.format( + raise tuf.InvalidConfigurationError(INVALID_NETWORK_LOCATION.format( network_location=self.network_location)) return hostname, port @@ -124,7 +124,7 @@ def get_repository_directory(self): # TODO: assert os.path.isdir(repository_directory) else: - raise tuf.InvalidConfiguration(INVALID_PARENT_REPOSITORY_DIRECTORY.format( + raise tuf.InvalidConfigurationError(INVALID_PARENT_REPOSITORY_DIRECTORY.format( network_location=self.network_location)) return repository_directory @@ -150,11 +150,11 @@ def get_ssl_certificates(self): ssl_certificates) if not os.path.isfile(ssl_certificates): - raise tuf.InvalidConfiguration(INVALID_SSL_CERTIFICATES.format( + raise tuf.InvalidConfigurationError(INVALID_SSL_CERTIFICATES.format( network_location=self.network_location)) else: - raise tuf.InvalidConfiguration( + raise tuf.InvalidConfigurationError( INVALID_PARENT_SSL_CERTIFICATES_DIRECTORY.format( network_location=self.network_location)) @@ -203,7 +203,7 @@ def get_repository_mirrors(self, hostname, port, ssl_certificates): error_message = \ INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror) Logger.exception(error_message) - raise tuf.InvalidConfiguration(error_message) + raise tuf.InvalidConfigurationError(error_message) return repository_mirrors @@ -237,7 +237,7 @@ def get_target_paths(self): error_message = \ INVALID_TARGET_PATH.format(network_location=self.network_location) Logger.exception(error_message) - raise tuf.InvalidConfiguration(error_message) + raise tuf.InvalidConfigurationError(error_message) return target_paths From 56b942a3418cba7d4904c85061eb4ad92cfa2984 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Fri, 1 Aug 2014 13:46:26 -0400 Subject: [PATCH 20/52] Changes in the error name in tuf.__init__ --- tuf/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index ca65f988..46c3fc94 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -347,16 +347,16 @@ def __str__(self): return all_errors -class NotFound(Error): +class NotFoundError(Error): """If any configuration hostname or repository mirror hostname are not found""" pass -class URLMatchesNoPattern(Error): +class URLMatchesNoPatternError(Error): """URL matches no user-specified regular expression pattern""" pass -class InvalidConfiguration(Error): +class InvalidConfigurationError(Error): """If the configuration do not match a particular pattern then InvalidConfiguration Exception is raised""" pass From 2a83d83ad70b222b0a1efe6abc66d51be7cfa57d Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Mon, 4 Aug 2014 14:18:43 -0400 Subject: [PATCH 21/52] Extend purpose of tuf.interposition.updater and change format style of Logger format in Logger is changed to 'msg'+repr() --- tuf/interposition/updater.py | 117 ++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 50 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index ce0a0aad..b6a2fe09 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -19,8 +19,24 @@ that all the processes which are taking place in the low-level integration will be done automatically. This layer of processes will be transparent to the client. + Updater.py have two classes named as Updater and UpdaterController. - #TODO: Add more description to purpose. + + tuf.interposition.updater.Updater contains those methods which are to be + performed on each individual updater. For example - refresh(), cleanup(), + download_target(target_filepath), get_target_filepath(source_url), open(url), + retrieve(url), switch_context(), all these for a particular updater. + + tuf.interposition.updater.UpdaterController contains those methods which + are performed on updaters as a group. It basically keeps track of all the + updaters. For example - add(configuration), get(configuration), + refresh(configuration), remove(configuration), all these are performed on the + list of updaters. tuf.interposition.updater.UpdaterController maintains a + map of updaters and a set of its mirrors. The map of updaters contains the + objects of tuf.interposition.updater.Updater for each updater. The set + contains all the mirrors. The addition and removal of these updaters and thei + mirrors depends on the methods of tuf.interposition.updater.UpdaterController. + #TODO: Add Pros and Cons of using interposition. @@ -222,12 +238,10 @@ def __init__(self, configuration): None. """ - CREATED_TEMPDIR_MESSAGE = "Created temporary directory at {tempdir}" - self.configuration = configuration # A temporary directory used for this updater over runtime. self.tempdir = tempfile.mkdtemp() - Logger.debug(CREATED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir)) + Logger.debug('Created temporary directory at '+repr(self.tempdir)) # Switching context before instantiating updater because updater depends # on some module (tuf.conf) variables. @@ -291,10 +305,9 @@ def cleanup(self): result of download. It then prints a message of deletion and also mentions the name of the deleted directory. """ - - DELETED_TEMPDIR_MESSAGE = "Deleted temporary directory at {tempdir}" + shutil.rmtree(self.tempdir) - Logger.debug(DELETED_TEMPDIR_MESSAGE.format(tempdir=self.tempdir)) + Logger.debug('Deleted temporary directory at '+repr(self.tempdir)) def download_target(self, target_filepath): @@ -328,7 +341,6 @@ def download_target(self, target_filepath): It returns destination_directory where the target is been stored and filename of the target file been stored in the directory. - """ # Download file into a temporary directory shared over runtime @@ -383,11 +395,7 @@ def get_target_filepath(self, source_url): It returns target_filepath. This is the target which TUF should download. - - """ - - WARNING_MESSAGE = "Possibly invalid target_paths for " + \ - "{network_location}! No TUF interposition for {url}" + """ parsed_source_url = urlparse.urlparse(source_url) target_filepath = None @@ -418,8 +426,9 @@ def get_target_filepath(self, source_url): raise tuf.URLMatchesNoPatternError(source_url) except: - Logger.exception(WARNING_MESSAGE.format( - network_location=self.configuration.network_location, url=source_url)) + Logger.exception('Possibly invalid target_paths for '+ \ + repr(self.configuration.network_location)+'! No TUF interposition for '\ + + repr(source_url)) raise else: @@ -431,14 +440,14 @@ def open(self, url, data=None): """ Open the URL url which can either be a string or a request object. - The file is opened in the binary read mode as a temporary file. + The file is opened in the binary read mode as a temporary file. This is + called when TUF wants to open an already existing updater's 'url'. url, the one which is to be opened. data must be a bytes object specifying additional data to be sent to the server or None, if no such data needed. - """ filename, headers = self.retrieve(url, data=data) @@ -477,11 +486,9 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): It returns the filename and the headers of the file just retrieved. - """ - INTERPOSITION_MESSAGE = "Interposing for {url}" - - Logger.info(INTERPOSITION_MESSAGE.format(url=url)) + + Logger.info('Interposing for '+ repr(url)) # What is the actual target to download given the URL? Sometimes we would # like to transform the given URL to the intended target; e.g. "/simple/" @@ -526,6 +533,7 @@ def switch_context(self): 1. Setting local repository directory 2. Setting the local SSL certificate PEM file """ + # Set the local repository directory containing the metadata files. tuf.conf.repository_directory = self.configuration.repository_directory @@ -534,8 +542,6 @@ def switch_context(self): - - class UpdaterController(object): """ @@ -556,14 +562,24 @@ class UpdaterController(object): add(configuration): This method adds the updater by adding an object of - tuf.interposition.updater.Updater in the __updater map and by adding repository - mirror's network location in the empty set initialized when the object of - tuf.interposition.updater.UpdaterController is created. + tuf.interposition.updater.Updater in the __updater map and by adding + repository mirror's network location in the empty set initialized when + the object of tuf.interposition.updater.UpdaterController is created. get(url): - refresh(configuration): - remove(configuration): + This method is to get the updater if it already exists. It takes the url + and parse it. Then it utilizes hostname and port of that url to check if + it already exists or not. If the updater exists, then it calls the + get_target_filepath() method which returns a target file path to be + downloaded. + refresh(configuration): + To refresh the top-level metadata of the given 'configuration'. + It updates the latest copies of the metadata for the top-level roles. + + remove(configuration): + Remove an Updater matching the given Configuration as well as its + associated mirrors. """ def __init__(self): @@ -571,8 +587,8 @@ def __init__(self): To initalize a private map of updaters and a private set of repository mirror network locations (hostname:port) once the object of - tuf.interposition.updater.UpdaterController is created. This empty map and set is - later used to add, get and remove updaters and their mirrors. + tuf.interposition.updater.UpdaterController is created. This empty map + and set is later used to add, get and remove updaters and their mirrors. None @@ -623,8 +639,6 @@ def __check_configuration_on_add(self, configuration): is returned which is the list of repository mirrors. """ - INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!" - # Updater has a "global" view of configurations, so it performs # additional checks after Configuration's own local checks. This will # check if everything in tuf.interposition.configuration.ConfigurationParser @@ -669,7 +683,7 @@ def __check_configuration_on_add(self, configuration): except (tuf.FormatError) as e: error_message = \ - INVALID_REPOSITORY_MIRROR.format(repository_mirror=mirror_network_location) + 'Invalid repository mirror '+repr(mirror_network_location) Logger.exception(error_message) raise @@ -687,7 +701,11 @@ def add(self, configuration): updater will be added. - tuf.FormatError + tuf.InvalidConfigurationError: + If the configuration is invalid. For example - wrong hostname, invalid + port number, wrong mirror format. + + tuf.FormatError: This exception is raised if the network location which tuf is trying to add is not unique. @@ -753,9 +771,10 @@ def refresh(self, configuration): # Check if the configuration.network_location is available in the updater or mirror # list. if configuration.network_location not in self.__updaters: - raise tuf.NotFoundError("Network location not found") + raise tuf.NotFoundError("Updater with "+repr(configuration.network_location)+" not found") + if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): - raise tuf.NotFoundError("Network location not found") + raise tuf.NotFoundError("Mirror with "+repr(repository_mirror_network_locations)+" not found") # Get the updater and refresh its top-level metadata. In the majority of # integrations, a software updater integrating TUF with interposition will @@ -800,11 +819,6 @@ def get(self, url): updater does not exists, it returns None. """ - GENERIC_WARNING_MESSAGE = "No updater or interposition for url={url}" - DIFFERENT_NETLOC_MESSAGE = "We have an updater for netloc={netloc1} but not for netlocs={netloc2}" - HOSTNAME_FOUND_MESSAGE = "Found updater for interposed network location: {netloc}" - HOSTNAME_NOT_FOUND_MESSAGE = "No updater for hostname={hostname}" - updater = None try: @@ -826,13 +840,15 @@ def get(self, url): updater = self.__updaters.get(network_location) if updater is None: - Logger.warn(HOSTNAME_NOT_FOUND_MESSAGE.format(hostname=hostname)) + Logger.warn('No updater for '+repr(hostname)) else: # Ensure that the updater is meant for this (hostname, port). if updater.configuration.network_location in network_locations: - Logger.info(HOSTNAME_FOUND_MESSAGE.format(netloc=network_location)) + Logger.info('Found updater for interposed network location: '+ \ + repr(network_location)) + # Raises an exception in case we do not recognize how to # transform this URL for TUF. In that case, there will be no # updater for this URL. @@ -840,17 +856,18 @@ def get(self, url): else: # Same hostname, but different (not user-specified) port. - Logger.warn(DIFFERENT_NETLOC_MESSAGE.format( - netloc1=updater.configuration.network_location, netloc2=network_locations)) + Logger.warn('We have an updater for '+ \ + repr(updater.configuration.network_location)+ \ + 'but not for '+ repr(network_locations)) updater = None except: - Logger.exception(GENERIC_WARNING_MESSAGE.format(url=url)) + Logger.exception('No updater or interposition for '+ repr(url)) updater = None finally: if updater is None: - Logger.warn(GENERIC_WARNING_MESSAGE.format(url=url)) + Logger.warn('No updater or interposition for '+ repr(url)) return updater @@ -882,8 +899,6 @@ def remove(self, configuration): None """ - UPDATER_REMOVED_MESSAGE = "Updater removed for interposed {configuration}." - # Check if the given configuration is valid or not. if not isinstance(configuration, Configuration): raise tuf.InvalidConfigurationError('Invalid configuration') @@ -906,10 +921,12 @@ def remove(self, configuration): # If everything works well, remove the stored Updater as well as its # associated repository mirror network locations. updater.cleanup() + # Delete the updater from the list of updaters. del self.__updaters[configuration.network_location] + # Remove the associated mirrors from the repository mirror set. self.__repository_mirror_network_locations.difference_update(repository_mirror_network_locations) # Log the message that the given updater is removed. - Logger.info(UPDATER_REMOVED_MESSAGE.format(configuration=configuration)) + Logger.info('Updater removed for interposed '+ repr(configuration)) From 1f9d7e6a70a4dff35beb6e17232baf4ca431cf6b Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Mon, 4 Aug 2014 16:07:53 -0400 Subject: [PATCH 22/52] Add exceptions to the methods of tuf.interposition.updater.Updater --- tuf/interposition/updater.py | 107 +++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index b6a2fe09..36cd993b 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -228,8 +228,20 @@ def __init__(self, configuration): 'confined_target_dirs': ['']}} - #TODO: Exceptions + tuf.FormatError: + If the arguments of tuf.client.updater.Updater are improperly formatted. + + tuf.RepositoryError: + If there is an error with the updater's repository files, such + as a missing 'root.json' file. + tuf.NoWorkingMirrorError: + If while refreshing, the metadata for any of the top-level roles cannot + be updated. + + tuf.ExpiredMetadataError: + While refreshing, if any metadata has expired. + The metadata files (e.g., 'root.json', 'targets.json') for the top-level roles are read from disk and stored in dictionaries. @@ -303,7 +315,19 @@ def cleanup(self): It will clean up all the temporary directories which were made as a result of download. It then prints a message of deletion and also - mentions the name of the deleted directory. + mentions the name of the deleted directory. + + + None + + + None + + + Removal of the temporary directory. + + + None """ shutil.rmtree(self.tempdir) @@ -333,7 +357,16 @@ def download_target(self, target_filepath): 'target_filepath' is the target's relative path on the remote repository. - #TODO: Exceptions + tuf.FormatError: + If 'target_filepath', 'updated_target' in + tuf.client.updater.download_target and arguments of updated_targets are + improperly formatted. + + tuf.UnknownTargetError: + If 'target_filepath' was not found. + + tuf.NoWorkingMirrorError: + If a 'target_filepath' could not be downloaded from any of the mirrors. A target file is saved to the local system. @@ -393,6 +426,14 @@ def get_target_filepath(self, source_url): source_url is passed while calling the function. This is the url which we want to retrieve. For this url, get_target_filepath() method is called. + + tuf.URLMatchesNoPatternError: + This exception is raised when no target_path url pattern is wrong and + does match regular expression. + + + None + It returns target_filepath. This is the target which TUF should download. """ @@ -447,7 +488,30 @@ def open(self, url, data=None): url, the one which is to be opened. data must be a bytes object specifying additional data to be sent to the - server or None, if no such data needed. + server or None, if no such data needed. + + + tuf.FormatError: + If 'target_filepath', 'updated_target' in + tuf.client.updater.download_target and arguments of updated_targets are + improperly formatted. + + tuf.UnknownTargetError: + If 'target_filepath' was not found. + + tuf.NoWorkingMirrorError: + If a 'target_filepath' could not be downloaded from any of the mirrors. + + tuf.URLMatchesNoPatternError: + This exception is raised when no target_path url pattern is wrong and + does match regular expression. + + + None + + + #TODO: + 'response' """ filename, headers = self.retrieve(url, data=data) @@ -483,7 +547,27 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): filename, if the is given then everywhere the given filename is used. If the filename is none, then temporary file is used. - + + + tuf.FormatError: + If 'target_filepath', 'updated_target' in + tuf.client.updater.download_target and arguments of updated_targets are + improperly formatted. + + tuf.UnknownTargetError: + If 'target_filepath' was not found. + + tuf.NoWorkingMirrorError: + If a 'target_filepath' could not be downloaded from any of the mirrors. + + tuf.URLMatchesNoPatternError: + This exception is raised when no target_path url pattern is wrong and + does match regular expression. + + + A target file is saved to the local system when the + download_target(target_filepath) is called. + It returns the filename and the headers of the file just retrieved. """ @@ -532,6 +616,19 @@ def switch_context(self): For this, two settings are required - 1. Setting local repository directory 2. Setting the local SSL certificate PEM file + + + None + + + None + + + The given configuration's repository_directory and ssl_certificates are + assigned to tuf.conf.repository_directory and tuf.conf.ssl_certificates. + + + None """ # Set the local repository directory containing the metadata files. From a51454618794b861596d4a4b8ee924be12f0fd7c Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Mon, 4 Aug 2014 16:18:53 -0400 Subject: [PATCH 23/52] Docstrings of tuf.interposition.updater completed --- tuf/interposition/updater.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 36cd993b..7e928eaf 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -34,7 +34,7 @@ list of updaters. tuf.interposition.updater.UpdaterController maintains a map of updaters and a set of its mirrors. The map of updaters contains the objects of tuf.interposition.updater.Updater for each updater. The set - contains all the mirrors. The addition and removal of these updaters and thei + contains all the mirrors. The addition and removal of these updaters and their mirrors depends on the methods of tuf.interposition.updater.UpdaterController. #TODO: Add Pros and Cons of using interposition. @@ -253,7 +253,7 @@ def __init__(self, configuration): self.configuration = configuration # A temporary directory used for this updater over runtime. self.tempdir = tempfile.mkdtemp() - Logger.debug('Created temporary directory at '+repr(self.tempdir)) + Logger.debug('Created temporary directory at '+ repr(self.tempdir)) # Switching context before instantiating updater because updater depends # on some module (tuf.conf) variables. @@ -272,7 +272,7 @@ def __init__(self, configuration): # Update the client's top-level metadata. The download_target() method does # not automatically refresh top-level prior to retrieving target files and # their associated Targets metadata, so update the top-level metadata here. - Logger.info('Refreshing top-level metadata for interposed '+repr(configuration)) + Logger.info('Refreshing top-level metadata for interposed '+ repr(configuration)) self.updater.refresh() @@ -331,7 +331,7 @@ def cleanup(self): """ shutil.rmtree(self.tempdir) - Logger.debug('Deleted temporary directory at '+repr(self.tempdir)) + Logger.debug('Deleted temporary directory at '+ repr(self.tempdir)) def download_target(self, target_filepath): @@ -510,8 +510,7 @@ def open(self, url, data=None): None - #TODO: - 'response' + 'response' which is a file object with info() and geturl() methods added. """ filename, headers = self.retrieve(url, data=data) @@ -760,10 +759,10 @@ def __check_configuration_on_add(self, configuration): # GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ... # BAD: A -> { B }, B -> { C }, C -> { A }, ... if configuration.network_location in self.__updaters: - raise tuf.FormatError("Updater with "+repr(configuration.network_location)+" already exists as an updater") + raise tuf.FormatError("Updater with "+ repr(configuration.network_location)+" already exists as an updater") if configuration.network_location in self.__repository_mirror_network_locations: - raise tuf.FormatError("Updater with "+repr(configuration.network_location)+" already exists as a mirror") + raise tuf.FormatError("Updater with "+ repr(configuration.network_location)+" already exists as a mirror") # Check for redundancy in server repository mirrors. repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() @@ -774,13 +773,13 @@ def __check_configuration_on_add(self, configuration): # unique across configurations; this prevents interposition cycles, # amongst other things. if mirror_network_location in self.__updaters: - raise tuf.FormatError("Mirror with "+repr(mirror_network_location)+" already exists as an updater") + raise tuf.FormatError("Mirror with "+ repr(mirror_network_location)+" already exists as an updater") if mirror_network_location in self.__repository_mirror_network_locations: - raise tuf.FormatError("Mirror with "+repr(mirror_network_location)+" already exists as a mirror") + raise tuf.FormatError("Mirror with "+ repr(mirror_network_location)+" already exists as a mirror") except (tuf.FormatError) as e: error_message = \ - 'Invalid repository mirror '+repr(mirror_network_location) + 'Invalid repository mirror '+ repr(mirror_network_location) Logger.exception(error_message) raise @@ -818,7 +817,7 @@ def add(self, configuration): repository_mirror_network_locations = self.__check_configuration_on_add(configuration) # If all is well, build and store an Updater, and remember network locations. - Logger.info('Adding updater for interposed '+repr(configuration)) + Logger.info('Adding updater for interposed '+ repr(configuration)) # Adding an object of the tuf.interposition.updater.Updater with the given # configuration. self.__updaters[configuration.network_location] = Updater(configuration) @@ -868,10 +867,10 @@ def refresh(self, configuration): # Check if the configuration.network_location is available in the updater or mirror # list. if configuration.network_location not in self.__updaters: - raise tuf.NotFoundError("Updater with "+repr(configuration.network_location)+" not found") + raise tuf.NotFoundError("Updater with "+ repr(configuration.network_location)+" not found") if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): - raise tuf.NotFoundError("Mirror with "+repr(repository_mirror_network_locations)+" not found") + raise tuf.NotFoundError("Mirror with "+ repr(repository_mirror_network_locations)+" not found") # Get the updater and refresh its top-level metadata. In the majority of # integrations, a software updater integrating TUF with interposition will @@ -937,7 +936,7 @@ def get(self, url): updater = self.__updaters.get(network_location) if updater is None: - Logger.warn('No updater for '+repr(hostname)) + Logger.warn('No updater for '+ repr(hostname)) else: From ccaa3ad648402202d8ef40964e7552425d1df2f6 Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Mon, 4 Aug 2014 17:18:39 -0400 Subject: [PATCH 24/52] Add purpose to tuf.interposition.README --- tuf/interposition/README.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index d6375169..8742b46b 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -1,4 +1,37 @@ -## Examples +## Interposition + +Interposition is the high-level integration of TUF. 'updater.py' is used to +perform high-level integration of TUF to the software updater. This means +that all the processes which are taking place in the low-level integration +will be done automatically. This layer of processes will be transparent to +the client. + +Updater.py have two classes named as Updater and UpdaterController. + +tuf.interposition.updater.Updater contains those methods which are to be +performed on each individual updater. For example - refresh(), cleanup(), +download_target(target_filepath), get_target_filepath(source_url), open(url), +retrieve(url), switch_context(), all these for a particular updater. + +tuf.interposition.updater.UpdaterController contains those methods which +are performed on updaters as a group. It basically keeps track of all the +updaters. For example - add(configuration), get(configuration), +refresh(configuration), remove(configuration), all these are performed on the +list of updaters. tuf.interposition.updater.UpdaterController maintains a +map of updaters and a set of its mirrors. The map of updaters contains the +objects of tuf.interposition.updater.Updater for each updater. The set +contains all the mirrors. The addition and removal of these updaters and their +mirrors depends on the methods of tuf.interposition.updater.UpdaterController + +### Integration with interposition example + +To implement interpostion, client only need to have the following- +First, a client module which is modified to include interposition library and +code and second, a JSON configuration file is created, each of which is +explained below - + +1. "interposition.py" is an example client updater module that is integrating + TUF with interposition. ```python import tuf.interposition From 1a5a746df22835f8c2de7b860cae67346713f89d Mon Sep 17 00:00:00 2001 From: Pankhuri Goyal Date: Mon, 4 Aug 2014 17:24:42 -0400 Subject: [PATCH 25/52] Add purpose to tuf.interposition.README --- tuf/interposition/README.md | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index 8742b46b..081e7768 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -1,37 +1,21 @@ ## Interposition -Interposition is the high-level integration of TUF. 'updater.py' is used to -perform high-level integration of TUF to the software updater. This means -that all the processes which are taking place in the low-level integration -will be done automatically. This layer of processes will be transparent to -the client. +Interposition is the high-level integration of TUF. 'updater.py' is used to perform high-level integration of TUF to the software updater. This means that all the processes which are taking place in the low-level integration will be done automatically. This layer of processes will be transparent to the client. -Updater.py have two classes named as Updater and UpdaterController. +Updater.py have two classes named as Updater and UpdaterController. -tuf.interposition.updater.Updater contains those methods which are to be -performed on each individual updater. For example - refresh(), cleanup(), -download_target(target_filepath), get_target_filepath(source_url), open(url), -retrieve(url), switch_context(), all these for a particular updater. +tuf.interposition.updater.Updater contains those methods which are to be performed on each individual updater. For example - refresh(), cleanup(),download_target(target_filepath), get_target_filepath(source_url), open(url), retrieve(url), switch_context(), all these for a particular updater. -tuf.interposition.updater.UpdaterController contains those methods which -are performed on updaters as a group. It basically keeps track of all the -updaters. For example - add(configuration), get(configuration), -refresh(configuration), remove(configuration), all these are performed on the -list of updaters. tuf.interposition.updater.UpdaterController maintains a -map of updaters and a set of its mirrors. The map of updaters contains the -objects of tuf.interposition.updater.Updater for each updater. The set -contains all the mirrors. The addition and removal of these updaters and their -mirrors depends on the methods of tuf.interposition.updater.UpdaterController +tuf.interposition.updater.UpdaterController contains those methods which are performed on updaters as a group. It basically keeps track of all the updaters. For example - add(configuration), get(configuration), refresh(configuration), remove(configuration), all these are performed on the list of updaters. +tuf.interposition.updater.UpdaterController maintains a map of updaters and a set of its mirrors. The map of updaters contains the objects of +tuf.interposition.updater.Updater for each updater. The set contains all the mirrors. The addition and removal of these updaters and their mirrors depends on the methods of tuf.interposition.updater.UpdaterController ### Integration with interposition example To implement interpostion, client only need to have the following- -First, a client module which is modified to include interposition library and -code and second, a JSON configuration file is created, each of which is -explained below - +First, a client module which is modified to include interposition library and code and second, a JSON configuration file is created, each of which is explained below - -1. "interposition.py" is an example client updater module that is integrating - TUF with interposition. +1. "interposition.py" is an example client updater module that is integrating TUF with interposition. ```python import tuf.interposition From 42a89063d360d166c50a881871e69511ae4ed63a Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Wed, 20 Aug 2014 12:24:54 -0400 Subject: [PATCH 26/52] Ignore coverage files stored in 'htmlcov/*'. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8dbe9769..5dbf7e80 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ build/* *.egg-info .coverage .tox/* +tests/htmlcov/* From 419b1ca85e92637c14a4880220598996ca869365 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Wed, 20 Aug 2014 12:27:06 -0400 Subject: [PATCH 27/52] Log debug message for roles that are skipped in util.find_delegated_role(). --- tests/test_util.py | 1 + tuf/util.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index 9c1c20ef..cd3a9b11 100755 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -394,6 +394,7 @@ def test_C2_find_delegated_role(self): self.assertTrue(tuf.formats.ROLELIST_SCHEMA.matches(role_list)) self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/tuf'), 1) self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/warehouse'), 0) + # Test for non-existent role. 'find_delegated_role()' returns 'None' # if the role is not found. self.assertEqual(tuf.util.find_delegated_role(role_list, 'targets/non-existent'), diff --git a/tuf/util.py b/tuf/util.py index ef426d77..ba401431 100755 --- a/tuf/util.py +++ b/tuf/util.py @@ -560,7 +560,7 @@ def find_delegated_role(roles, delegated_role): # This role has a different name. else: - continue + logger.debug('Skipping delegated role: ' + repr(delegated_role)) return role_index From 25440ecd95e87076e49b18553b9e3e87e9258511 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Wed, 20 Aug 2014 13:00:53 -0400 Subject: [PATCH 28/52] Fix white space in 'tuf/client/updater.py'. --- tuf/client/updater.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 407bed86..2f1f8c22 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -324,7 +324,7 @@ def __init__(self, updater_name, repository_mirrors): # Ensure the current path is valid/exists before saving it. if not os.path.exists(current_path): - message = 'Missing '+repr(current_path)+'. This path must exist and, ' \ + message = 'Missing ' + repr(current_path) + '. This path must exist and, ' \ 'at a minimum, contain the root metadata file.' raise tuf.RepositoryError(message) self.metadata_directory['current'] = current_path @@ -334,7 +334,7 @@ def __init__(self, updater_name, repository_mirrors): # Ensure the previous path is valid/exists. if not os.path.exists(previous_path): - message = 'Missing '+repr(previous_path)+'. This path must exist.' + message = 'Missing ' + repr(previous_path) + '. This path must exist.' raise tuf.RepositoryError(message) self.metadata_directory['previous'] = previous_path @@ -402,7 +402,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # Ensure we have a valid metadata set. if metadata_set not in ['current', 'previous']: - raise tuf.Error('Invalid metadata set: '+repr(metadata_set)) + raise tuf.Error('Invalid metadata set: ' + repr(metadata_set)) # Save and construct the full metadata path. metadata_directory = self.metadata_directory[metadata_set] From f649749e10385c1f49b2a089dd7403084f7c87f6 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Tue, 2 Sep 2014 14:36:44 -0400 Subject: [PATCH 29/52] Add whitespace to logger strings in updater.py and keydb.py --- tuf/client/updater.py | 14 +++++++------- tuf/keydb.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 2f1f8c22..dbe8d70c 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -515,7 +515,7 @@ def _import_delegations(self, parent_role): keys_info = current_parent_metadata['delegations'].get('keys', {}) roles_info = current_parent_metadata['delegations'].get('roles', []) - logger.debug('Adding roles delegated from '+repr(parent_role)+'.') + logger.debug('Adding roles delegated from ' + repr(parent_role) + '.') # Iterate through the keys of the delegated roles of 'parent_role' # and load them. @@ -532,12 +532,12 @@ def _import_delegations(self, parent_role): pass except (tuf.FormatError, tuf.Error) as e: - logger.exception('Failed to add keyid: '+repr(keyid)+'.') - logger.error('Aborting role delegation for parent role '+parent_role+'.') + logger.exception('Invalid key for keyid: ' + repr(keyid) + '.') + logger.error('Aborting role delegation for parent role ' + parent_role + '.') raise else: - logger.warning('Invalid key type for '+repr(keyid)+'.') + logger.warning('Invalid key type for ' + repr(keyid) + '.') continue # Add the roles to the role database. @@ -546,14 +546,14 @@ def _import_delegations(self, parent_role): # NOTE: tuf.roledb.add_role will take care of the case where rolename # is None. rolename = roleinfo.get('name') - logger.debug('Adding delegated role: '+str(rolename)+'.') + logger.debug('Adding delegated role: ' + str(rolename) + '.') tuf.roledb.add_role(rolename, roleinfo) except tuf.RoleAlreadyExistsError as e: - logger.warning('Role already exists: '+rolename) + logger.warning('Role already exists: ' + rolename) except: - logger.exception('Failed to add delegated role: '+rolename+'.') + logger.exception('Failed to add delegated role: ' + rolename + '.') raise diff --git a/tuf/keydb.py b/tuf/keydb.py index dbb34670..60ac62fb 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -164,9 +164,9 @@ def add_key(key_dict, keyid=None): # Raise 'tuf.FormatError' if the check fails. tuf.formats.KEYID_SCHEMA.check_match(keyid) - # Check if the keyid found in 'rsakey_dict' matches 'keyid'. + # Check if the keyid found in 'key_dict' matches 'keyid'. if keyid != key_dict['keyid']: - raise tuf.Error('Incorrect keyid '+key_dict['keyid']+' expected '+keyid) + raise tuf.Error('Incorrect keyid ' + key_dict['keyid'] + ' expected ' + keyid) # Check if the keyid belonging to 'rsakey_dict' is not already # available in the key database before returning. From 3c9b1d0068075fdac6ed928f8e5a27ac3253ca5e Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Tue, 2 Sep 2014 14:46:36 -0400 Subject: [PATCH 30/52] Add missed test coverage for updater.py's init(). Add test condition for missing previous directory in init(). --- tests/test_updater.py | 52 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/tests/test_updater.py b/tests/test_updater.py index 5323cdd4..3c75837f 100755 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -243,18 +243,25 @@ def test_1__init__exceptions(self): shutil.move(self.client_metadata_previous, previous_backup) self.assertRaises(tuf.RepositoryError, updater.Updater, 'test_repository', self.repository_mirrors) + # Restore the client's previous directory. The required 'current' directory # is still missing. shutil.move(previous_backup, self.client_metadata_previous) - - # Test: repository with only a '{repository_directory/metadata/previous' + # Test: repository with only a '{repository_directory}/metadata/previous' # directory. self.assertRaises(tuf.RepositoryError, updater.Updater, 'test_repository', self.repository_mirrors) # Restore the client's current directory. shutil.move(current_backup, self.client_metadata_current) + # Test: repository with a '{repository_directory}/metadata/current' + # directory, but the 'previous' directory is missing. + shutil.move(self.client_metadata_previous, previous_backup) + self.assertRaises(tuf.RepositoryError, updater.Updater, 'test_repository', + self.repository_mirrors) + shutil.move(previous_backup, self.client_metadata_previous) + # Test: repository missing the required 'root.json' file. client_root_file = os.path.join(self.client_metadata_current, 'root.json') backup_root_file = client_root_file + '.backup' @@ -290,10 +297,16 @@ def test_1__load_metadata_from_file(self): # (i.e., only the 'root.json' file should have been loaded. self.assertEqual(len(self.repository_updater.metadata['current']), 5) - # Verify that the content of root metadata is valid. + # Verify that the content of root metadata is valid. self.assertEqual(self.repository_updater.metadata['current']['targets/role1'], role1_meta['signed']) + # Test invalid metadata set argument (must be either + # 'current' or 'previous'.) + self.assertRaises(tuf.Error, + self.repository_updater._load_metadata_from_file, + 'bad_metadata_set', 'targets/role1') + @@ -413,7 +426,40 @@ def test_2__import_delegations(self): for keyid in keyids: self.assertTrue(keyid in tuf.keydb._keydb_dict) + # Verify that _import_delegations() ignores invalid keytypes in the 'keys' + # field of parent role's 'delegations' + existing_keyid = keyids[0] + + self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keytype'] = 'bad_keytype' + self.repository_updater._import_delegations('targets') + # Restore the keytype of 'existing_keyid'. + self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keytype'] = 'rsa' + # Verify that _import_delegations() raises an exception if any key in + # 'delegations' is improperly formatted (i.e., bad keyid.) + tuf.keydb.clear_keydb() + self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keyid'] = '123' + + print(repr(self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keyid'])) + self.repository_updater._import_delegations('targets') + #self.assertRaises(tuf.Error, self.repository_updater._import_delegations, + # 'targets') + + # Restore the keyid of 'existing_keyids2'. + self.repository_updater.metadata['current']['targets']\ + ['delegations']['keys'][existing_keyid]['keyid'] = existing_keyid + + # Verify that _import_delegations() raises an exception if it fails to add + # one of the roles loaded from parent role's 'delegations'. + + + + + From a2928d60b2e0edb909c82790dccabf89bd3eeeb0 Mon Sep 17 00:00:00 2001 From: Pankh Date: Mon, 29 Sep 2014 14:21:12 -0400 Subject: [PATCH 31/52] Exceptions updater in tuf.interposition.updater --- tuf/interposition/updater.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 7e928eaf..2e7fdd9c 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -379,6 +379,7 @@ def download_target(self, target_filepath): # Download file into a temporary directory shared over runtime destination_directory = self.tempdir + tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) # A new path is generated by joining the destination directory path that is # our temporary directory path and target file path. # Note: join() discards 'destination_directory' if 'target_filepath' From 7f64aad061de3532f2b705337cd5db9ad5aa4614 Mon Sep 17 00:00:00 2001 From: Pankh Date: Mon, 29 Sep 2014 14:34:11 -0400 Subject: [PATCH 32/52] Test cases for tuf.interposition.updater with 98% coverage --- tests/test_interpose_updater.py | 447 ++++++++++++++++++++++++++++++++ 1 file changed, 447 insertions(+) create mode 100644 tests/test_interpose_updater.py diff --git a/tests/test_interpose_updater.py b/tests/test_interpose_updater.py new file mode 100644 index 00000000..21c4e19f --- /dev/null +++ b/tests/test_interpose_updater.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python + +""" + +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +import os +import sys +import tempfile +import subprocess +import random +import shutil +import logging +import time +import copy +import json + +import tuf +import tuf.util +import tuf.conf +import tuf.log +import tuf.interposition.updater as updater +import tuf.interposition.configuration as configuration +import tuf.unittest_toolbox as unittest_toolbox + +if sys.version_info >= (2, 7): + import unittest + +else: + import unittest2 as unittest + + +logger = logging.getLogger('tuf.test_interpose_updater') + + +class TestUpdaterController(unittest_toolbox.Modified_TestCase): + + @classmethod + def setUpClass(cls): + # This method is called before tests in individual class are executed. + + # Create a temporary directory to store the repository, metadata, and target + # files. 'temporary_directory' must be deleted in TearDownModule() so that + # temporary files are always removed, even when exceptions occur. + cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) + + # Launch a SimpleHTTPServer (serves files in the current directory). + # Test cases will request metadata and target files that have been + # pre-generated in 'tuf/tests/repository_data', which will be served + # by the SimpleHTTPServer launched here. The test cases of 'test_updater.py' + # assume the pre-generated metadata files have a specific structure, such + # as a delegated role 'targets/role1', three target files, five key files, + # etc. + cls.SERVER_PORT = random.randint(30000, 45000) + command = ['python', 'simple_server.py', str(cls.SERVER_PORT)] + cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE) + logger.info('\n\tServer process started.') + logger.info('\tServer process id: '+str(cls.server_process.pid)) + logger.info('\tServing on port: '+str(cls.SERVER_PORT)) + cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep + + time.sleep(1) + + + + @classmethod + def tearDownClass(cls): + # Remove the temporary directory after all the tests are done. + shutil.rmtree(cls.temporary_directory) + + # Kill the SimpleHTTPServer Process + if cls.server_process is None: + logger.info('\tServer process '+str(cls.server_process.pid)+' terminated.') + cls.server_process.kill() + + + + def setUp(self): + # We are inheriting from custom class. + unittest_toolbox.Modified_TestCase.setUp(self) + + # Copy the original repository files provided in the test folder so that + # any modifications made to repository files are restricted to the copies. + # The 'repository_data' directory is expected to exist in 'tuf.tests/'. + original_repository_files = os.path.join(os.getcwd(), 'repository_data') + temporary_repository_root = \ + self.make_temp_directory(directory=self.temporary_directory) + + # The original repository, keystore, and client directories will be copied + # for each test case. + original_repository = os.path.join(original_repository_files, 'repository') + original_keystore = os.path.join(original_repository_files, 'keystore') + original_client = os.path.join(original_repository_files, 'client') + + # Save references to the often-needed client repository directories. + # Test cases need these references to access metadata and target files. + self.repository_directory = \ + os.path.join(temporary_repository_root, 'repository') + self.keystore_directory = \ + os.path.join(temporary_repository_root, 'keystore') + self.client_directory = os.path.join(temporary_repository_root, 'client') + self.client_metadata = os.path.join(self.client_directory, 'metadata') + self.client_metadata_current = os.path.join(self.client_metadata, 'current') + self.client_metadata_previous = \ + os.path.join(self.client_metadata, 'previous') + + # Copy the original 'repository', 'client', and 'keystore' directories + # to the temporary repository the test cases can use. + shutil.copytree(original_repository, self.repository_directory) + shutil.copytree(original_client, self.client_directory) + shutil.copytree(original_keystore, self.keystore_directory) + + # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'. + repository_basepath = self.repository_directory[len(os.getcwd()):] + + # Test Set 1 - + port = self.SERVER_PORT + url_prefix = \ + 'http://localhost:' + str(port) + repository_basepath + + # Setting 'tuf.conf.repository_directory' with the temporary client + # directory copied from the original repository files. + tuf.conf.repository_directory = self.client_directory + + self.repository_mirrors = {'mirror': {'url_prefix': url_prefix, + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']} + } + + self.target_filepath = [{".*/targets":"/file1.txt"}] + + self.good_configuration = configuration.Configuration('localhost', 8001, + self.client_directory, + self.repository_mirrors, + self.target_filepath, None) + + self.test1_configuration = configuration.Configuration('localhost', port, + self.client_directory, + self.repository_mirrors, + 'targets', None) + + self.test2_configuration = configuration.Configuration('localhost', 8002, + self.client_directory, + self.repository_mirrors, + 'targets', None) + + test_server_port=random.randint(30000,45000) + + self.test3_configuration = configuration.Configuration('localhost', test_server_port, + self.client_directory, + self.repository_mirrors, + 'targets', None) + + url_prefix_test = \ + 'http://localhost:' + str(test_server_port) + repository_basepath + + + self.repository_mirrors = {'mirror': {'url_prefix': url_prefix_test, + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']} + } + + self.test4_configuration = configuration.Configuration('localhost', 8004, + self.client_directory, + self.repository_mirrors, + 'targets', None) + + + + def tearDown(self): + # We are inheriting from custom class. + unittest_toolbox.Modified_TestCase.tearDown(self) + + + + # Unit Tests + def test_add(self): + updater_controller = updater.UpdaterController() + + # Given good configuration, the UpdaterController.add() should work. + updater_controller.add(self.good_configuration) + + # Instead of configuration, if some number is given. + self.assertRaises(tuf.InvalidConfigurationError, updater_controller.add, 8) + + # Hostname already exists, should raise exception. + self.assertRaises(tuf.FormatError, updater_controller.add, + self.good_configuration) + + # Hostname already exists as a mirror, should raise an exception. + self.assertRaises(tuf.FormatError, updater_controller.add, + self.test1_configuration) + + # Repository mirror already exists as another mirror. + self.assertRaises(tuf.FormatError, updater_controller.add, + self.test2_configuration) + + # Remove the old updater. + updater_controller.remove(self.good_configuration) + + # Add a new updater for this test. + updater_controller.add(self.test3_configuration) + + # Repository mirror already exists as an updater. + self.assertRaises(tuf.FormatError, updater_controller.add, + self.test4_configuration) + + # Remove the updater once the testing is completed. + updater_controller.remove(self.test3_configuration) + + + def test_refresh(self): + updater_controller = updater.UpdaterController() + + # To check refresh() method, add a configuration for test. + updater_controller.add(self.good_configuration) + + updater_controller.refresh(self.good_configuration) + + # Check for invalid configuration error. + self.assertRaises(tuf.InvalidConfigurationError, updater_controller.refresh, 8) + + # Check if the updater not added in the updater list is refreshed, gives an + # error or not. + self.assertRaises(tuf.NotFoundError, updater_controller.refresh, + self.test1_configuration) + + # Giving the same port number and network location as good_configuration. + self.test4_configuration.port= 8001 + self.test4_configuration.network_location = 'localhost:8001' + + # Check if the mirror not added is refreshed, gives an error or not. + self.assertRaises(tuf.NotFoundError, updater_controller.refresh, + self.test4_configuration) + + # Make an object of tuf.interposition.updater.Updater of good configuration + # for testing. + good_updater = updater.Updater(self.good_configuration) + good_updater.refresh() + + self.good_configuration.repository_mirrors['mirror']['url_prefix'] = 'http://localhost:99999999' + + # To check if a bad url_prefix of a mirror raises an exception or not. + self.assertRaises(tuf.NoWorkingMirrorError, good_updater.refresh) + + + + def test_get(self): + updater_controller = updater.UpdaterController() + + updater_controller.add(self.good_configuration) + + url = 'http://localhost:8001' + updater_controller.get(url) + + wrong_url = 'http://localhost:9999' + updater_controller.get(wrong_url) + + good_updater = updater.Updater(self.good_configuration) + self.assertRaises(tuf.URLMatchesNoPatternError, good_updater.get_target_filepath, url) + + + + def test_remove(self): + updater_controller = updater.UpdaterController() + + # To check remove() method, add a configuration for test. + updater_controller.add(self.good_configuration) + + # Check for invalid configuration error. + self.assertRaises(tuf.InvalidConfigurationError, updater_controller.remove, 8) + + self.assertRaises(tuf.NotFoundError, updater_controller.remove, self.test1_configuration) + + # Giving the same port number and network location as good_configuration. + self.test4_configuration.port= 8001 + self.test4_configuration.network_location = 'localhost:8001' + + self.assertRaises(tuf.NotFoundError, updater_controller.remove, self.test4_configuration) + + +class TestUpdater(unittest_toolbox.Modified_TestCase): + + @classmethod + def setUpClass(cls): + # This method is called before tests in individual class are executed. + + # Create a temporary directory to store the repository, metadata, and target + # files. 'temporary_directory' must be deleted in TearDownModule() so that + # temporary files are always removed, even when exceptions occur. + cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) + + # Launch a SimpleHTTPServer (serves files in the current directory). + # Test cases will request metadata and target files that have been + # pre-generated in 'tuf/tests/repository_data', which will be served + # by the SimpleHTTPServer launched here. The test cases of 'test_updater.py' + # assume the pre-generated metadata files have a specific structure, such + # as a delegated role 'targets/role1', three target files, five key files, + # etc. + cls.SERVER_PORT = random.randint(30000, 45000) + command = ['python', 'simple_server.py', str(cls.SERVER_PORT)] + cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE) + logger.info('\n\tServer process started.') + logger.info('\tServer process id: '+str(cls.server_process.pid)) + logger.info('\tServing on port: '+str(cls.SERVER_PORT)) + cls.url = 'http://localhost:'+str(cls.SERVER_PORT) + os.path.sep + + time.sleep(1) + + @classmethod + def tearDownClass(cls): + # Remove the temporary directory after all the tests are done. + shutil.rmtree(cls.temporary_directory) + + # Kill the SimpleHTTPServer Process + if cls.server_process is None: + logger.info('\tServer process '+str(cls.server_process.pid)+' terminated.') + cls.server_process.kill() + + + def setUp(self): + # We are inheriting from custom class. + unittest_toolbox.Modified_TestCase.setUp(self) + + # Copy the original repository files provided in the test folder so that + # any modifications made to repository files are restricted to the copies. + # The 'repository_data' directory is expected to exist in 'tuf.tests/'. + original_repository_files = os.path.join(os.getcwd(), 'repository_data') + temporary_repository_root = \ + self.make_temp_directory(directory=self.temporary_directory) + + # The original repository, keystore, and client directories will be copied + # for each test case. + original_repository = os.path.join(original_repository_files, 'repository') + original_keystore = os.path.join(original_repository_files, 'keystore') + original_client = os.path.join(original_repository_files, 'client') + + # Save references to the often-needed client repository directories. + # Test cases need these references to access metadata and target files. + self.repository_directory = \ + os.path.join(temporary_repository_root, 'repository') + self.keystore_directory = \ + os.path.join(temporary_repository_root, 'keystore') + self.client_directory = os.path.join(temporary_repository_root, 'client') + self.client_metadata = os.path.join(self.client_directory, 'metadata') + self.client_metadata_current = os.path.join(self.client_metadata, 'current') + self.client_metadata_previous = \ + os.path.join(self.client_metadata, 'previous') + + # Copy the original 'repository', 'client', and 'keystore' directories + # to the temporary repository the test cases can use. + shutil.copytree(original_repository, self.repository_directory) + shutil.copytree(original_client, self.client_directory) + shutil.copytree(original_keystore, self.keystore_directory) + + # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'. + repository_basepath = self.repository_directory[len(os.getcwd()):] + + # Test Set 1 - + port = self.SERVER_PORT + url_prefix = \ + 'http://localhost:' + str(port) + repository_basepath + + # Setting 'tuf.conf.repository_directory' with the temporary client + # directory copied from the original repository files. + tuf.conf.repository_directory = self.client_directory + + self.repository_mirrors = {'mirror': {'url_prefix': url_prefix, + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']} + } + + self.target_paths = [{".*/targets":"/file1.txt"}] + + self.good_configuration = configuration.Configuration('localhost', 8001, + self.client_directory, + self.repository_mirrors, + self.target_paths, None) + + + + def tearDown(self): + # We are inheriting from custom class. + unittest_toolbox.Modified_TestCase.tearDown(self) + + + # Unit Tests + def test_download_target(self): + myUpdater = updater.Updater(self.good_configuration) + + target_filepath = 'file.txt' + self.assertRaises(tuf.UnknownTargetError, myUpdater.download_target, target_filepath) + + self.assertRaises(tuf.FormatError, myUpdater.download_target, 8) + + target_filepath='file1.txt' + myUpdater.download_target(target_filepath) + + + def test_get_target_filepath(self): + myUpdater = updater.Updater(self.good_configuration) + + self.assertRaises(AttributeError, myUpdater.get_target_filepath, 8) + + test_source_url = 'http://localhost:9999' + self.assertRaises(tuf.URLMatchesNoPatternError, myUpdater.get_target_filepath, test_source_url) + + test_source_url = 'http://localhost:8001/targets/file.txt' + myUpdater.get_target_filepath(test_source_url) + + + def test_open(self): + myUpdater = updater.Updater(self.good_configuration) + + self.assertRaises(AttributeError, myUpdater.open, 8) + + url = 'http://localhost:8001/targets/file1.txt' + myUpdater.open(url, 'interposition.json') + + + def test_retrieve(self): + myUpdater = updater.Updater(self.good_configuration) + + self.assertRaises(AttributeError, myUpdater.retrieve, 8) + + test_source_url = 'http://localhost:8001/targets/file1.txt' + myUpdater.retrieve(test_source_url,'interposition.json') + + #self.assertRaises(tuf.NoWorkingMirrorError, myUpdater.retrieve, test_source_url) + + test_source_url = 'http://6767:localhost' + self.assertRaises(tuf.URLMatchesNoPatternError, myUpdater.retrieve, test_source_url) + + test_source_url = 'http://localhost:8001/targets/file1.txt' + myUpdater.retrieve(test_source_url) + + +if __name__ == '__main__': + unittest.main() From f6e0fecf077273d320c0583968f24e508623af3b Mon Sep 17 00:00:00 2001 From: Pankh Date: Wed, 22 Oct 2014 14:44:31 -0400 Subject: [PATCH 33/52] Add header to test_interpose_updater --- tests/test_interpose_updater.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_interpose_updater.py b/tests/test_interpose_updater.py index 21c4e19f..f35fa37f 100644 --- a/tests/test_interpose_updater.py +++ b/tests/test_interpose_updater.py @@ -1,7 +1,20 @@ #!/usr/bin/env python """ + + test_interpose_updater.py + + Pankhuri Goyal + + + August 2014. + + + See LICENSE for licensing information. + + + Unit test for tuf.interposition.updater. """ from __future__ import print_function From 045e7c4bcd80b7d142121fb926c5d0e11103260e Mon Sep 17 00:00:00 2001 From: Pankh Date: Wed, 22 Oct 2014 14:52:22 -0400 Subject: [PATCH 34/52] Remove unnecessary explanation from README.md --- tuf/interposition/README.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index 081e7768..c151eed5 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -2,14 +2,6 @@ Interposition is the high-level integration of TUF. 'updater.py' is used to perform high-level integration of TUF to the software updater. This means that all the processes which are taking place in the low-level integration will be done automatically. This layer of processes will be transparent to the client. -Updater.py have two classes named as Updater and UpdaterController. - -tuf.interposition.updater.Updater contains those methods which are to be performed on each individual updater. For example - refresh(), cleanup(),download_target(target_filepath), get_target_filepath(source_url), open(url), retrieve(url), switch_context(), all these for a particular updater. - -tuf.interposition.updater.UpdaterController contains those methods which are performed on updaters as a group. It basically keeps track of all the updaters. For example - add(configuration), get(configuration), refresh(configuration), remove(configuration), all these are performed on the list of updaters. -tuf.interposition.updater.UpdaterController maintains a map of updaters and a set of its mirrors. The map of updaters contains the objects of -tuf.interposition.updater.Updater for each updater. The set contains all the mirrors. The addition and removal of these updaters and their mirrors depends on the methods of tuf.interposition.updater.UpdaterController - ### Integration with interposition example To implement interpostion, client only need to have the following- From c0f1936faf9c22adc1cebd6300220fdada33b984 Mon Sep 17 00:00:00 2001 From: Pankh Date: Mon, 10 Nov 2014 13:52:36 -0500 Subject: [PATCH 35/52] Add tuf.interposition.refresh in tuf.interposition.README.md --- tuf/interposition/README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index c151eed5..a95ae438 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -5,23 +5,27 @@ Interposition is the high-level integration of TUF. 'updater.py' is used to perf ### Integration with interposition example To implement interpostion, client only need to have the following- -First, a client module which is modified to include interposition library and code and second, a JSON configuration file is created, each of which is explained below - +First, a client module which is modified to include interposition library and code. Second, a JSON configuration file is created, each of which is explained below - 1. "interposition.py" is an example client updater module that is integrating TUF with interposition. ```python import tuf.interposition -# Configurations are simply a JSON object which allows you to answer these questions: -# - Which network locations get intercepted? -# - Given a network location, which TUF mirrors should we forward requests to? -# - Given a network location, which paths should be intercepted? -# - Given a TUF mirror, how do we verify its SSL certificate? -tuf.interposition.configure() + +// configure() is used to tell TUF to start interposing for a url given below in the option one. +configuration = tuf.interposition.configure() + +// It is required to refresh the top-level metadata which is done using the following. +tuf.interposition.refresh(configuration) + +// deconfigure() is used to stop the interposition +tuf.interposition.deconfigure(configuration) ``` ### Option one ```python +// Importing this will interpose all the urllib contained between configure and deconfigure. from tuf.interposition import urllib_tuf as urllib from tuf.interposition import urllib2_tuf as urllib2 From ee1331205ef84c9a6be3119851ef141af388b372 Mon Sep 17 00:00:00 2001 From: Pankh Date: Mon, 10 Nov 2014 13:57:47 -0500 Subject: [PATCH 36/52] Add tuf.interposition.refresh in tuf.interposition.README.md --- tuf/interposition/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index a95ae438..8312b796 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -12,20 +12,20 @@ First, a client module which is modified to include interposition library and co ```python import tuf.interposition -// configure() is used to tell TUF to start interposing for a url given below in the option one. +# configure() is used to tell TUF to start interposing for a url given below in the option one. configuration = tuf.interposition.configure() -// It is required to refresh the top-level metadata which is done using the following. +# It is required to refresh the top-level metadata which is done using the following. tuf.interposition.refresh(configuration) -// deconfigure() is used to stop the interposition +# deconfigure() is used to stop the interposition tuf.interposition.deconfigure(configuration) ``` ### Option one ```python -// Importing this will interpose all the urllib contained between configure and deconfigure. +# Importing this will interpose all the urllib contained between configure and deconfigure. from tuf.interposition import urllib_tuf as urllib from tuf.interposition import urllib2_tuf as urllib2 From 6be5ce8c08d304f15bccda9d3f0a0bf0887722a4 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Wed, 28 Jan 2015 15:28:00 -0500 Subject: [PATCH 37/52] Edit the docstrings of the interposition Exceptions Modified the docstrings of interposition's Exception classes to be less specific. --- tuf/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tuf/__init__.py b/tuf/__init__.py index 46c3fc94..b2810181 100755 --- a/tuf/__init__.py +++ b/tuf/__init__.py @@ -293,7 +293,7 @@ class UnknownTargetError(Error): class InvalidNameError(Error): - """Indicate an error while trying to validate any type of named object""" + """Indicate an error while trying to validate any type of named object.""" pass @@ -348,15 +348,15 @@ def __str__(self): class NotFoundError(Error): - """If any configuration hostname or repository mirror hostname are not found""" + """If a required configuration or resource is not found.""" pass class URLMatchesNoPatternError(Error): - """URL matches no user-specified regular expression pattern""" + """If a URL does not match a user-specified regular expression.""" pass class InvalidConfigurationError(Error): - """If the configuration do not match a particular pattern then InvalidConfiguration Exception is raised""" + """If a configuration object does not match the expected format.""" pass From 2efe923c8d4b41e5022325358c18a88f9d42778c Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Thu, 29 Jan 2015 17:18:58 -0500 Subject: [PATCH 38/52] Edit interposition text --- tuf/interposition/README.md | 68 +++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index 8312b796..29f2f58b 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -1,47 +1,57 @@ ## Interposition -Interposition is the high-level integration of TUF. 'updater.py' is used to perform high-level integration of TUF to the software updater. This means that all the processes which are taking place in the low-level integration will be done automatically. This layer of processes will be transparent to the client. - -### Integration with interposition example +The interposition package (tuf/interposition/) can be used to integrate TUF +into a software updater. It is an integration method that requires the least +amount of effort from developers who are performing the integration. The +integration method used by interposition is considered high-level because the +integrator does not explicitly call TUF methods to refresh metadata and +download target files. For example, performing a low-level integration with +*tuf/client/updater.py* requires the integrator to instantiate an updater +object, call updater.refresh() to refresh TUF metadata, and +updater.download_target() to download target files referenced in TUF metadata. +In contrast, an integrator may utilize interposition to load some configuration +settings to indicate which URLs requested by Python urllib calls should be +interposed by TUF. This means that all the update calls for metadata and +target requests are made transparently by the low level *tuf/client/updater.py* +module. -To implement interpostion, client only need to have the following- -First, a client module which is modified to include interposition library and code. Second, a JSON configuration file is created, each of which is explained below - -1. "interposition.py" is an example client updater module that is integrating TUF with interposition. +### Interposition Example + +To use interposition, integrators must: + +1. Create an interposition configuration file. +2. Import interposition, and load the configuration file with configure(). +3. Perform updater urllib calls that may be interposed. +4. Deconfigure interposition. + ```python -import tuf.interposition - -# configure() is used to tell TUF to start interposing for a url given below in the option one. -configuration = tuf.interposition.configure() - -# It is required to refresh the top-level metadata which is done using the following. -tuf.interposition.refresh(configuration) - -# deconfigure() is used to stop the interposition -tuf.interposition.deconfigure(configuration) -``` - -### Option one - -```python -# Importing this will interpose all the urllib contained between configure and deconfigure. from tuf.interposition import urllib_tuf as urllib from tuf.interposition import urllib2_tuf as urllib2 +# configure() loads the interposition configuration file that indicates which +# URLs should be interposed by TUF. Any urllib calls that occur after +# configure() are subject to interposition. + +configuration = tuf.interposition.configure() + url = 'http://example.com/path/to/document' + urllib.urlopen(url) -urllib.urlretrieve(url) +urllib.urlretrieve(url, 'mytarget') urllib2.urlopen(url) + +# deconfigure() is used to stop interposition. Any urllib calls that occur +# after deconfigure() are not interposed. +tuf.interposition.deconfigure(configuration) + ``` -### Option two +Note: tuf.interposition.refresh(configuration) may be called to force a +refresh of the TUF metadata. Interposition normally performs a refresh of TUF +metadata when configure() is called. -```python -@tuf.interposition.open_url -def instancemethod(self, url, ...): - ... -``` ## Configuration From 705d396dff56d6224e20e37d5fe55e5f6aab0799 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Fri, 30 Jan 2015 11:33:07 -0500 Subject: [PATCH 39/52] Review and make minor edits to __init__.py --- tuf/interposition/__init__.py | 71 ++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index d7844eee..41ac4d97 100644 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -12,7 +12,7 @@ from tuf.interposition.configuration import ConfigurationParser from tuf.interposition.updater import UpdaterController -Logger = logging.getLogger('tuf.interposition.__init__') +logger = logging.getLogger('tuf.interposition.__init__') # Export nothing when: from tuf.interposition import * __all__ = [] @@ -61,15 +61,17 @@ def __monkey_patch(): - """Build and monkey patch public copies of the urllib and urllib2 modules. + """ + Build and monkey patch public copies of the urllib and urllib2 modules. - We prefer simplicity, which leads to easier proof of security, even if it may - come at the cost of not honouring some provisions of the urllib and urllib2 - module contracts unrelated to security. + We prefer simplicity, which leads to easier proof of security, even if it may + come at the cost of not honouring some provisions of the urllib and urllib2 + module contracts unrelated to security. - References: - http://stackoverflow.com/a/11285504 - http://docs.python.org/2/library/imp.html""" + References: + http://stackoverflow.com/a/11285504 + http://docs.python.org/2/library/imp.html + """ global urllib_tuf global urllib2_tuf @@ -80,8 +82,10 @@ def __monkey_patch(): urllib_tuf = \ imp.load_module( "urllib_tuf", module_file, pathname, description) module_file.close() + except: raise + else: urllib_tuf.urlopen = __urllib_urlopen urllib_tuf.urlretrieve = __urllib_urlretrieve @@ -92,8 +96,10 @@ def __monkey_patch(): urllib2_tuf = \ imp.load_module( "urllib2_tuf", module_file, pathname, description) module_file.close() + except: raise + else: urllib2_tuf.urlopen = __urllib2_urlopen @@ -108,6 +114,7 @@ def __urllib_urlopen(url, data=None, proxies=None): if updater is None: return urllib.urlopen(url, data=data, proxies=proxies) + else: return updater.open(url, data=data) @@ -122,6 +129,7 @@ def __urllib_urlretrieve(url, filename=None, reporthook=None, data=None): if updater is None: return urllib.urlretrieve(url, filename=filename, reporthook=reporthook, data=data) + else: return updater.retrieve(url, filename=filename, reporthook=reporthook, data=data) @@ -143,22 +151,27 @@ def __urllib2_urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): if url.get_method() == "GET": # ...then you should check with TUF. updater = __updater_controller.get(url.get_full_url()) + else: # ...otherwise, revert to default behaviour. - Logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url.get_method(), + logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url.get_method(), url=url.get_full_url())) return urllib2.urlopen(url, data=data, timeout=timeout) + else: # ...otherwise, we assume this is a string. updater = __updater_controller.get(url) if updater is None: return urllib2.urlopen(url, data=data, timeout=timeout) + else: response = updater.open(url, data=data) + # See urllib2.AbstractHTTPHandler.do_open # TODO: let Updater handle this response.msg = "" + return response @@ -175,12 +188,13 @@ def __read_configuration(configuration_handler, parent_ssl_certificates_directory=None): """ A generic function to read TUF interposition configurations off a file, and - then handle those configurations with a given function. configuration_handler - must be a function which accepts a tuf.interposition.Configuration - instance. + then handle those configurations with a given function. + configuration_handler must be a function which accepts a + tuf.interposition.Configuration instance. Returns the parsed configurations as a dictionary of configurations indexed - by hostnames.""" + by hostnames. + """ INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" @@ -190,9 +204,6 @@ def __read_configuration(configuration_handler, parsed_configurations = {} try: - # open() is function in class Updater. It opens the file with given url - # as a temporary file in the binary mode and remains transparent to the - # software updater. with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) configurations = tuf_interpositions.get("configurations", {}) @@ -207,20 +218,19 @@ def __read_configuration(configuration_handler, configuration, parent_repository_directory=parent_repository_directory, parent_ssl_certificates_directory=parent_ssl_certificates_directory) - # configuration_parser.parse() returns - # tuf.interposition.Configuration which makes configuration an - # object. The integration of interposition is done on the basis - # of 'configuration' which is an object. + # configuration_parser.parse() returns a + # 'tuf.interposition.Configuration' object, which interposition + # uses to determine which URLs should be interposed. configuration = configuration_parser.parse() configuration_handler(configuration) parsed_configurations[configuration.hostname] = configuration except: - Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) + logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) raise except: - Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) + logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise else: @@ -236,7 +246,8 @@ def configure(filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): - """The optional parent_repository_directory parameter is used to specify the + """ + The optional parent_repository_directory parameter is used to specify the containing parent directory of the "repository_directory" specified in a configuration for *all* network locations, because sometimes the absolute location of the "repository_directory" is only known at runtime. If you @@ -278,7 +289,8 @@ def configure(filename="tuf.interposition.json", optional; it must specify certificates bundled as PEM (RFC 1422). Returns the parsed configurations as a dictionary of configurations indexed - by hostnames.""" + by hostnames. + """ configurations = \ __read_configuration(__updater_controller.add, filename=filename, @@ -320,8 +332,10 @@ def deconfigure(configurations): def open_url(instancemethod): - """Decorate an instance method of the form - instancemethod(self, url, ...) with me in order to pass it to TUF.""" + """ + Decorate an instance method of the form + instancemethod(self, url, ...) with me in order to pass it to TUF. + """ @functools.wraps(instancemethod) def wrapper(self, *args, **kwargs): @@ -336,11 +350,13 @@ def wrapper(self, *args, **kwargs): if url_object.get_method() == "GET": # ...then you should check with TUF. url = url_object.get_full_url() + else: # ...otherwise, revert to default behaviour. - Logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url_object.get_method(), + logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url_object.get_method(), url=url_object.get_full_url())) return instancemethod(self, *args, **kwargs) + # ...otherwise, we assume this is a string. else: url = url_object @@ -351,6 +367,7 @@ def wrapper(self, *args, **kwargs): if updater is None: # ...then revert to default behaviour. return instancemethod(self, *args, **kwargs) + else: # ...otherwise, use TUF to get this document. return updater.open(url, data=data) From 9887bb976d3ddbd364d1ffa4bde63fba8ca34d27 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Fri, 30 Jan 2015 11:37:25 -0500 Subject: [PATCH 40/52] Re-add option 2 example of interposition --- tuf/interposition/README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tuf/interposition/README.md b/tuf/interposition/README.md index 29f2f58b..1892f502 100644 --- a/tuf/interposition/README.md +++ b/tuf/interposition/README.md @@ -16,7 +16,7 @@ target requests are made transparently by the low level *tuf/client/updater.py* module. -### Interposition Example +### Interposition Examples To use interposition, integrators must: @@ -26,6 +26,8 @@ To use interposition, integrators must: 4. Deconfigure interposition. +## Option 1 + ```python from tuf.interposition import urllib_tuf as urllib from tuf.interposition import urllib2_tuf as urllib2 @@ -48,6 +50,15 @@ tuf.interposition.deconfigure(configuration) ``` +## Option 2 + +```python +@tuf.interposition.open_url +def instancemethod(self, url, ...): + ... +``` + + Note: tuf.interposition.refresh(configuration) may be called to force a refresh of the TUF metadata. Interposition normally performs a refresh of TUF metadata when configure() is called. From cb06aee960c0112120ebe6a447f8f8b6db2984b8 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Fri, 30 Jan 2015 14:05:56 -0500 Subject: [PATCH 41/52] Reviewed and edits configuration.py Add missing docstring labels (to be fully populated in a future pull request that includes unit tests and that validates arguments. Add missing header block. Renamed 'Logger' object to 'logger, to be consistent with the rest of the modules. Minor formatting edits. --- tuf/interposition/configuration.py | 197 +++++++++++++++++++++++++---- 1 file changed, 173 insertions(+), 24 deletions(-) diff --git a/tuf/interposition/configuration.py b/tuf/interposition/configuration.py index bb36491e..f453210a 100644 --- a/tuf/interposition/configuration.py +++ b/tuf/interposition/configuration.py @@ -1,6 +1,18 @@ -#!/usr/bin/env python - """ + + configuration.py + + + Trishank Kuppusamy + Pankhuri Goyal + Vladimir Diaz + + + + + See LICENSE for licensing information. + + """ @@ -20,18 +32,42 @@ import tuf.log -Logger = logging.getLogger('tuf.interposition.configuration') +logger = logging.getLogger('tuf.interposition.configuration') class Configuration(object): - """Holds TUF interposition configuration information about a network - location which is important to an updater for that network location.""" - + """ + + Holds TUF interposition configuration information about a network + location which is important to an updater for that network location. + """ def __init__(self, hostname, port, repository_directory, repository_mirrors, target_paths, ssl_certificates): - """Constructor assumes that its parameters are valid.""" + """ + + Constructor assumes that its parameters are valid. + + + hostname: + + port: + + repository_directory: + + repository_mirrors: + + target_paths: + + ssl_certificates: + + + + + + + """ self.hostname = hostname self.port = port @@ -49,8 +85,19 @@ def __repr__(self): def get_repository_mirror_hostnames(self): - """Get a set of hostnames of every repository mirror of this - configuration.""" + """ + + Get a set of hostnames of every repository mirror of this configuration. + + + None. + + + + + + + """ # Parse TUF server repository mirrors. repository_mirrors = self.repository_mirrors @@ -74,14 +121,36 @@ def get_repository_mirror_hostnames(self): class ConfigurationParser(object): - """Parses TUF interposition configuration information about a network - location, stored as a JSON object, and returns it as a Configuration.""" + """ + + Parses TUF interposition configuration information about a network + location, stored as a JSON object, and returns it as a Configuration. + """ def __init__(self, network_location, configuration, parent_repository_directory=None, parent_ssl_certificates_directory=None): + """ + + + network_location: + + configuration: + + parent_repository_directory: + + parent_ssl_certificates_directory: + + + + + + + None. + """ + self.network_location = network_location self.configuration = configuration self.parent_repository_directory = parent_repository_directory @@ -89,7 +158,20 @@ def __init__(self, network_location, configuration, def get_network_location(self): - """Check network location.""" + """ + + Check network location. + + + None. + + + + + + + + """ INVALID_NETWORK_LOCATION = "Invalid network location {network_location}!" @@ -107,7 +189,20 @@ def get_network_location(self): def get_repository_directory(self): - """Locate TUF client metadata repository.""" + """ + + Locate TUF client metadata repository. + + + None. + + + + + + + + """ INVALID_PARENT_REPOSITORY_DIRECTORY = \ "Invalid parent_repository_directory for {network_location}!" @@ -131,7 +226,20 @@ def get_repository_directory(self): def get_ssl_certificates(self): - """Get any PEM certificate bundle.""" + """ + + Get any PEM certificate bundle. + + + None. + + + + + + + + """ INVALID_SSL_CERTIFICATES = \ "Invalid ssl_certificates for {network_location}!" @@ -162,7 +270,24 @@ def get_ssl_certificates(self): def get_repository_mirrors(self, hostname, port, ssl_certificates): - """Parse TUF server repository mirrors.""" + """ + + Parse TUF server repository mirrors. + + + hostname: + + port: + + ssl_certificates: + + + + + + + + """ INVALID_REPOSITORY_MIRROR = "Invalid repository mirror {repository_mirror}!" @@ -202,7 +327,7 @@ def get_repository_mirrors(self, hostname, port, ssl_certificates): except: error_message = \ INVALID_REPOSITORY_MIRROR.format(repository_mirror=repository_mirror) - Logger.exception(error_message) + logger.exception(error_message) raise tuf.InvalidConfigurationError(error_message) return repository_mirrors @@ -210,10 +335,21 @@ def get_repository_mirrors(self, hostname, port, ssl_certificates): def get_target_paths(self): """ - Within a network_location, we match URLs with this list of regular - expressions, which tell us to map from a source URL to a target URL. - If there are multiple regular expressions which match a source URL, - the order of appearance will be used to resolve ambiguity. + + Within a network_location, we match URLs with this list of regular + expressions, which tell us to map from a source URL to a target URL. + If there are multiple regular expressions which match a source URL, + the order of appearance will be used to resolve ambiguity. + + + None. + + + + + + + """ INVALID_TARGET_PATH = "Invalid target path in {network_location}!" @@ -236,7 +372,7 @@ def get_target_paths(self): except: error_message = \ INVALID_TARGET_PATH.format(network_location=self.network_location) - Logger.exception(error_message) + logger.exception(error_message) raise tuf.InvalidConfigurationError(error_message) return target_paths @@ -244,7 +380,20 @@ def get_target_paths(self): # TODO: more input sanity checks? def parse(self): - """Parse, check and get the required configuration parameters.""" + """ + + Parse, check, and get the required configuration parameters. + + + None. + + + + + + + + """ hostname, port = self.get_network_location() ssl_certificates = self.get_ssl_certificates() @@ -255,5 +404,5 @@ def parse(self): self.get_repository_mirrors(hostname, port, ssl_certificates) # If everything passes, we return a Configuration. - return Configuration(hostname, port, repository_directory, repository_mirrors, - target_paths, ssl_certificates) + return Configuration(hostname, port, repository_directory, + repository_mirrors, target_paths, ssl_certificates) From 1d6f997e5573d3b68e0dce1e920e7f99ddb60d43 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Mon, 2 Feb 2015 10:03:02 -0500 Subject: [PATCH 42/52] Do not capitalize the 'logger' variable name --- tuf/interposition/updater.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 2e7fdd9c..e60a3484 100644 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python - """ updater.py + Trishank Kuppusamy Pankhuri Goyal @@ -72,7 +71,7 @@ configurations = tuf.interposition.configure() url = 'http://example.com/path/to/document' - # This is the standard way of opening and retrieving url in python. + # This is the standard way of opening and retrieving url in Python. urllib.urlopen(url) urllib.urlretrieve(url) urllib2.urlopen(url) @@ -152,7 +151,7 @@ from tuf.interposition.configuration import Configuration -Logger = logging.getLogger('tuf.interposition.updater') +logger = logging.getLogger('tuf.interposition.updater') class Updater(object): @@ -253,7 +252,7 @@ def __init__(self, configuration): self.configuration = configuration # A temporary directory used for this updater over runtime. self.tempdir = tempfile.mkdtemp() - Logger.debug('Created temporary directory at '+ repr(self.tempdir)) + logger.debug('Created temporary directory at '+ repr(self.tempdir)) # Switching context before instantiating updater because updater depends # on some module (tuf.conf) variables. @@ -272,7 +271,7 @@ def __init__(self, configuration): # Update the client's top-level metadata. The download_target() method does # not automatically refresh top-level prior to retrieving target files and # their associated Targets metadata, so update the top-level metadata here. - Logger.info('Refreshing top-level metadata for interposed '+ repr(configuration)) + logger.info('Refreshing top-level metadata for interposed '+ repr(configuration)) self.updater.refresh() @@ -331,7 +330,7 @@ def cleanup(self): """ shutil.rmtree(self.tempdir) - Logger.debug('Deleted temporary directory at '+ repr(self.tempdir)) + logger.debug('Deleted temporary directory at '+ repr(self.tempdir)) def download_target(self, target_filepath): @@ -468,7 +467,7 @@ def get_target_filepath(self, source_url): raise tuf.URLMatchesNoPatternError(source_url) except: - Logger.exception('Possibly invalid target_paths for '+ \ + logger.exception('Possibly invalid target_paths for '+ \ repr(self.configuration.network_location)+'! No TUF interposition for '\ + repr(source_url)) raise @@ -572,7 +571,7 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): It returns the filename and the headers of the file just retrieved. """ - Logger.info('Interposing for '+ repr(url)) + logger.info('Interposing for '+ repr(url)) # What is the actual target to download given the URL? Sometimes we would # like to transform the given URL to the intended target; e.g. "/simple/" @@ -781,7 +780,7 @@ def __check_configuration_on_add(self, configuration): except (tuf.FormatError) as e: error_message = \ 'Invalid repository mirror '+ repr(mirror_network_location) - Logger.exception(error_message) + logger.exception(error_message) raise return repository_mirror_network_locations @@ -818,7 +817,7 @@ def add(self, configuration): repository_mirror_network_locations = self.__check_configuration_on_add(configuration) # If all is well, build and store an Updater, and remember network locations. - Logger.info('Adding updater for interposed '+ repr(configuration)) + logger.info('Adding updater for interposed '+ repr(configuration)) # Adding an object of the tuf.interposition.updater.Updater with the given # configuration. self.__updaters[configuration.network_location] = Updater(configuration) @@ -881,7 +880,7 @@ def refresh(self, configuration): # Although interposition was designed to remain transparent, for software # updaters that require an explicit refresh of top-level metadata, this # method is provided. - Logger.info('Refreshing top-level metadata for '+ repr(configuration)) + logger.info('Refreshing top-level metadata for '+ repr(configuration)) # If everything is good then fetch the updater from __updaters with the # given configurations. @@ -937,13 +936,13 @@ def get(self, url): updater = self.__updaters.get(network_location) if updater is None: - Logger.warn('No updater for '+ repr(hostname)) + logger.warn('No updater for '+ repr(hostname)) else: # Ensure that the updater is meant for this (hostname, port). if updater.configuration.network_location in network_locations: - Logger.info('Found updater for interposed network location: '+ \ + logger.info('Found updater for interposed network location: '+ \ repr(network_location)) # Raises an exception in case we do not recognize how to @@ -953,18 +952,18 @@ def get(self, url): else: # Same hostname, but different (not user-specified) port. - Logger.warn('We have an updater for '+ \ + logger.warn('We have an updater for '+ \ repr(updater.configuration.network_location)+ \ 'but not for '+ repr(network_locations)) updater = None except: - Logger.exception('No updater or interposition for '+ repr(url)) + logger.exception('No updater or interposition for '+ repr(url)) updater = None finally: if updater is None: - Logger.warn('No updater or interposition for '+ repr(url)) + logger.warn('No updater or interposition for '+ repr(url)) return updater @@ -1026,4 +1025,4 @@ def remove(self, configuration): self.__repository_mirror_network_locations.difference_update(repository_mirror_network_locations) # Log the message that the given updater is removed. - Logger.info('Updater removed for interposed '+ repr(configuration)) + logger.info('Updater removed for interposed '+ repr(configuration)) From 90a283d5892118aff6c1b4fb79543d0934034518 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Wed, 4 Feb 2015 17:45:24 -0500 Subject: [PATCH 43/52] Add initial Python 2+3 compatibility --- tests/test_interpose_updater.py | 2 +- tuf/interposition/__init__.py | 51 ++++++++++++++++++++---------- tuf/interposition/configuration.py | 12 +++---- tuf/interposition/updater.py | 29 +++++++++-------- 4 files changed, 56 insertions(+), 38 deletions(-) mode change 100644 => 100755 tests/test_interpose_updater.py mode change 100644 => 100755 tuf/interposition/__init__.py mode change 100644 => 100755 tuf/interposition/configuration.py mode change 100644 => 100755 tuf/interposition/updater.py diff --git a/tests/test_interpose_updater.py b/tests/test_interpose_updater.py old mode 100644 new mode 100755 index f35fa37f..540e8a89 --- a/tests/test_interpose_updater.py +++ b/tests/test_interpose_updater.py @@ -14,7 +14,7 @@ See LICENSE for licensing information. - Unit test for tuf.interposition.updater. + Unit test for 'tuf.interposition.updater.py'. """ from __future__ import print_function diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py old mode 100644 new mode 100755 index 41ac4d97..4f900dfc --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -1,14 +1,23 @@ +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + + import functools import imp import json import socket -import urllib -import urllib2 import logging import tuf.log +import tuf._vendor.six as six -# We import them directly into our namespace so that there is no name conflict. +# We import the following directly into our namespace so that there is no name +# conflict. from tuf.interposition.configuration import ConfigurationParser from tuf.interposition.updater import UpdaterController @@ -21,10 +30,12 @@ # TODO: # - Document design decisions. # - Interposition: Honour urllib/urllib2 contract. -# - Review security issues resulting from regular expressions (e.g. complexity attacks). +# - Review security issues resulting from regular expressions +# (e.g. complexity attacks). # - Warn user when TUF is used without any configuration. # - Override other default (e.g. HTTPS) urllib2 handlers. -# - Failsafe: If TUF fails, offer option to unsafely resort back to urllib/urllib2? +# - Failsafe: If TUF fails, offer option to unsafely resort back to +# urllib/urllib2? @@ -77,8 +88,12 @@ def __monkey_patch(): global urllib2_tuf if urllib_tuf is None: + urllib_module_name = 'urllib' + if six.PY3: + urllib_module_name = 'urllib/request' + try: - module_file, pathname, description = imp.find_module("urllib") + module_file, pathname, description = imp.find_module(urllib_module_name) urllib_tuf = \ imp.load_module( "urllib_tuf", module_file, pathname, description) module_file.close() @@ -91,8 +106,12 @@ def __monkey_patch(): urllib_tuf.urlretrieve = __urllib_urlretrieve if urllib2_tuf is None: + urllib2_module_name = 'urllib2' + if six.PY3: + urllib2_module_name = 'urllib/request' + try: - module_file, pathname, description = imp.find_module("urllib2") + module_file, pathname, description = imp.find_module(urllib2_module_name) urllib2_tuf = \ imp.load_module( "urllib2_tuf", module_file, pathname, description) module_file.close() @@ -113,7 +132,7 @@ def __urllib_urlopen(url, data=None, proxies=None): updater = __updater_controller.get(url) if updater is None: - return urllib.urlopen(url, data=data, proxies=proxies) + return six.moves.urllib.request.urlopen(url, data=data, proxies=proxies) else: return updater.open(url, data=data) @@ -128,7 +147,7 @@ def __urllib_urlretrieve(url, filename=None, reporthook=None, data=None): updater = __updater_controller.get(url) if updater is None: - return urllib.urlretrieve(url, filename=filename, reporthook=reporthook, data=data) + return six.moves.urllib.request.urlretrieve(url, filename=filename, reporthook=reporthook, data=data) else: return updater.retrieve(url, filename=filename, reporthook=reporthook, data=data) @@ -146,7 +165,7 @@ def __urllib2_urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): updater = None # If this is a urllib2.Request... - if isinstance(url, urllib2.Request): + if isinstance(url, six.moves.request.Request): # If this is a GET HTTP method... if url.get_method() == "GET": # ...then you should check with TUF. @@ -156,14 +175,14 @@ def __urllib2_urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): # ...otherwise, revert to default behaviour. logger.warn(NON_GET_HTTP_METHOD_MESSAGE.format(method=url.get_method(), url=url.get_full_url())) - return urllib2.urlopen(url, data=data, timeout=timeout) + return six.moves.urllib.request.urlopen(url, data=data, timeout=timeout) else: # ...otherwise, we assume this is a string. updater = __updater_controller.get(url) if updater is None: - return urllib2.urlopen(url, data=data, timeout=timeout) + return six.moves.urllib.request.urlopen(url, data=data, timeout=timeout) else: response = updater.open(url, data=data) @@ -212,7 +231,7 @@ def __read_configuration(configuration_handler, raise tuf.InvalidConfigurationError(NO_CONFIGURATIONS.format(filename=filename)) else: - for network_location, configuration in configurations.iteritems(): + for network_location, configuration in six.iteritems(configurations): try: configuration_parser = ConfigurationParser(network_location, configuration, parent_repository_directory=parent_repository_directory, @@ -314,7 +333,7 @@ def refresh(configurations): # Although interposition was designed to remain transparent, for software # updaters that require an explicit refresh of top-level metadata, this # method is provided. - for configuration in configurations.itervalues(): + for configuration in six.itervalues(configurations): __updater_controller.refresh(configuration) @@ -324,7 +343,7 @@ def refresh(configurations): def deconfigure(configurations): """Remove TUF interposition for previously read configurations.""" - for configuration in configurations.itervalues(): + for configuration in six.itervalues(configuration): __updater_controller.remove(configuration) @@ -345,7 +364,7 @@ def wrapper(self, *args, **kwargs): data = kwargs.get("data") # If this is a urllib2.Request... - if isinstance(url_object, urllib2.Request): + if isinstance(url_object, six.moves.request.Request): # If this is a GET HTTP method... if url_object.get_method() == "GET": # ...then you should check with TUF. diff --git a/tuf/interposition/configuration.py b/tuf/interposition/configuration.py old mode 100644 new mode 100755 index f453210a..a1bc7aba --- a/tuf/interposition/configuration.py +++ b/tuf/interposition/configuration.py @@ -25,12 +25,10 @@ from __future__ import unicode_literals import os.path -import types -import urlparse import logging import tuf.log - +import tuf._vendor.six as six logger = logging.getLogger('tuf.interposition.configuration') @@ -107,7 +105,7 @@ def get_repository_mirror_hostnames(self): mirror_configuration = repository_mirrors[repository_mirror] url_prefix = mirror_configuration["url_prefix"] - parsed_url = urlparse.urlparse(url_prefix) + parsed_url = six.moves.urllib.parse.urlparse(url_prefix) mirror_hostname = parsed_url.hostname mirror_port = parsed_url.port mirror_network_location = \ @@ -299,7 +297,7 @@ def get_repository_mirrors(self, hostname, port, ssl_certificates): try: url_prefix = mirror_configuration["url_prefix"] - parsed_url = urlparse.urlparse(url_prefix) + parsed_url = six.moves.urllib.parse.urlparse(url_prefix) mirror_hostname = parsed_url.hostname mirror_port = parsed_url.port or 80 mirror_scheme = parsed_url.scheme @@ -360,13 +358,13 @@ def get_target_paths(self): target_paths = self.configuration.get("target_paths", [WILD_TARGET_PATH]) # target_paths: [ target_path, ... ] - assert isinstance(target_paths, types.ListType) + assert isinstance(target_paths, list) for target_path in target_paths: try: # target_path: { "regex_with_groups", "target_with_group_captures" } # e.g. { ".*(/some/directory)/$", "{0}/index.html" } - assert isinstance(target_path, types.DictType) + assert isinstance(target_path, dict) assert len(target_path) == 1 except: diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py old mode 100644 new mode 100755 index e60a3484..9725a556 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -13,15 +13,15 @@ See LICENSE for licensing information. - Interposition is the high-level integration of TUF. 'updater.py' is used to + Extend 'updater.py' is used to perform high-level integration of TUF to the software updater. This means that all the processes which are taking place in the low-level integration will be done automatically. This layer of processes will be transparent to the client. - Updater.py have two classes named as Updater and UpdaterController. + This module provides two classes: Updater and UpdaterController. - tuf.interposition.updater.Updater contains those methods which are to be + 'tuf.interposition.updater.Updater contains those methods which are to be performed on each individual updater. For example - refresh(), cleanup(), download_target(target_filepath), get_target_filepath(source_url), open(url), retrieve(url), switch_context(), all these for a particular updater. @@ -139,13 +139,12 @@ import re import shutil import tempfile -import urllib -import urlparse import logging import tuf.client.updater import tuf.conf import tuf.log +import tuf._vendor.six as six # We import them directly into our namespace so that there is no name conflict. from tuf.interposition.configuration import Configuration @@ -202,10 +201,10 @@ class Updater(object): def __init__(self, configuration): """ - Constructor. Instantiating an updater object causes creation of a - temporary directory. This temporary directory is used for the - tuf.interposition.updater.Updater. After that the tuf.client.updater module which - performs the low-level integration is called. + Constructor. Instantiating an updater object causes creation of a + temporary directory. This temporary directory is used for the + tuf.interposition.updater.Updater. After that the tuf.client.updater + module which performs the low-level integration is called. configuration: @@ -438,7 +437,7 @@ def get_target_filepath(self, source_url): It returns target_filepath. This is the target which TUF should download. """ - parsed_source_url = urlparse.urlparse(source_url) + parsed_source_url = six.moves.urllib.parse.urlparse(source_url) target_filepath = None try: @@ -449,7 +448,7 @@ def get_target_filepath(self, source_url): #TODO: What these two lines are doing? # target_path: { "regex_with_groups", "target_with_group_captures" } # e.g. { ".*(/some/directory)/$", "{0}/index.html" } - source_path_pattern, target_path_pattern = target_path.items()[0] + source_path_pattern, target_path_pattern = list(target_path.items())[0] source_path_match = re.match(source_path_pattern, parsed_source_url.path) # TODO: A failure in string formatting is *critical*. @@ -528,7 +527,7 @@ def open(self, url, data=None): # Extend temporary_file with info(), getcode(), geturl() # http://docs.python.org/2/library/urllib.html#urllib.urlopen # addinfourl() works as a context manager. - response = urllib.addinfourl(temporary_file, headers, url, code=200) + response = six.moves.urllib.response.addinfourl(temporary_file, headers, url, code=200) return response @@ -774,6 +773,7 @@ def __check_configuration_on_add(self, configuration): # amongst other things. if mirror_network_location in self.__updaters: raise tuf.FormatError("Mirror with "+ repr(mirror_network_location)+" already exists as an updater") + if mirror_network_location in self.__repository_mirror_network_locations: raise tuf.FormatError("Mirror with "+ repr(mirror_network_location)+" already exists as a mirror") @@ -919,7 +919,7 @@ def get(self, url): try: # Parse the given url to access individual parts of it. - parsed_url = urlparse.urlparse(url) + parsed_url = six.moves.urllib.parse.urlparse(url) hostname = parsed_url.hostname port = parsed_url.port or 80 netloc = parsed_url.netloc @@ -1001,7 +1001,8 @@ def remove(self, configuration): # If the configuration is valid, get the repository mirrors associated with # it. - repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() + repository_mirror_network_locations = \ + configuration.get_repository_mirror_hostnames() # Check if network location of the given configuration exists or not. if configuration.network_location not in self.__updaters: From 948b596fc7c23239104a717acb518d2e885149ea Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Wed, 4 Feb 2015 17:55:30 -0500 Subject: [PATCH 44/52] Do not use logging.warn() - deprecated in Python 3 --- tuf/interposition/__init__.py | 4 ++-- tuf/interposition/updater.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index 4f900dfc..a3f5c02e 100755 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -95,7 +95,7 @@ def __monkey_patch(): try: module_file, pathname, description = imp.find_module(urllib_module_name) urllib_tuf = \ - imp.load_module( "urllib_tuf", module_file, pathname, description) + imp.load_module('urllib_tuf', module_file, pathname, description) module_file.close() except: @@ -113,7 +113,7 @@ def __monkey_patch(): try: module_file, pathname, description = imp.find_module(urllib2_module_name) urllib2_tuf = \ - imp.load_module( "urllib2_tuf", module_file, pathname, description) + imp.load_module('urllib2_tuf', module_file, pathname, description) module_file.close() except: diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 9725a556..50e2394d 100755 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -817,7 +817,7 @@ def add(self, configuration): repository_mirror_network_locations = self.__check_configuration_on_add(configuration) # If all is well, build and store an Updater, and remember network locations. - logger.info('Adding updater for interposed '+ repr(configuration)) + logger.info('Adding updater for interposed ' + repr(configuration)) # Adding an object of the tuf.interposition.updater.Updater with the given # configuration. self.__updaters[configuration.network_location] = Updater(configuration) @@ -880,7 +880,7 @@ def refresh(self, configuration): # Although interposition was designed to remain transparent, for software # updaters that require an explicit refresh of top-level metadata, this # method is provided. - logger.info('Refreshing top-level metadata for '+ repr(configuration)) + logger.info('Refreshing top-level metadata for ' + repr(configuration)) # If everything is good then fetch the updater from __updaters with the # given configurations. @@ -936,7 +936,7 @@ def get(self, url): updater = self.__updaters.get(network_location) if updater is None: - logger.warn('No updater for '+ repr(hostname)) + logger.warning('No updater for ' + repr(hostname)) else: @@ -952,18 +952,18 @@ def get(self, url): else: # Same hostname, but different (not user-specified) port. - logger.warn('We have an updater for '+ \ - repr(updater.configuration.network_location)+ \ - 'but not for '+ repr(network_locations)) + logger.warning('We have an updater for ' + \ + repr(updater.configuration.network_location) + \ + 'but not for ' + repr(network_locations)) updater = None except: - logger.exception('No updater or interposition for '+ repr(url)) + logger.exception('No updater or interposition for ' + repr(url)) updater = None finally: if updater is None: - logger.warn('No updater or interposition for '+ repr(url)) + logger.warning('No updater or interposition for ' + repr(url)) return updater @@ -1026,4 +1026,4 @@ def remove(self, configuration): self.__repository_mirror_network_locations.difference_update(repository_mirror_network_locations) # Log the message that the given updater is removed. - logger.info('Updater removed for interposed '+ repr(configuration)) + logger.info('Updater removed for interposed ' + repr(configuration)) From e2a4d9fe2c8f11ee3e036a7b0b901cc6ecf4a16d Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Thu, 5 Feb 2015 14:22:01 -0500 Subject: [PATCH 45/52] Test interposition with real example and fix issues encountered Fix issues with 'configuration' variable name and incorrect reference to 'six.moves.urllib.request.Request'. --- tuf/interposition/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index a3f5c02e..e6f61bc0 100755 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -165,7 +165,7 @@ def __urllib2_urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): updater = None # If this is a urllib2.Request... - if isinstance(url, six.moves.request.Request): + if isinstance(url, six.moves.urllib.request.Request): # If this is a GET HTTP method... if url.get_method() == "GET": # ...then you should check with TUF. @@ -343,7 +343,7 @@ def refresh(configurations): def deconfigure(configurations): """Remove TUF interposition for previously read configurations.""" - for configuration in six.itervalues(configuration): + for configuration in six.itervalues(configurations): __updater_controller.remove(configuration) From fdbc4e98a8270b0eebdab5ca6f427655442d70ba Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Thu, 5 Feb 2015 14:22:25 -0500 Subject: [PATCH 46/52] Cosmetic edits to updater.py --- tuf/interposition/updater.py | 75 ++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 50e2394d..412bce87 100755 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -44,6 +44,7 @@ First, a client module which is modified to include interposition library and code and second, a JSON configuration file is created, each of which is explained below - + 1. "interposition.py" is an example client updater module that is integrating TUF with interposition. @@ -178,8 +179,8 @@ class Updater(object): metadata of the updated targets. get_target_filepath(source_url): - source_url is the url of the file to be updated. This method will find the - updated target for this file. + source_url is the url of the file to be updated. This method will find + the updated target for this file. open(url, data): Open the URL url which can either be a string or a request object. @@ -382,7 +383,8 @@ def download_target(self, target_filepath): # our temporary directory path and target file path. # Note: join() discards 'destination_directory' if 'target_filepath' # contains a leading path separator (i.e., is treated as an absolute path). - filename = os.path.join(destination_directory, target_filepath.lstrip(os.sep)) + filename = os.path.join(destination_directory, + target_filepath.lstrip(os.sep)) # Switch TUF context. Switching context before instantiating updater # because updater depends on some module (tuf.conf) variables. @@ -449,11 +451,13 @@ def get_target_filepath(self, source_url): # target_path: { "regex_with_groups", "target_with_group_captures" } # e.g. { ".*(/some/directory)/$", "{0}/index.html" } source_path_pattern, target_path_pattern = list(target_path.items())[0] - source_path_match = re.match(source_path_pattern, parsed_source_url.path) + source_path_match = \ + re.match(source_path_pattern, parsed_source_url.path) # TODO: A failure in string formatting is *critical*. if source_path_match is not None: - target_filepath = target_path_pattern.format(*source_path_match.groups()) + target_filepath = \ + target_path_pattern.format(*source_path_match.groups()) # If there is more than one regular expression which # matches source_url, we resolve ambiguity by order of @@ -466,9 +470,9 @@ def get_target_filepath(self, source_url): raise tuf.URLMatchesNoPatternError(source_url) except: - logger.exception('Possibly invalid target_paths for '+ \ - repr(self.configuration.network_location)+'! No TUF interposition for '\ - + repr(source_url)) + logger.exception('Possibly invalid target_paths for ' + \ + repr(self.configuration.network_location) + \ + '! No TUF interposition for ' + repr(source_url)) raise else: @@ -518,8 +522,8 @@ def open(self, url, data=None): # end-of-line characters and prevents binary files from properly loading on # Windows. # http://docs.python.org/2/tutorial/inputoutput.html#reading-and-writing-files - # TODO: like tempfile, ensure file is deleted when closed? - # open() in the line below is a predefined function in python. + # TODO: like tempfile, ensure file is deleted when closed? open() in the + # line below is a predefined function in python. temporary_file = open(filename, 'rb') #TODO: addinfourl is not in urllib package anymore. We need to check if @@ -527,7 +531,8 @@ def open(self, url, data=None): # Extend temporary_file with info(), getcode(), geturl() # http://docs.python.org/2/library/urllib.html#urllib.urlopen # addinfourl() works as a context manager. - response = six.moves.urllib.response.addinfourl(temporary_file, headers, url, code=200) + response = six.moves.urllib.response.addinfourl(temporary_file, headers, + url, code=200) return response @@ -589,7 +594,8 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): } # Download the target filepath determined by the original URL. - temporary_directory, temporary_filename = self.download_target(target_filepath) + temporary_directory, temporary_filename = \ + self.download_target(target_filepath) if filename is None: # If no filename is given, use the temporary file. @@ -750,7 +756,7 @@ def __check_configuration_on_add(self, configuration): # tuf.interposition.configuration.Configuration as an object which makes # configuration an instance of tuf.interposition.configuration.Configuration if not isinstance(configuration, Configuration): - raise tuf.InvalidConfigurationError("Invalid configuration") + raise tuf.InvalidConfigurationError('Invalid configuration') # Restrict each (incoming, outgoing) network location pair to be unique across # configurations; this prevents interposition cycles, amongst other @@ -758,10 +764,14 @@ def __check_configuration_on_add(self, configuration): # GOOD: A -> { A:X, A:Y, B, ... }, C -> { D }, ... # BAD: A -> { B }, B -> { C }, C -> { A }, ... if configuration.network_location in self.__updaters: - raise tuf.FormatError("Updater with "+ repr(configuration.network_location)+" already exists as an updater") + message = 'Updater with ' + repr(configuration.network_location) + \ + ' already exists as an updater.' + raise tuf.FormatError() if configuration.network_location in self.__repository_mirror_network_locations: - raise tuf.FormatError("Updater with "+ repr(configuration.network_location)+" already exists as a mirror") + message = 'Updater with ' + repr(configuration.network_location) + \ + ' already exists as a mirror.' + raise tuf.FormatError(message) # Check for redundancy in server repository mirrors. repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() @@ -772,14 +782,18 @@ def __check_configuration_on_add(self, configuration): # unique across configurations; this prevents interposition cycles, # amongst other things. if mirror_network_location in self.__updaters: - raise tuf.FormatError("Mirror with "+ repr(mirror_network_location)+" already exists as an updater") + message = 'Mirror with ' + repr(mirror_network_location) + \ + ' already exists as an updater.' + raise tuf.FormatError(message) if mirror_network_location in self.__repository_mirror_network_locations: - raise tuf.FormatError("Mirror with "+ repr(mirror_network_location)+" already exists as a mirror") + message = 'Mirror with ' + repr(mirror_network_location) + \ + ' already exists as a mirror.' + raise tuf.FormatError(message) except (tuf.FormatError) as e: - error_message = \ - 'Invalid repository mirror '+ repr(mirror_network_location) + error_message = 'Invalid repository mirror ' + \ + repr(mirror_network_location) logger.exception(error_message) raise @@ -806,9 +820,9 @@ def add(self, configuration): add is not unique. - The object of tuf.interposition.updater.Updater is added in the list of updaters. - Also, the mirrors of this updater are added into a - repository_mirror_network_locations are added. + The object of 'tuf.interposition.updater.Updater' is added in the list of + updaters. Also, the mirrors of this updater are added to + 'repository_mirror_network_locations'. None @@ -859,18 +873,22 @@ def refresh(self, configuration): # Check if the configuration is valid else raise an exception. if not isinstance(configuration, Configuration): - raise tuf.InvalidConfigurationError("Invalid configuration") + raise tuf.InvalidConfigurationError('Invalid configuration') # Get the repository mirrors of the given configuration. repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() - # Check if the configuration.network_location is available in the updater or mirror - # list. + # Check if the configuration.network_location is available in the updater + # or mirror list. if configuration.network_location not in self.__updaters: - raise tuf.NotFoundError("Updater with "+ repr(configuration.network_location)+" not found") + message = 'Update with ' + repr(configuration.network_location) + \ + ' not found.' + raise tuf.NotFoundError(message) if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): - raise tuf.NotFoundError("Mirror with "+ repr(repository_mirror_network_locations)+" not found") + message = 'Mirror with ' + repr(repository_mirror_network_location) + \ + ' not found.' + raise tuf.NotFoundError(message) # Get the updater and refresh its top-level metadata. In the majority of # integrations, a software updater integrating TUF with interposition will @@ -926,7 +944,8 @@ def get(self, url): # Combine the hostname and port number and assign it to network_location. # The combination of hostname and port is used to identify an updater. - network_location = "{hostname}:{port}".format(hostname=hostname, port=port) + network_location = \ + "{hostname}:{port}".format(hostname=hostname, port=port) # There can be a case when parsed_url.netloc does not have a port (e.g. # 80). To avoid errors because of this case, tuf.interposition again set From 623820c6acd5af3a74cdde44d4b5d66eef98334d Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Thu, 5 Feb 2015 15:24:43 -0500 Subject: [PATCH 47/52] Cosmetic edits to test_interpose_updater.py --- tests/test_interpose_updater.py | 112 ++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 50 deletions(-) diff --git a/tests/test_interpose_updater.py b/tests/test_interpose_updater.py index 540e8a89..ef59dc9b 100755 --- a/tests/test_interpose_updater.py +++ b/tests/test_interpose_updater.py @@ -57,18 +57,18 @@ class TestUpdaterController(unittest_toolbox.Modified_TestCase): def setUpClass(cls): # This method is called before tests in individual class are executed. - # Create a temporary directory to store the repository, metadata, and target - # files. 'temporary_directory' must be deleted in TearDownModule() so that - # temporary files are always removed, even when exceptions occur. + # Create a temporary directory to store the repository, metadata, and + # target files. 'temporary_directory' must be deleted in TearDownModule() + # so that temporary files are always removed, even when exceptions occur. cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) - # Launch a SimpleHTTPServer (serves files in the current directory). - # Test cases will request metadata and target files that have been - # pre-generated in 'tuf/tests/repository_data', which will be served - # by the SimpleHTTPServer launched here. The test cases of 'test_updater.py' - # assume the pre-generated metadata files have a specific structure, such - # as a delegated role 'targets/role1', three target files, five key files, - # etc. + # Launch a SimpleHTTPServer (serves files in the current directory). + # Test cases will request metadata and target files that have been + # pre-generated in 'tuf/tests/repository_data', which will be served + # by the SimpleHTTPServer launched here. The test cases of + # 'test_updater.py' assume the pre-generated metadata files have a specific + # structure, such as a delegated role 'targets/role1', three target + # files, five key files, etc. cls.SERVER_PORT = random.randint(30000, 45000) command = ['python', 'simple_server.py', str(cls.SERVER_PORT)] cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE) @@ -88,7 +88,9 @@ def tearDownClass(cls): # Kill the SimpleHTTPServer Process if cls.server_process is None: - logger.info('\tServer process '+str(cls.server_process.pid)+' terminated.') + message = '\tServer process ' + str(cls.server_process.pid) + \ + ' terminated.' + logger.info(message) cls.server_process.kill() @@ -133,8 +135,7 @@ def setUp(self): # Test Set 1 - port = self.SERVER_PORT - url_prefix = \ - 'http://localhost:' + str(port) + repository_basepath + url_prefix = 'http://localhost:' + str(port) + repository_basepath # Setting 'tuf.conf.repository_directory' with the temporary client # directory copied from the original repository files. @@ -163,21 +164,22 @@ def setUp(self): self.repository_mirrors, 'targets', None) - test_server_port=random.randint(30000,45000) + test_server_port=random.randint(30000, 45000) - self.test3_configuration = configuration.Configuration('localhost', test_server_port, - self.client_directory, - self.repository_mirrors, - 'targets', None) + self.test3_configuration = configuration.Configuration('localhost', + test_server_port, + self.client_directory, + self.repository_mirrors, + 'targets', None) url_prefix_test = \ 'http://localhost:' + str(test_server_port) + repository_basepath self.repository_mirrors = {'mirror': {'url_prefix': url_prefix_test, - 'metadata_path': 'metadata', - 'targets_path': 'targets', - 'confined_target_dirs': ['']} + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']} } self.test4_configuration = configuration.Configuration('localhost', 8004, @@ -238,7 +240,8 @@ def test_refresh(self): updater_controller.refresh(self.good_configuration) # Check for invalid configuration error. - self.assertRaises(tuf.InvalidConfigurationError, updater_controller.refresh, 8) + self.assertRaises(tuf.InvalidConfigurationError, + updater_controller.refresh, 8) # Check if the updater not added in the updater list is refreshed, gives an # error or not. @@ -246,7 +249,7 @@ def test_refresh(self): self.test1_configuration) # Giving the same port number and network location as good_configuration. - self.test4_configuration.port= 8001 + self.test4_configuration.port = 8001 self.test4_configuration.network_location = 'localhost:8001' # Check if the mirror not added is refreshed, gives an error or not. @@ -258,7 +261,8 @@ def test_refresh(self): good_updater = updater.Updater(self.good_configuration) good_updater.refresh() - self.good_configuration.repository_mirrors['mirror']['url_prefix'] = 'http://localhost:99999999' + self.good_configuration.repository_mirrors['mirror']['url_prefix'] = \ + 'http://localhost:99999999' # To check if a bad url_prefix of a mirror raises an exception or not. self.assertRaises(tuf.NoWorkingMirrorError, good_updater.refresh) @@ -277,7 +281,8 @@ def test_get(self): updater_controller.get(wrong_url) good_updater = updater.Updater(self.good_configuration) - self.assertRaises(tuf.URLMatchesNoPatternError, good_updater.get_target_filepath, url) + self.assertRaises(tuf.URLMatchesNoPatternError, + good_updater.get_target_filepath, url) @@ -288,15 +293,18 @@ def test_remove(self): updater_controller.add(self.good_configuration) # Check for invalid configuration error. - self.assertRaises(tuf.InvalidConfigurationError, updater_controller.remove, 8) + self.assertRaises(tuf.InvalidConfigurationError, + updater_controller.remove, 8) - self.assertRaises(tuf.NotFoundError, updater_controller.remove, self.test1_configuration) + self.assertRaises(tuf.NotFoundError, updater_controller.remove, + self.test1_configuration) # Giving the same port number and network location as good_configuration. - self.test4_configuration.port= 8001 + self.test4_configuration.port = 8001 self.test4_configuration.network_location = 'localhost:8001' - self.assertRaises(tuf.NotFoundError, updater_controller.remove, self.test4_configuration) + self.assertRaises(tuf.NotFoundError, updater_controller.remove, + self.test4_configuration) class TestUpdater(unittest_toolbox.Modified_TestCase): @@ -305,18 +313,18 @@ class TestUpdater(unittest_toolbox.Modified_TestCase): def setUpClass(cls): # This method is called before tests in individual class are executed. - # Create a temporary directory to store the repository, metadata, and target - # files. 'temporary_directory' must be deleted in TearDownModule() so that - # temporary files are always removed, even when exceptions occur. + # Create a temporary directory to store the repository, metadata, and + # target files. 'temporary_directory' must be deleted in TearDownModule() + # so that temporary files are always removed, even when exceptions occur. cls.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) - # Launch a SimpleHTTPServer (serves files in the current directory). - # Test cases will request metadata and target files that have been - # pre-generated in 'tuf/tests/repository_data', which will be served - # by the SimpleHTTPServer launched here. The test cases of 'test_updater.py' - # assume the pre-generated metadata files have a specific structure, such - # as a delegated role 'targets/role1', three target files, five key files, - # etc. + # Launch a SimpleHTTPServer (serves files in the current directory). + # Test cases will request metadata and target files that have been + # pre-generated in 'tuf/tests/repository_data', which will be served + # by the SimpleHTTPServer launched here. The test cases of + # 'test_updater.py' assume the pre-generated metadata files have a specific + # structure, such as a delegated role 'targets/role1', three target + # files, five key files, etc. cls.SERVER_PORT = random.randint(30000, 45000) command = ['python', 'simple_server.py', str(cls.SERVER_PORT)] cls.server_process = subprocess.Popen(command, stderr=subprocess.PIPE) @@ -334,7 +342,9 @@ def tearDownClass(cls): # Kill the SimpleHTTPServer Process if cls.server_process is None: - logger.info('\tServer process '+str(cls.server_process.pid)+' terminated.') + message = '\tServer process ' + str(cls.server_process.pid) + \ + ' terminated.') + logger.info(message) cls.server_process.kill() @@ -378,17 +388,16 @@ def setUp(self): # Test Set 1 - port = self.SERVER_PORT - url_prefix = \ - 'http://localhost:' + str(port) + repository_basepath + url_prefix = 'http://localhost:' + str(port) + repository_basepath # Setting 'tuf.conf.repository_directory' with the temporary client # directory copied from the original repository files. tuf.conf.repository_directory = self.client_directory self.repository_mirrors = {'mirror': {'url_prefix': url_prefix, - 'metadata_path': 'metadata', - 'targets_path': 'targets', - 'confined_target_dirs': ['']} + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']} } self.target_paths = [{".*/targets":"/file1.txt"}] @@ -410,11 +419,12 @@ def test_download_target(self): myUpdater = updater.Updater(self.good_configuration) target_filepath = 'file.txt' - self.assertRaises(tuf.UnknownTargetError, myUpdater.download_target, target_filepath) + self.assertRaises(tuf.UnknownTargetError, myUpdater.download_target, + target_filepath) self.assertRaises(tuf.FormatError, myUpdater.download_target, 8) - target_filepath='file1.txt' + target_filepath = 'file1.txt' myUpdater.download_target(target_filepath) @@ -424,7 +434,8 @@ def test_get_target_filepath(self): self.assertRaises(AttributeError, myUpdater.get_target_filepath, 8) test_source_url = 'http://localhost:9999' - self.assertRaises(tuf.URLMatchesNoPatternError, myUpdater.get_target_filepath, test_source_url) + self.assertRaises(tuf.URLMatchesNoPatternError, + myUpdater.get_target_filepath, test_source_url) test_source_url = 'http://localhost:8001/targets/file.txt' myUpdater.get_target_filepath(test_source_url) @@ -445,12 +456,13 @@ def test_retrieve(self): self.assertRaises(AttributeError, myUpdater.retrieve, 8) test_source_url = 'http://localhost:8001/targets/file1.txt' - myUpdater.retrieve(test_source_url,'interposition.json') + myUpdater.retrieve(test_source_url, 'interposition.json') #self.assertRaises(tuf.NoWorkingMirrorError, myUpdater.retrieve, test_source_url) test_source_url = 'http://6767:localhost' - self.assertRaises(tuf.URLMatchesNoPatternError, myUpdater.retrieve, test_source_url) + self.assertRaises(tuf.URLMatchesNoPatternError, myUpdater.retrieve, + test_source_url) test_source_url = 'http://localhost:8001/targets/file1.txt' myUpdater.retrieve(test_source_url) From 7d91e6e2251cee5365acad0d920bd5689ecfd32a Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Fri, 6 Feb 2015 14:49:27 -0500 Subject: [PATCH 48/52] Add docstring header to __init__.py and TODOs --- tuf/interposition/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tuf/interposition/__init__.py b/tuf/interposition/__init__.py index e6f61bc0..eeac65ee 100755 --- a/tuf/interposition/__init__.py +++ b/tuf/interposition/__init__.py @@ -1,3 +1,21 @@ +""" + + __init__.py + + + Trishank Kuppusamy. + Pankhuri Goyal + + + + + See LICENSE for licensing information. + + + TODO: Add pros / cons of using interposition. Also, should we move code + here to its own module (instead of __init__.py)? +""" + # Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. @@ -406,4 +424,3 @@ def wrapper(self, *args, **kwargs): # Build and monkey patch public copies of the urllib and urllib2 modules. __monkey_patch() - From fc80976fc44393fe59a8a044ed983e668c34c688 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Fri, 6 Feb 2015 14:51:57 -0500 Subject: [PATCH 49/52] Remove extraneous closing parenthesis --- tests/test_interpose_updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_interpose_updater.py b/tests/test_interpose_updater.py index ef59dc9b..89beb037 100755 --- a/tests/test_interpose_updater.py +++ b/tests/test_interpose_updater.py @@ -343,7 +343,7 @@ def tearDownClass(cls): # Kill the SimpleHTTPServer Process if cls.server_process is None: message = '\tServer process ' + str(cls.server_process.pid) + \ - ' terminated.') + ' terminated.' logger.info(message) cls.server_process.kill() From daf0d561b21143a8cec3d7b3c9a836bd4bb42671 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Fri, 6 Feb 2015 14:55:01 -0500 Subject: [PATCH 50/52] Review & edit updater.py up to __init__() --- tuf/interposition/updater.py | 200 ++++++++++++++++++----------------- 1 file changed, 103 insertions(+), 97 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 412bce87..5aa9e2a1 100755 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -8,123 +8,128 @@ June 2014. + Refactored and unit tested by Pankhuri. See LICENSE for licensing information. - Extend 'updater.py' is used to - perform high-level integration of TUF to the software updater. This means - that all the processes which are taking place in the low-level integration - will be done automatically. This layer of processes will be transparent to - the client. - - This module provides two classes: Updater and UpdaterController. + Assist with high-level integrations, which means that all the processes that + are taking place in the low-level 'tuf.client.updater.py' will be automated + by this module. This layer of automation will be transparent to the software + updater; urllib-type calls will be intercepted and TUF metadata automatically + fetched along with the packages requested by the software updater. - 'tuf.interposition.updater.Updater contains those methods which are to be - performed on each individual updater. For example - refresh(), cleanup(), + This module provides two classes: Updater and UpdaterController: + + 'tuf.interposition.updater.Updater' contains those methods which are to be + performed on each individual updater. For example: refresh(), cleanup(), download_target(target_filepath), get_target_filepath(source_url), open(url), - retrieve(url), switch_context(), all these for a particular updater. + retrieve(url), switch_context(); all these methods act on particular updater. - tuf.interposition.updater.UpdaterController contains those methods which - are performed on updaters as a group. It basically keeps track of all the - updaters. For example - add(configuration), get(configuration), + 'tuf.interposition.updater.UpdaterController' contains those methods which + are performed on updaters as a group. It basically keeps track of all the + updaters. For example: add(configuration), get(configuration), refresh(configuration), remove(configuration), all these are performed on the - list of updaters. tuf.interposition.updater.UpdaterController maintains a + list of updaters. 'tuf.interposition.updater.UpdaterController' maintains a map of updaters and a set of its mirrors. The map of updaters contains the - objects of tuf.interposition.updater.Updater for each updater. The set - contains all the mirrors. The addition and removal of these updaters and their - mirrors depends on the methods of tuf.interposition.updater.UpdaterController. + objects of 'tuf.interposition.updater.Updater' for each updater. The set + contains all the mirrors. The addition and removal of these updaters and + their mirrors depends on the methods of + 'tuf.interposition.updater.UpdaterController'. + + + + To integrate TUF into a software updater with interposition, integrators only + need to complete two main tasks: First, a JSON configuration file for + interposition is created. Second, the software updater is modified to import + the interposition library and configure interposition. - #TODO: Add Pros and Cons of using interposition. + 1. 'interposition.py' (code included below) is a basic example software + updater that is integrating TUF with interposition. - - - To implement interpostion client only need to have- - First, a client module which is modified to include interposition library and - code and second, a JSON configuration file is created, each of which is - explained below - - - 1. "interposition.py" is an example client updater module that is integrating - TUF with interposition. - - # First import the main module called interposition which contains all - # the required directories and classes. + # First import the interposition package, which contains all of the + # required classes and functions to use TUF and interposition. import tuf.interposition - # urllib_tuf and urllib2_tuf are TUF's copy of urllib and urllib2 + # Next, explicitly import the urllib modules that interposition will be + # interposing/overwriting. 'urllib_tuf' and 'urllib2_tuf' are TUF's + # copies of urllib and urllib2 that are modified to perform updates using + # the framework and the TUF metadata. from tuf.interposition import urllib_tuf as urllib from tuf.interposition import urllib2_tuf as urllib2 - # From tuf.interposition, configure() method is called. - # configure() is within __init__.py - # It takes 3 arguments, one of which is filename of a JSON file. + # The configure() method must now be called. It takes 3 optional + # arguments, one of which is the filename of a JSON configuration file. # This JSON file contains a set of configurations. To make this file, - # follow the second point below. - # Ways to call this method are as follows : - # First, configure() - By default, the configuration object is expected - # to be situated in the current working directory in the file with the - # name "tuf.interposition.json". - # Second, configure(filename="/path/to/json") - # Configure() returns a dictionary of configurations - # Internally, configure() calls add(configuration) function which is in - # the tuf.interposition.updater.UpdaterController. + # follow the second point below. Ways to call this method are as follows: + # First, configure() - By default, the configuration object is expected to + # be located in the current working directory in the file with the name + # "tuf.interposition.json". Second, configure(filename="/path/to/json") + # Configure() returns a dictionary of configurations. Internally, + # configure() calls add(configuration) function which is in the + # 'tuf.interposition.updater.UpdaterController' class. configurations = tuf.interposition.configure() - url = 'http://example.com/path/to/document' - # This is the standard way of opening and retrieving url in Python. + url = 'http://example.com/path/to/file' + + # This is the standard way of opening and retrieving URLs in Python. + # All three urllib calls below are intercepted by TUF's interposition. urllib.urlopen(url) urllib.urlretrieve(url) urllib2.urlopen(url) # Remove TUF interposition for previously read configurations. That is # remove the updater object. - # Deconfigure() takes only one argument i.e. configurations. - # It calls remove(configuration) function which is in - # tuf.interposition.updater.UpdaterController. + # Deconfigure() takes only one argument (i.e. configurations). + # It calls the remove(configuration) function which is in + # 'tuf.interposition.updater.UpdaterController'. tuf.interposition.deconfigure(configurations) - 2. The filename passed as a parameter in configure function is a JSON file. - It is called as configurations. It is a JSON object which tells - tuf.interposition which URLs to intercept, how to transform them (if - necessary), and where to forward them (possibly over SSL) for secure - responses via TUF. By default, the name of the file is - tuf.interposition.json which is as follows - + 2. The filename passed as a argument to configure() is a JSON file. + It is loaded as a JSON object, which tells tuf.interposition which URLs to + intercept, how to transform them (if necessary), and where to forward them + (possibly over SSL) for secure responses via TUF. By default, the name of + the file is tuf.interposition.json. An example of a configuration file + follows. - # configurations are simply a JSON object which allows you to answer + # configurations are simply a JSON object that allows you to answer # these questions - - # - Which network location get intercepted? + # - Which network locations get intercepted? # - Given a network location, which TUF mirrors should we forward # requests to? # - Given a network location, which paths should be intercepted? # - Given a TUF mirror, how do we verify its SSL certificate? { - # This is required root object. - "configurations": { + # This is a required root object. + "configurations": { # Which network location should be intercepted? # Network locations may be specified as "hostname" or "hostname:port". - "localhost": { + "localhost": { + # Where do we find the client copy of the TUF server metadata? - "repository_directory": ".", - # Where do we forward the requests to localhost? - "repository_mirrors" : { - "mirror1": { - # In this case, we forward them to http://localhost:8001 - "url_prefix": "http://localhost:8001", - # You do not have to worry about these default parameters. - "metadata_path": "metadata", - "targets_path": "targets", - "confined_target_dirs": [ "" ] - } + "repository_directory": ".", + + # Where do we forward the requests to localhost? + "repository_mirrors" : { + "mirror1": { + # In this case, we forward them to http://localhost:8001 + "url_prefix": "http://localhost:8001", + + # You do not have to worry about these default parameters. + "metadata_path": "metadata", + "targets_path": "targets", + "confined_target_dirs": [""] } } } } - # After making these two files on the client side, run interposition.py. This - # will start the interposition process. It generates a log file named tuf.log - # in the same directory, which can be used for a review. + # After creating 'tuf.configuration.json' and the example updater module, run + # 'interposition.py'. The urllib calls will be intercepted, and information + # about the update process is generated to a log file named 'tuf.log' in the + # same directory, which can be reviewed. """ # Help with Python 3 compatibility where the print statement is a function, an @@ -147,7 +152,6 @@ import tuf.log import tuf._vendor.six as six -# We import them directly into our namespace so that there is no name conflict. from tuf.interposition.configuration import Configuration @@ -158,43 +162,43 @@ class Updater(object): """ Provide a class that can download target files securely. It performs all - those things which client/updator.py performs. But it performs it in the - background, transparent to the client. + the actions of 'tuf/client/updater.py', but adds methods to handle HTTP + requests and multiple updater objects. refresh(): - This method refresh top-level metadata. It calls the refresh() method of - tuf.client.updater. refresh() method of tuf.client.updater downloads, - verifies, and loads metadata for the top-level roles in a specific order + This method refreshes top-level metadata. It calls the refresh() method of + 'tuf.client.updater'. refresh() method of 'tuf.client.updater' downloads, + verifies, and loads metadata of the top-level roles in a specific order (i.e., timestamp -> snapshot -> root -> targets). The expiration time for - downloaded metadata is also verified. + downloaded metadata is also verified. cleanup(): - It will clean up all the temporary directories which were made as a result - of download. It then prints a message of deletion and also mentions the - name of the deleted directory. + It will clean up all the temporary directories that are made following a + download request. It also logs a message when a temporary file/directory + is deleted. download_target(target_filepath): - It downloads the target from the target_filepath. It also downloads the - metadata of the updated targets. + It downloads the 'target_filepath' repository file. It also downloads any + required metadata to securely satisfy the 'target_filepath' request. get_target_filepath(source_url): - source_url is the url of the file to be updated. This method will find + 'source_url' is the URL of the file to be updated. This method will find the updated target for this file. open(url, data): - Open the URL url which can either be a string or a request object. + Open the 'url' URL, which can either be a string or a request object. The file is opened in the binary read mode as a temporary file. retrieve(url, filename, reporthook, data): - retrieve() method first get the target file path by calling - get_target_filepath(url) which in tuf.interposition.updater.Updater and - then calls download_target() method for the above file path. + retrieve() method first gets the target file path by calling + get_target_filepath(url), which is in 'tuf.interposition.updater.Updater' + and then calls download_target() for the above file path. switch_context(): There is an updater object for each network location that is interposed. Context switching is required because there are multiple - tuf.client.updater objects and each one depends on tuf.conf settings + 'tuf.client.updater' objects and each one depends on tuf.conf settings that are shared. """ @@ -202,9 +206,9 @@ class Updater(object): def __init__(self, configuration): """ - Constructor. Instantiating an updater object causes creation of a - temporary directory. This temporary directory is used for the - tuf.interposition.updater.Updater. After that the tuf.client.updater + Constructor. Instantiating an updater object causes the creation of a + temporary directory. This temporary directory is used by + 'tuf.interposition.updater.Updater'. After that the tuf.client.updater module which performs the low-level integration is called. @@ -774,7 +778,8 @@ def __check_configuration_on_add(self, configuration): raise tuf.FormatError(message) # Check for redundancy in server repository mirrors. - repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() + repository_mirror_network_locations = \ + configuration.get_repository_mirror_hostnames() for mirror_network_location in repository_mirror_network_locations: try: @@ -876,7 +881,8 @@ def refresh(self, configuration): raise tuf.InvalidConfigurationError('Invalid configuration') # Get the repository mirrors of the given configuration. - repository_mirror_network_locations = configuration.get_repository_mirror_hostnames() + repository_mirror_network_locations = \ + configuration.get_repository_mirror_hostnames() # Check if the configuration.network_location is available in the updater # or mirror list. @@ -886,7 +892,7 @@ def refresh(self, configuration): raise tuf.NotFoundError(message) if not repository_mirror_network_locations.issubset(self.__repository_mirror_network_locations): - message = 'Mirror with ' + repr(repository_mirror_network_location) + \ + message = 'Mirror with ' + repr(repository_mirror_network_locations) + \ ' not found.' raise tuf.NotFoundError(message) From a0ab601b7ffb963fe2c22c617bb0672773390218 Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Mon, 9 Feb 2015 16:04:40 -0500 Subject: [PATCH 51/52] Review up to download_target() --- tuf/interposition/updater.py | 82 +++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 5aa9e2a1..795e3350 100755 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -206,33 +206,37 @@ class Updater(object): def __init__(self, configuration): """ - Constructor. Instantiating an updater object causes the creation of a - temporary directory. This temporary directory is used by - 'tuf.interposition.updater.Updater'. After that the tuf.client.updater - module which performs the low-level integration is called. + Constructor for an updater object that may be used to satisfy TUF update + requests, and can be used independent of other updater objects. A + temporary directory is created when this updater object is instantiated, + which is needed by 'tuf.interposition.updater.Updater', and the top-level + roles are refreshed. The 'tuf.client.updater' module performs the + low-level calls. configuration: - A dictionary holding information like the following - - - Which network location get intercepted? - - Given a network location, which TUF mirrors should we forward requests - to? - - Given a network location, which paths should be intercepted? - - Given a TUF mirror, how do we verify its SSL certificate? + A dictionary holding information like the following: + + - Which network location get intercepted? + - Given a network location, which TUF mirrors should we forward requests + to? + - Given a network location, which paths should be intercepted? + - Given a TUF mirror, how do we verify its SSL certificate? This dictionary holds repository mirror information, conformant to 'tuf.formats.MIRRORDICT_SCHEMA'. Information such as the directory containing the metadata and target files, the server's URL prefix, and - the target content directories the client should be confined to. + the target directories the client should be confined to. repository_mirrors = {'mirror1': {'url_prefix': 'http://localhost:8001', - 'metadata_path': 'metadata', - 'targets_path': 'targets', - 'confined_target_dirs': ['']}} + 'metadata_path': 'metadata', + 'targets_path': 'targets', + 'confined_target_dirs': ['']}} tuf.FormatError: - If the arguments of tuf.client.updater.Updater are improperly formatted. + If the arguments of 'tuf.client.updater.Updater' are improperly + formatted. tuf.RepositoryError: If there is an error with the updater's repository files, such @@ -254,40 +258,41 @@ def __init__(self, configuration): """ self.configuration = configuration + # A temporary directory used for this updater over runtime. self.tempdir = tempfile.mkdtemp() - logger.debug('Created temporary directory at '+ repr(self.tempdir)) + logger.debug('Created temporary directory at ' + repr(self.tempdir)) # Switching context before instantiating updater because updater depends # on some module (tuf.conf) variables. self.switch_context() - # Instantiating an tuf.client.updater object causes all the configurations - # for the top-level roles to be read from disk, including the key and role - # information for the delegated targets of 'targets'. The actual metadata - # for delegated roles is not loaded in __init__. The metadata for these - # delegated roles, including nested delegated roles, are loaded, updated, - # and saved to the 'self.metadata' store by the target methods, like + # Instantiating a 'tuf.client.updater' object causes all the configurations + # for the top-level roles to be read from disk, including the key and role + # information for the delegated targets of 'targets'. The actual metadata + # for delegated roles is not loaded in __init__. The metadata for these + # delegated roles, including nested delegated roles, are loaded, updated, + # and saved to the 'self.metadata' store by the target methods, like # all_targets() and targets_of_role(). self.updater = tuf.client.updater.Updater(self.configuration.hostname, self.configuration.repository_mirrors) - # Update the client's top-level metadata. The download_target() method does - # not automatically refresh top-level prior to retrieving target files and - # their associated Targets metadata, so update the top-level metadata here. - logger.info('Refreshing top-level metadata for interposed '+ repr(configuration)) + # Update the client's top-level metadata. The download_target() method + # does not automatically refresh top-level prior to retrieving target files + # and their associated Targets metadata, so update the top-level metadata + # here. + logger.info('Refreshing top-level metadata for interposed ' + repr(configuration)) self.updater.refresh() def refresh(self): """ - This method refresh top-level metadata. It calls the refresh() method of - tuf.client.updater. - refresh() method of tuf.client.updater.py downloads, verifies, and loads - metadata for the top-level roles in a specific order (i.e., timestamp -> - snapshot -> root -> targets) - The expiration time for downloaded metadata is also verified. + This method refreshes the top-level metadata. It calls the refresh() + method of 'tuf.client.updater'. refresh() method of + 'tuf.client.updater.py' downloads, verifies, and loads metadata for the + top-level roles in a specific order (i.e., timestamp -> snapshot -> root + -> targets) The expiration time for downloaded metadata is also verified. This refresh() method should be called by the client before any target requests. Therefore to automate the process, it is called here. @@ -300,11 +305,11 @@ def refresh(self): If the metadata for any of the top-level roles cannot be updated. tuf.ExpiredMetadataError: - if any metadata has expired. + If any metadata has expired. Updates the metadata files of the top-level roles with the latest - information + information. None @@ -316,9 +321,8 @@ def refresh(self): def cleanup(self): """ - It will clean up all the temporary directories which were made as a - result of download. It then prints a message of deletion and also - mentions the name of the deleted directory. + Clean up the updater object's temporary directory (and any + sub-directories). None @@ -327,14 +331,14 @@ def cleanup(self): None - Removal of the temporary directory. + Removal of the temporary 'self.tempdir' directory. None """ shutil.rmtree(self.tempdir) - logger.debug('Deleted temporary directory at '+ repr(self.tempdir)) + logger.debug('Deleted temporary directory at ' + repr(self.tempdir)) def download_target(self, target_filepath): From 4bcfead253b70fb0611e8fe520203ae2d277d91a Mon Sep 17 00:00:00 2001 From: Vladimir Diaz Date: Mon, 9 Feb 2015 20:08:38 -0500 Subject: [PATCH 52/52] Review remainder of updater.py --- tuf/interposition/updater.py | 178 ++++++++++++++++++----------------- 1 file changed, 94 insertions(+), 84 deletions(-) diff --git a/tuf/interposition/updater.py b/tuf/interposition/updater.py index 795e3350..c157f865 100755 --- a/tuf/interposition/updater.py +++ b/tuf/interposition/updater.py @@ -321,8 +321,9 @@ def refresh(self): def cleanup(self): """ - Clean up the updater object's temporary directory (and any - sub-directories). + Remove the updater object's temporary directory (and any sub-directories) + created when the updater object is instantiated to store downloaded + targets and metadata. None @@ -344,30 +345,29 @@ def cleanup(self): def download_target(self, target_filepath): """ - It downloads the target files from the path provided named - target_filepath. - Everything here is performed in a temporary directory. - It identifies the target information for target_filepath by calling - target() method of tuf.client.updater. This method also downloads the - metadata of the updated targets. By doing this, the client retrieves the - target information for the targets they want to update. When client - retrieves all the information, the updated_targets() method of - tuf.client.updater is called to determine the list of targets which have - been changed from those saved locally on disk. - tuf.client.upater.download_target() downloads all the targets in the list - in the destination directory which is our temporary directory. + Download the 'target_filepath' target file. Everything here is performed + in a temporary directory. It identifies the target information for + 'target_filepath' by calling the target() method of 'tuf.client.updater'. + This method also downloads the metadata of the updated targets. By doing + this, the client retrieves the target information for the targets they + want to update. When client retrieves all the information, the + updated_targets() method of 'tuf.client.updater' is called to determine + the list of targets which have been changed from those saved locally on + disk. tuf.client.upater.download_target() downloads all the targets in + the list in the destination directory, which is our temporary directory. - This will only store the file at 'destination_directory' if the downloaded - file matches the description of the file in the trusted metadata. + This will only store the file in the temporary directory if the + downloaded file matches the description of the file in the trusted + metadata. - 'target_filepath' is the target's relative path on the remote repository. + target_filepath: + The target's relative path on the remote repository. tuf.FormatError: - If 'target_filepath', 'updated_target' in - tuf.client.updater.download_target and arguments of updated_targets are - improperly formatted. + If 'target_filepath', 'updated_target' in + 'tuf.client.updater.download_target', is improperly formatted. tuf.UnknownTargetError: If 'target_filepath' was not found. @@ -379,20 +379,21 @@ def download_target(self, target_filepath): A target file is saved to the local system. - It returns destination_directory where the target is been stored and - filename of the target file been stored in the directory. + It returns a (destination directory, filename) tuple where the target is + been stored and filename of the target file been stored in the directory. """ + + tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) # Download file into a temporary directory shared over runtime destination_directory = self.tempdir - tuf.formats.RELPATH_SCHEMA.check_match(target_filepath) # A new path is generated by joining the destination directory path that is # our temporary directory path and target file path. # Note: join() discards 'destination_directory' if 'target_filepath' # contains a leading path separator (i.e., is treated as an absolute path). - filename = os.path.join(destination_directory, - target_filepath.lstrip(os.sep)) + filename = \ + os.path.join(destination_directory, target_filepath.lstrip(os.sep)) # Switch TUF context. Switching context before instantiating updater # because updater depends on some module (tuf.conf) variables. @@ -413,7 +414,8 @@ def download_target(self, target_filepath): # determine which targets have changed from those saved locally on disk. # All the targets that have changed are returned in a list. From this list, # a request to download is made by calling 'download_target()'. - updated_targets = self.updater.updated_targets(targets, destination_directory) + updated_targets = \ + self.updater.updated_targets(targets, destination_directory) # The download_target() method in tuf.client.updater performs the actual # download of the specified target. The file is saved to the @@ -424,7 +426,7 @@ def download_target(self, target_filepath): return destination_directory, filename - # TODO: decide prudent course of action in case of failure + # TODO: decide prudent course of action in case of failure. def get_target_filepath(self, source_url): """ @@ -432,8 +434,8 @@ def get_target_filepath(self, source_url): download when a URL is given. - source_url is passed while calling the function. This is the url which - we want to retrieve. For this url, get_target_filepath() method is called. + source_url: + The URL of the target we want to retrieve. tuf.URLMatchesNoPatternError: @@ -444,7 +446,8 @@ def get_target_filepath(self, source_url): None - It returns target_filepath. This is the target which TUF should download. + If the target filepath is matched, return the filepath, otherwise raise + an exception. """ parsed_source_url = six.moves.urllib.parse.urlparse(source_url) @@ -487,7 +490,7 @@ def get_target_filepath(self, source_url): return target_filepath - # TODO: distinguish between urllib and urllib2 contracts + # TODO: distinguish between urllib and urllib2 contracts. def open(self, url, data=None): """ @@ -496,19 +499,16 @@ def open(self, url, data=None): called when TUF wants to open an already existing updater's 'url'. - url, the one which is to be opened. + url: + The one which is to be opened. - data must be a bytes object specifying additional data to be sent to the - server or None, if no such data needed. + data: + Must be a bytes object specifying additional data to be sent to the + server or None, if no such data needed. - tuf.FormatError: - If 'target_filepath', 'updated_target' in - tuf.client.updater.download_target and arguments of updated_targets are - improperly formatted. - - tuf.UnknownTargetError: - If 'target_filepath' was not found. + tuf.FormatError: + TODO: validate arguments. tuf.NoWorkingMirrorError: If a 'target_filepath' could not be downloaded from any of the mirrors. @@ -523,6 +523,9 @@ def open(self, url, data=None): 'response' which is a file object with info() and geturl() methods added. """ + + # TODO: validate arguments. + filename, headers = self.retrieve(url, data=data) # TUF should always open files in binary mode and remain transparent to the @@ -549,15 +552,16 @@ def open(self, url, data=None): def retrieve(self, url, filename=None, reporthook=None, data=None): """ - retrieve() method first get the target file path by calling - get_target_filepath(url) which is in tuf.interposition.updater.Updater - and then calls download_target() method for the above file path. + Get the target file path by calling self.get_target_filepath(url) and + then self.download_target() method for the above file path. - url, which is to be retrieved. + url: + The URL of the target file to retrieve. - filename, if the is given then everywhere the given filename is used. - If the filename is none, then temporary file is used. + filename: + If given, then the given filename is used. If the filename is none, + then temporary file is used. tuf.FormatError: @@ -606,28 +610,29 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): self.download_target(target_filepath) if filename is None: - # If no filename is given, use the temporary file. - filename = temporary_filename + # If no filename is given, use the temporary file. + filename = temporary_filename else: - # Otherwise, copy TUF-downloaded file in its own directory - # to the location user specified. - shutil.copy2(temporary_filename, filename) + # Otherwise, copy TUF-downloaded file in its own directory + # to the location user specified. + shutil.copy2(temporary_filename, filename) return filename, headers - # TODO: thread-safety, perhaps with a context manager + # TODO: thread-safety, perhaps with a context manager. def switch_context(self): """ - There is an updater object for each network location that is interposed. - Context switching is required because there are multiple - tuf.client.updater objects and each one depends on tuf.conf settings + There is an updater object for each network location that is interposed. + Context switching is required because there are multiple + 'tuf.client.updater' objects and each one depends on tuf.conf settings that are shared. - For this, two settings are required - - 1. Setting local repository directory - 2. Setting the local SSL certificate PEM file + For this, two settings are required: + + 1. Setting local repository directory in 'tuf.conf'. + 2. Setting the local SSL certificate PEM file. None @@ -636,8 +641,9 @@ def switch_context(self): None - The given configuration's repository_directory and ssl_certificates are - assigned to tuf.conf.repository_directory and tuf.conf.ssl_certificates. + The given configuration's 'repository_directory' and ssl_certificates + settings are set to 'tuf.conf.repository_directory' and + 'tuf.conf.ssl_certificates', respectively. None @@ -654,9 +660,8 @@ def switch_context(self): class UpdaterController(object): """ - tuf.interposition.updater.UpdaterController is a controller of the Updaters. - Given a configuration, it can build and store an Updater, which can be - used later with the help of get() method. + A controller of Updater() objects. Given a configuration, it can build and + store an Updater, which can be used later with the help of get() method. @@ -670,34 +675,34 @@ class UpdaterController(object): It checks if the given configuration is valid or not. add(configuration): - This method adds the updater by adding an object of - tuf.interposition.updater.Updater in the __updater map and by adding - repository mirror's network location in the empty set initialized when - the object of tuf.interposition.updater.UpdaterController is created. + This method adds the updater by adding an object of + 'tuf.interposition.updater.Updater' in the __updater map and by adding + repository mirror's network location in the empty set initialized when + the object of 'tuf.interposition.updater.UpdaterController' is created. get(url): - This method is to get the updater if it already exists. It takes the url - and parse it. Then it utilizes hostname and port of that url to check if - it already exists or not. If the updater exists, then it calls the - get_target_filepath() method which returns a target file path to be + Get the updater if it already exists. It takes the url and parses it. + Then it utilizes hostname and port of that url to check if it already + exists or not. If the updater exists, then it calls the + get_target_filepath() method which returns a target file path to be downloaded. refresh(configuration): - To refresh the top-level metadata of the given 'configuration'. - It updates the latest copies of the metadata for the top-level roles. + Refreshes the top-level metadata of the given 'configuration'. It + updates the latest copies of the metadata of the top-level roles. remove(configuration): - Remove an Updater matching the given Configuration as well as its + Remove an Updater matching the given 'configuration' as well as its associated mirrors. """ def __init__(self): """ - To initalize a private map of updaters and a private set of repository + Initalize a private map of updaters and a private set of repository mirror network locations (hostname:port) once the object of - tuf.interposition.updater.UpdaterController is created. This empty map - and set is later used to add, get and remove updaters and their mirrors. + 'tuf.interposition.updater.UpdaterController' is created. This empty map + and set is later used to add, get, and remove updaters and their mirrors. None @@ -812,8 +817,8 @@ def __check_configuration_on_add(self, configuration): def add(self, configuration): """ - Add an Updater based on the given Configuration. Tuf keeps the track of - the updaters so that it can be fetched for later use. + Add an Updater based on the given 'configuration'. TUF keeps track of the + updaters so that it can be fetched for later use. 'configuration' is an object and on the basis of this configuration, an @@ -837,13 +842,17 @@ def add(self, configuration): None """ - repository_mirror_network_locations = self.__check_configuration_on_add(configuration) + repository_mirror_network_locations = \ + self.__check_configuration_on_add(configuration) - # If all is well, build and store an Updater, and remember network locations. + # If all is well, build and store an Updater, and remember network + # locations. logger.info('Adding updater for interposed ' + repr(configuration)) + # Adding an object of the tuf.interposition.updater.Updater with the given # configuration. self.__updaters[configuration.network_location] = Updater(configuration) + # Adding the new the repository mirror network locations to the list. self.__repository_mirror_network_locations.update(repository_mirror_network_locations) @@ -928,7 +937,8 @@ def get(self, url): downloaded. - url, for which tuf is trying to get an updater. Assumption that url is a + url: + URL which TUF is trying to get an updater. Assumption that url is a string. @@ -1000,7 +1010,7 @@ def get(self, url): def remove(self, configuration): """ - Remove an Updater matching the given Configuration as well as its + Remove an Updater matching the given 'configuration', as well as its associated mirrors.