From ed45fb24c234c8ef8c4dae3754db8bcca7fe95a5 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Fri, 26 Jun 2020 16:14:42 +0100 Subject: [PATCH] Move find_bin_for_hash() to repository_lib Move repository_tool._find_bin_for_hash() and helper functions it uses to non-protected functions in repository_lib. _find_bin_for_hash() becomes find_bin_for_target_hash() These functions will be useful to adopters using the WIP low-level API for updating metadata files (see #1048) Signed-off-by: Joshua Lock --- tuf/repository_lib.py | 108 +++++++++++++++++++++++++++++++++++++++++ tuf/repository_tool.py | 93 ++--------------------------------- 2 files changed, 113 insertions(+), 88 deletions(-) diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index 23e9361c..e121ecc0 100755 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -980,6 +980,114 @@ def get_metadata_versioninfo(rolename, repository_name): +def create_bin_name(low, high, prefix_len): + """ + + Create a string name of a delegated hash bin, where name will be a range of + zero-padded (up to prefix_len) strings i.e. for low=00, high=07, + prefix_len=3 the returned name would be '000-007'. + + + low: + The low end of the prefix range to be binned + + high: + The high end of the prefix range to be binned + + prefix_len: + The length of the prefix range components + + + A string bin name, with each end of the range zero-padded up to prefix_len + """ + if low == high: + return "{low:0{len}x}".format(low=low, len=prefix_len) + + return "{low:0{len}x}-{high:0{len}x}".format(low=low, high=high, + len=prefix_len) + + + + + +def get_bin_numbers(number_of_bins): + """ + + Given the desired number of bins (number_of_bins) calculate the prefix + length (prefix_length), total number of prefixes (prefix_count) and the + number of prefixes to be stored in each bin (bin_size). + Example: number_of_bins = 32 + prefix_length = 2 + prefix_count = 256 + bin_size = 8 + That is, each of the 32 hashed bins are responsible for 8 hash prefixes, + i.e. 00-07, 08-0f, ..., f8-ff. + + + number_of_bins: + The number of hashed bins in use + + + A tuple of three values: + 1. prefix_length: the length of each prefix + 2. prefix_count: the total number of prefixes in use + 3. bin_size: the number of hash prefixes to be stored in each bin + """ + # Convert 'number_of_bins' to hexadecimal and determine the number of + # hexadecimal digits needed by each hash prefix + prefix_length = len("{:x}".format(number_of_bins - 1)) + # Calculate the total number of hash prefixes (e.g., 000 - FFF total values) + prefix_count = 16 ** prefix_length + # Determine how many prefixes to assign to each bin + bin_size = prefix_count // number_of_bins + + # For simplicity, ensure that 'prefix_count' (16 ^ n) can be evenly + # distributed over 'number_of_bins' (must be 2 ^ n). Each bin will contain + # (prefix_count / number_of_bins) hash prefixes. + if prefix_count % number_of_bins != 0: + # Note: x % y != 0 does not guarantee that y is not a power of 2 for + # arbitrary x and y values. However, due to the relationship between + # number_of_bins and prefix_count, it is true for them. + raise securesystemslib.exceptions.Error('The "number_of_bins" argument' + ' must be a power of 2.') + + return prefix_length, prefix_count, bin_size + + + + + +def find_bin_for_target_hash(target_hash, number_of_bins): + """ + + For a given hashed filename, target_hash, calculate the name of a hashed bin + into which this file would be delegated given number_of_bins bins are in + use. + + + target_hash: + The hash of the target file's path + + number_of_bins: + The number of hashed_bins in use + + + The name of the hashed bin target_hash would be binned into + """ + + prefix_length, _, bin_size = get_bin_numbers(number_of_bins) + + prefix = int(target_hash[:prefix_length], 16) + + low = prefix - (prefix % bin_size) + high = (low + bin_size - 1) + + return create_bin_name(low, high, prefix_length) + + + + + def get_target_hash(target_filepath): """ diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index b0d98527..56f17f48 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -1,3 +1,4 @@ + #!/usr/bin/env python # Copyright 2013 - 2017, New York University and the TUF contributors @@ -2529,7 +2530,7 @@ def delegate_hashed_bins(self, list_of_targets, keys_of_hashed_bins, securesystemslib.formats.ANYKEYLIST_SCHEMA.check_match(keys_of_hashed_bins) tuf.formats.NUMBINS_SCHEMA.check_match(number_of_bins) - prefix_length, prefix_count, bin_size = _get_bin_numbers(number_of_bins) + prefix_length, prefix_count, bin_size = repo_lib.get_bin_numbers(number_of_bins) logger.info('Creating hashed bin delegations.\n' + repr(len(list_of_targets)) + ' total targets.\n' + @@ -2543,7 +2544,7 @@ def delegate_hashed_bins(self, list_of_targets, keys_of_hashed_bins, ordered_roles = [] for idx in range(0, prefix_count, bin_size): high = idx + bin_size - 1 - name = _create_bin_name(idx, high, prefix_length) + name = repo_lib.create_bin_name(idx, high, prefix_length) if bin_size == 1: target_hash_prefixes = [name] else: @@ -2675,7 +2676,7 @@ def add_target_to_bin(self, target_filepath, number_of_bins=DEFAULT_NUM_BINS, # TODO: check target_filepath is sane path_hash = repo_lib.get_target_hash(target_filepath) - bin_name = _find_bin_for_hash(path_hash, number_of_bins) + bin_name = repo_lib.find_bin_for_target_hash(path_hash, number_of_bins) # Ensure the Targets object has delegated to hashed bins if not self._delegated_roles.get(bin_name, None): @@ -2737,7 +2738,7 @@ def remove_target_from_bin(self, target_filepath, # TODO: check target_filepath is sane? path_hash = repo_lib.get_target_hash(target_filepath) - bin_name = _find_bin_for_hash(path_hash, number_of_bins) + bin_name = repo_lib.find_bin_for_target_hash(path_hash, number_of_bins) # Ensure the Targets object has delegated to hashed bins if not self._delegated_roles.get(bin_name, None): @@ -2839,90 +2840,6 @@ def _keys_to_keydict(keys): - -def _create_bin_name(low, high, prefix_len): - """ - - Create a string name of a delegated hash bin, where name will be a range of - zero-padded (up to prefix_len) strings i.e. for low=00, high=07, - prefix_len=3 the returned name would be '000-007'. - """ - if low == high: - return "{low:0{len}x}".format(low=low, len=prefix_len) - - return "{low:0{len}x}-{high:0{len}x}".format(low=low, high=high, - len=prefix_len) - - - - - -def _get_bin_numbers(number_of_bins): - """ - Given the desired number of bins (number_of_bins) calculate the prefix length - (prefix_length), total number of prefixes (prefix_count) and the number of - prefixes to be stored in each bin (bin_size). - Example: number_of_bins = 32 - prefix_length = 2 - prefix_count = 256 - bin_size = 8 - That is, each of the 32 hashed bins are responsible for 8 hash prefixes, i.e. - 00-07, 08-0f, ..., f8-ff. - """ - # Convert 'number_of_bins' to hexadecimal and determine the number of - # hexadecimal digits needed by each hash prefix - prefix_length = len("{:x}".format(number_of_bins - 1)) - # Calculate the total number of hash prefixes (e.g., 000 - FFF total values) - prefix_count = 16 ** prefix_length - # Determine how many prefixes to assign to each bin - bin_size = prefix_count // number_of_bins - - # For simplicity, ensure that 'prefix_count' (16 ^ n) can be evenly - # distributed over 'number_of_bins' (must be 2 ^ n). Each bin will contain - # (prefix_count / number_of_bins) hash prefixes. - if prefix_count % number_of_bins != 0: - # Note: x % y != 0 does not guarantee that y is not a power of 2 for - # arbitrary x and y values. However, due to the relationship between - # number_of_bins and prefix_count, it is true for them. - raise securesystemslib.exceptions.Error('The "number_of_bins" argument' - ' must be a power of 2.') - - return prefix_length, prefix_count, bin_size - - - - -def _find_bin_for_hash(path_hash, number_of_bins): - """ - - For a given hashed filename, path_hash, calculate the name of a hashed bin - into which this file would be delegated given number_of_bins bins are in - use. - - - path_hash: - The hash of the target file's path - - number_of_bins: - The number of hashed_bins in use - - - The name of the hashed bin path_hash would be binned into. - """ - - prefix_length, _, bin_size = _get_bin_numbers(number_of_bins) - - prefix = int(path_hash[:prefix_length], 16) - - low = prefix - (prefix % bin_size) - high = (low + bin_size - 1) - - return _create_bin_name(low, high, prefix_length) - - - - - def create_new_repository(repository_directory, repository_name='default', storage_backend=None): """