Merge pull request #1153 from jku/allow-no-metadata-or-targets-in-client-config

mirrors: Make targets_path and metadata_path optional

tests/test_mirrors.py

@@ -58,21 +58,29 @@ def setUp(self):
                  'confined_target_dirs' : ['targets/release/',
                                             'targets/release/']},
      'mirror3': {'url_prefix' : 'http://mirror3.com',
-                 'metadata_path' : 'metadata',
                  'targets_path' : 'targets',
-                 'confined_target_dirs' : ['targets/release/',
-                                            'targets/release/']}}
+                 'confined_target_dirs' : ['targets/release/v2/']},
+     'mirror4': {'url_prefix' : 'http://mirror4.com',
+                 'metadata_path' : 'metadata',
+                 'confined_target_dirs' : []},
+    }
 
 
 
   def test_get_list_of_mirrors(self):
     # Test: Normal case.
+
+    # 3 matches: Metadata found on 3 mirrors
     mirror_list = mirrors.get_list_of_mirrors('meta', 'release.txt', self.mirrors)
     self.assertEqual(len(mirror_list), 3)
-    for mirror, mirror_info in six.iteritems(self.mirrors):
-      url = mirror_info['url_prefix'] + '/metadata/release.txt'
-      self.assertTrue(url in mirror_list)
+    self.assertTrue(self.mirrors['mirror1']['url_prefix']+'/metadata/release.txt' in \
+                    mirror_list)
+    self.assertTrue(self.mirrors['mirror2']['url_prefix']+'/metadata/release.txt' in \
+                    mirror_list)
+    self.assertTrue(self.mirrors['mirror4']['url_prefix']+'/metadata/release.txt' in \
+                    mirror_list)
 
+    # 1 match: a mirror without target directory confinement
     mirror_list = mirrors.get_list_of_mirrors('target', 'a.txt', self.mirrors)
     self.assertEqual(len(mirror_list), 1)
     self.assertTrue(self.mirrors['mirror1']['url_prefix']+'/targets/a.txt' in \
@@ -83,12 +91,21 @@ def test_get_list_of_mirrors(self):
     self.assertTrue(self.mirrors['mirror1']['url_prefix']+'/targets/a/b' in \
                     mirror_list)
 
+    # No matches
     mirror1 = self.mirrors['mirror1']
     del self.mirrors['mirror1']
     mirror_list = mirrors.get_list_of_mirrors('target', 'a/b', self.mirrors)
     self.assertFalse(mirror_list)
     self.mirrors['mirror1'] = mirror1
 
+    # 2 matches: One with non-confined targets and one with matching confinement
+    mirror_list = mirrors.get_list_of_mirrors('target', 'release/v2/c', self.mirrors)
+    self.assertEqual(len(mirror_list), 2)
+    self.assertTrue(self.mirrors['mirror1']['url_prefix']+'/targets/release/v2/c' in \
+                    mirror_list)
+    self.assertTrue(self.mirrors['mirror3']['url_prefix']+'/targets/release/v2/c' in \
+                    mirror_list)
+
     # Test: Invalid 'file_type'.
     self.assertRaises(securesystemslib.exceptions.Error, mirrors.get_list_of_mirrors,
                       self.random_string(), 'a', self.mirrors)

tuf/formats.py

@@ -403,8 +403,8 @@
 MIRROR_SCHEMA = SCHEMA.Object(
   object_name = 'MIRROR_SCHEMA',
   url_prefix = securesystemslib.formats.URL_SCHEMA,
-  metadata_path = RELPATH_SCHEMA,
-  targets_path = RELPATH_SCHEMA,
+  metadata_path = SCHEMA.Optional(RELPATH_SCHEMA),
+  targets_path = SCHEMA.Optional(RELPATH_SCHEMA),
   confined_target_dirs = RELPATHS_SCHEMA,
   custom = SCHEMA.Optional(SCHEMA.Object()))
 

tuf/mirrors.py

@@ -92,6 +92,7 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict):
   if file_type not in _SUPPORTED_FILE_TYPES:
     raise securesystemslib.exceptions.Error('Invalid file_type argument.'
       '  Supported file types: ' + repr(_SUPPORTED_FILE_TYPES))
+  path_key = 'metadata_path' if file_type == 'meta' else 'targets_path'
 
   # Reference to 'securesystemslib.util.file_in_confined_directories()' (improve
   # readability).  This function checks whether a mirror should serve a file to
@@ -103,18 +104,17 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict):
 
   list_of_mirrors = []
   for junk, mirror_info in six.iteritems(mirrors_dict):
-    if file_type == 'meta':
-      base = os.path.join(mirror_info['url_prefix'], mirror_info['metadata_path'])
+    # Does mirror serve this file type at all?
+    path = mirror_info.get(path_key)
+    if path is None:
+      continue
 
-    # 'file_type' == 'target'.  'file_type' should have been verified to
-    # contain a supported string value above (either 'meta' or 'target').
-    else:
-      targets_path = mirror_info['targets_path']
-      full_filepath = os.path.join(targets_path, file_path)
+    # for targets, ensure directory confinement
+    if path_key == 'targets_path':
+      full_filepath = os.path.join(path, file_path)
       if not in_confined_directory(full_filepath,
           mirror_info['confined_target_dirs']):
         continue
-      base = os.path.join(mirror_info['url_prefix'], mirror_info['targets_path'])
 
     # urllib.quote(string) replaces special characters in string using the %xx
     # escape.  This is done to avoid parsing issues of the URL on the server
@@ -122,8 +122,10 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict):
     # the URL as UTF-8. We need a long-term solution with #61.
     # http://bugs.python.org/issue1712522
     file_path = six.moves.urllib.parse.quote(file_path)
-    url = os.path.join(base, file_path)
+    url = os.path.join(mirror_info['url_prefix'], path, file_path)
 
+    # The above os.path.join() result as well as input file_path may be
+    # invalid on windows (might contain both separator types), see #1077.
     # Make sure the URL doesn't contain backward slashes on Windows.
     list_of_mirrors.append(url.replace('\\', '/'))