Repository: boto/botocore

Issue:
Unwanted reversed() in loader.py?
I _think_ I might have found a bug in loader.py's `_load_data()` method, though I might just be misreading various other docs I've found.

Specifically, `_load_data()` seems to perform an unwanted `reversed()` on the search path:

```
def _load_data(self, data_path):
    # This is the uncached version for use with ``load_service_model``.
    # We go in-order, returning the first matching path we find.
    for possible_path in reversed(self.get_search_paths()):
```

The problem is `get_search_paths()` already returns the correct order, putting the hardcoded `BOTOCORE_ROOT` entry at the end:

```
def get_search_paths(self):
    paths = []
    # <deleted for brevity>        
    # Automatically add ./botocore/data to the end of the
    # data search path.
    paths.append(os.path.join(BOTOCORE_ROOT, 'data'))
    return paths
```

As a result, `BOTOCORE_ROOT` is always the first value for `possible_path`, making it impossible to set `BOTO_DATA_PATH` to use (for example) a customized `_endpoints.json` file (when `loader.load_data('aws/_endpoints')` is called).  (I'm working with someone who is using a proxy to try to play some games with IAM credentials here.)

I think I've confirmed my intuition by logging the state of both `get_search_paths()` and `_load_data()`, and, indeed, removing the `reversed()` call gives me the expected behavior with the `BOTO_DATA_PATH` directory loaded first.



Discussion:
I think I remember why this was added.  It was because previously we'd recursively merge in updates.  So if I had my own custom `ec2.json` somewhere in the data path, my custom file would only have to contain the differences instead of a complete copy.  In which case the loading logic would be to iterate in reverse order and recursively merge in overrides on top.

However, botocore no longer does any of this recursive deep merging, it's a first found wins which I think makes a lot of sense.

Looks like the loaders could could use some updating.  Marking as confirmed/bug.

This is affecting us too. It stops us from supplying a custom endpoints file in a clean way.


Resolving patch:
diff --git a/botocore/loaders.py b/botocore/loaders.py
--- a/botocore/loaders.py
+++ b/botocore/loaders.py
@@ -204,18 +204,17 @@ def load_data(self, data_path):
 
     def _load_data(self, data_path):
         # This is the uncached version for use with ``load_service_model``.
-        # We go in-order, returning the first matching path we find.
-        for possible_path in reversed(self.get_search_paths()):
+        # We go in-order, returning the first matching path we find
+        # based on the search paths.
+        for possible_path in self.get_search_paths():
             full_path = os.path.join(
                 possible_path,
                 data_path + self.extension
             )
 
             try:
-                # Attempt loading it.
                 return self.file_loader.load_file(full_path)
             except IOError:
-                # It wasn't there. Try the next path & hope for the best.
                 continue
 
         # We didn't find anything that matched on any path.
@@ -402,8 +401,9 @@ def determine_latest(self, data_path, api_version=None):
             # We need to look for an API version that either matches or is
             # the first to come before that (and hence, backward-compatible).
             for opt in all_options:
-                if opt == api_version:
-                    # Exact match. We win!
+                # ``opt`` will be something like "2014-01-01.api" so we need
+                # to strip off the ".api" part.
+                if opt.split('.')[0] == api_version:
                     best_match = opt
                     break
                 elif opt < api_version:


Test patch:
diff --git a/tests/unit/data/aws/s3/2006-03-01.api.json b/tests/unit/data/aws/s3/2006-03-01.api.json
new file mode 100644
--- /dev/null
+++ b/tests/unit/data/aws/s3/2006-03-01.api.json
@@ -0,0 +1,3 @@
+{
+    "WAS_OVERRIDEN_VIA_DATA_PATH": true
+}
diff --git a/tests/unit/test_loaders.py b/tests/unit/test_loaders.py
--- a/tests/unit/test_loaders.py
+++ b/tests/unit/test_loaders.py
@@ -1,4 +1,4 @@
-#1 Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the
@@ -18,148 +18,160 @@
 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
-#
-#import os
-#
-#import mock
-#
-#from botocore.exceptions import ApiVersionNotFoundError
-#from botocore.exceptions import DataNotFoundError
-#from botocore.loaders import cachable
-#from botocore.loaders import JSONFileLoader
-#from botocore.loaders import Loader
-#import botocore.session
-#
-#from tests import unittest, BaseEnvVar
-#
-#
-#class JSONFileLoaderTestCase(BaseEnvVar):
-#    def setUp(self):
-#        super(JSONFileLoaderTestCase, self).setUp()
-#        self.data_path = os.path.join(os.path.dirname(__file__), 'data')
-#        self.file_loader = JSONFileLoader()
-#
-#    def test_load_file(self):
-#        data = self.file_loader.load_file(
-#            os.path.join(self.data_path, 'foo.json')
-#        )
-#        self.assertEqual(len(data), 3)
-#        self.assertTrue('test_key_1' in data)
-#
-#
-#class LoaderTestCase(BaseEnvVar):
-#    def setUp(self):
-#        super(LoaderTestCase, self).setUp()
-#        self.data_path = os.path.join(os.path.dirname(__file__), 'data')
-#        self.environ['BOTO_DATA_PATH'] = self.data_path
-#        self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
-#
-#        # Make sure the cache is clear.
-#        self.loader._cache.clear()
-#
-#    def test_data_path_not_required(self):
-#        loader = Loader()
-#        self.assertEqual(loader.data_path, '')
-#        loader.data_path = 'foo:bar'
-#        self.assertEqual(loader.data_path, 'foo:bar')
-#
-#    def test_get_search_paths(self):
-#        paths = self.loader.get_search_paths()
-#        self.assertTrue(self.data_path in paths)
-#
-#    def test_determine_latest_no_version(self):
-#        path = self.loader.determine_latest('someservice')
-#        self.assertEqual(path, os.path.join('someservice', '2013-08-21.api'))
-#
-#    def test_determine_latest_with_version(self):
-#        path = self.loader.determine_latest(
-#            'someservice',
-#            api_version='2012-10-01'
-#        )
-#        self.assertEqual(path, os.path.join('someservice', '2012-10-01.api'))
-#
-#    def test_determine_latest_with_version_the_wrong_way(self):
-#        with self.assertRaises(ApiVersionNotFoundError):
-#            self.loader.determine_latest('someservice/2012-10-01')
-#
-#    def test_determine_latest_with_version_not_found(self):
-#        with self.assertRaises(ApiVersionNotFoundError):
-#            path = self.loader.determine_latest(
-#                'someservice',
-#                api_version='2010-02-02'
-#            )
-#
-#    def test_load_data_plain_file(self):
-#        data = self.loader.load_data('foo')
-#        self.assertEqual(data['test_key_1'], 'test_value_1')
-#
-#    def test_load_data_plain_file_nonexistant(self):
-#        with self.assertRaises(DataNotFoundError):
-#            data = self.loader.load_data('i_totally_dont_exist')
-#
-#    def test_load_service_model_latest_without_version(self):
-#        data = self.loader.load_service_model('someservice')
-#        self.assertEqual(data['api_version'], '2013-08-21')
-#
-#    def test_load_service_model_with_version(self):
-#        data = self.loader.load_service_model(
-#            'someservice',
-#            api_version='2012-10-01'
-#        )
-#        self.assertEqual(data['api_version'], '2012-10-01')
-#
-#    def test_load_service_model_version_not_found(self):
-#        with self.assertRaises(ApiVersionNotFoundError):
-#            data = self.loader.load_service_model(
-#                'someservice',
-#                api_version='2010-02-02'
-#            )
-#
-#    def test_list_available_services(self):
-#        avail = self.loader.list_available_services('')
-#        self.assertEqual(sorted(avail), [
-#            'aws',
-#            'someservice',
-#            'sub',
-#        ])
-#
-#        aws_avail = self.loader.list_available_services('aws')
-#        self.assertTrue(len(aws_avail) > 10)
-#        self.assertTrue('ec2' in aws_avail)
-#
-#    def test_load_data_overridden(self):
-#        self.overrides_path = os.path.join(
-#            os.path.dirname(__file__),
-#            'data_overrides'
-#        )
-#        self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
-#            self.overrides_path,
-#            os.pathsep,
-#            self.data_path
-#        )
-#        loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
-#        # This should load the data the first data it finds.
-#        data = loader.load_service_model(
-#            'someservice',
-#            api_version='2012-10-01'
-#        )
-#        # An overridden key.
-#        self.assertEqual(data['api_version'], '2012-10-01')
-#        # A key unique to the base.
-#        self.assertEqual(data['something-else'], 'another')
-#        # Ensure a key present in other variants is not there.
-#        self.assertTrue('Purpose' not in data)
-#
-#    @mock.patch('os.pathsep', ';')
-#    def test_search_path_on_windows(self):
-#        # On windows, the search path is separated by ';' chars.
-#        self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
-#        # The builtin botocore data path is added as the last element
-#        # so we're only interested in checking the two that we've added.
-#        loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
-#        paths = loader.get_search_paths()[:-1]
-#        self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
-#
-#
-#if __name__ == "__main__":
-#    unittest.main()
+
+import os
+
+import mock
+
+from botocore.exceptions import ApiVersionNotFoundError
+from botocore.exceptions import DataNotFoundError
+from botocore.loaders import cachable
+from botocore.loaders import JSONFileLoader
+from botocore.loaders import Loader
+import botocore.session
+
+from tests import unittest, BaseEnvVar
+
+
+class JSONFileLoaderTestCase(BaseEnvVar):
+    def setUp(self):
+        super(JSONFileLoaderTestCase, self).setUp()
+        self.data_path = os.path.join(os.path.dirname(__file__), 'data')
+        self.file_loader = JSONFileLoader()
+
+    def test_load_file(self):
+        data = self.file_loader.load_file(
+            os.path.join(self.data_path, 'foo.json')
+        )
+        self.assertEqual(len(data), 3)
+        self.assertTrue('test_key_1' in data)
+
+
+class LoaderTestCase(BaseEnvVar):
+    def setUp(self):
+        super(LoaderTestCase, self).setUp()
+        self.data_path = os.path.join(os.path.dirname(__file__), 'data')
+        self.environ['BOTO_DATA_PATH'] = self.data_path
+        self.loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
+
+        # Make sure the cache is clear.
+        self.loader._cache.clear()
+
+    def test_data_path_not_required(self):
+        loader = Loader()
+        self.assertEqual(loader.data_path, '')
+        loader.data_path = 'foo:bar'
+        self.assertEqual(loader.data_path, 'foo:bar')
+
+    def test_get_search_paths(self):
+        paths = self.loader.get_search_paths()
+        self.assertTrue(self.data_path in paths)
+
+    def test_determine_latest_no_version(self):
+        path = self.loader.determine_latest('someservice')
+        self.assertEqual(path, os.path.join('someservice', '2013-08-21.api'))
+
+    def test_determine_latest_with_version(self):
+        path = self.loader.determine_latest(
+            'someservice',
+            api_version='2012-10-01'
+        )
+        self.assertEqual(path, os.path.join('someservice', '2012-10-01.api'))
+
+    def test_determine_latest_with_version_the_wrong_way(self):
+        with self.assertRaises(ApiVersionNotFoundError):
+            self.loader.determine_latest('someservice/2012-10-01')
+
+    def test_determine_latest_with_version_not_found(self):
+        with self.assertRaises(ApiVersionNotFoundError):
+            path = self.loader.determine_latest(
+                'someservice',
+                api_version='2010-02-02'
+            )
+
+    def test_load_data_plain_file(self):
+        data = self.loader.load_data('foo')
+        self.assertEqual(data['test_key_1'], 'test_value_1')
+
+    def test_load_data_plain_file_nonexistant(self):
+        with self.assertRaises(DataNotFoundError):
+            data = self.loader.load_data('i_totally_dont_exist')
+
+    def test_load_service_model_latest_without_version(self):
+        data = self.loader.load_service_model('someservice')
+        self.assertEqual(data['api_version'], '2013-08-21')
+
+    def test_load_service_model_with_version(self):
+        data = self.loader.load_service_model(
+            'someservice',
+            api_version='2012-10-01'
+        )
+        self.assertEqual(data['api_version'], '2012-10-01')
+
+    def test_load_service_model_version_not_found(self):
+        with self.assertRaises(ApiVersionNotFoundError):
+            data = self.loader.load_service_model(
+                'someservice',
+                api_version='2010-02-02'
+            )
+
+    def test_load_service_model_data_path_order(self):
+        # There's an s3/ directory both in our custom BOTO_DATA_PATH
+        # directory as well as in the botocore/data/ directory.
+        # Our path should win since the default built in path is always
+        # last.
+        data = self.loader.load_service_model('aws/s3')
+        self.assertTrue(data.get('WAS_OVERRIDEN_VIA_DATA_PATH'),
+                        "S3 model was loaded from botocore's default "
+                        "data path instead of from the BOTO_DATA_PATH"
+                        " directory.")
+
+    def test_list_available_services(self):
+        avail = self.loader.list_available_services('')
+        self.assertEqual(sorted(avail), [
+            'aws',
+            'aws',
+            'someservice',
+            'sub',
+        ])
+
+        aws_avail = self.loader.list_available_services('aws')
+        self.assertTrue(len(aws_avail) > 10)
+        self.assertTrue('ec2' in aws_avail)
+
+    def test_load_data_overridden(self):
+        self.overrides_path = os.path.join(
+            os.path.dirname(__file__),
+            'data_overrides'
+        )
+        self.environ['BOTO_DATA_PATH'] = "{0}{1}{2}".format(
+            self.overrides_path,
+            os.pathsep,
+            self.data_path
+        )
+        loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
+        # This should load the data the first data it finds.
+        data = loader.load_service_model(
+            'someservice',
+            api_version='2012-10-01'
+        )
+        # An overridden key.
+        self.assertEqual(data['api_version'], '2012-10-01')
+        # A key unique to the base.
+        self.assertEqual(data['something-else'], 'another')
+        # Ensure a key present in other variants is not there.
+        self.assertTrue('Purpose' not in data)
+
+    @mock.patch('os.pathsep', ';')
+    def test_search_path_on_windows(self):
+        # On windows, the search path is separated by ';' chars.
+        self.environ['BOTO_DATA_PATH'] = 'c:\\path1;c:\\path2'
+        # The builtin botocore data path is added as the last element
+        # so we're only interested in checking the two that we've added.
+        loader = Loader(data_path=self.environ['BOTO_DATA_PATH'])
+        paths = loader.get_search_paths()[:-1]
+        self.assertEqual(paths, ['c:\\path1', 'c:\\path2'])
+
+
+if __name__ == "__main__":
+    unittest.main()