Repository: boto/botocore
Pull request #806 (2016-02-24 02:26:59)

Nested docstring lists
Fixes boto/boto3#310

There were three issues at play:

1. We were completely disabling newlines when going through lists
2. Nested lists need to be indented, but the top level list should not be.
3. The doc strings contain, in some cases, several spaces between the start of an `li` and the first instance of text. This causes all sorts of problems in rst.

To make things easier, I separated this PR into three commits, one for each of the items above. Item 3 is the trickiest customer, and required a significant change.

Basically, we need to have full context of the lists before we write them so that we can remove vestigial spaces. What I have done here is change the DocStringParser so that it writes to a tree, and then flushes it all out at close. This allows us to do inspections before anything is written.

We could, alternatively, pull in beautifulsoup to generate this tree for us. The advantage would be that it is far more robust, but at the cost of a large dependency that may take more time than our specialized need requires.

Another potential option is to modify `ReSTStyle` such that it generates a string that is then output to rst, allowing us to apply transformations after generation but before writing. Since this affects more than just doc strings, however, it could result in massive strings being buffered into memory. While we can't be sure of the size of doc strings, they aren't usually more than a few paragraphs.

cc @jamesls @rayluo @kyleknap 


Patch:
diff --git a/botocore/compat.py b/botocore/compat.py
index 028d02d65..936589c63 100644
--- a/botocore/compat.py
+++ b/botocore/compat.py
@@ -16,15 +16,10 @@ import datetime
 import sys
 import inspect
 import warnings
-import hashlib
-import logging
 
 from botocore.vendored import six
-from botocore.exceptions import MD5UnavailableError
 from botocore.vendored.requests.packages.urllib3 import exceptions
 
-logger = logging.getLogger(__name__)
-
 
 if six.PY3:
     from six.moves import http_client
@@ -242,30 +237,3 @@ def total_seconds(delta):
     day_in_seconds = delta.days * 24 * 3600.0
     micro_in_seconds = delta.microseconds / 10.0**6
     return day_in_seconds + delta.seconds + micro_in_seconds
-
-
-# Checks to see if md5 is available on this system. A given system might not
-# have access to it for various reasons, such as FIPS mode being enabled.
-try:
-    hashlib.md5()
-    MD5_AVAILABLE = True
-except ValueError:
-    MD5_AVAILABLE = False
-
-
-def get_md5(*args, **kwargs):
-    """
-    Attempts to get an md5 hashing object.
-
-    :param raise_error_if_unavailable: raise an error if md5 is unavailable on
-        this system. If False, None will be returned if it is unavailable.
-    :type raise_error_if_unavailable: bool
-    :param args: Args to pass to the MD5 constructor
-    :param kwargs: Key word arguments to pass to the MD5 constructor
-    :return: An MD5 hashing object if available. If it is unavailable, None
-        is returned if raise_error_if_unavailable is set to False.
-    """
-    if MD5_AVAILABLE:
-        return hashlib.md5(*args, **kwargs)
-    else:
-        raise MD5UnavailableError()
diff --git a/botocore/docs/bcdoc/docstringparser.py b/botocore/docs/bcdoc/docstringparser.py
index d00471817..868bd5d89 100644
--- a/botocore/docs/bcdoc/docstringparser.py
+++ b/botocore/docs/bcdoc/docstringparser.py
@@ -11,10 +11,9 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 from botocore.compat import six
-from six.moves import html_parser
 
 
-class DocStringParser(html_parser.HTMLParser):
+class DocStringParser(six.moves.html_parser.HTMLParser):
     """
     A simple HTML parser.  Focused on converting the subset of HTML
     that appears in the documentation strings of the JSON models into
@@ -22,30 +21,180 @@ class DocStringParser(html_parser.HTMLParser):
     """
 
     def __init__(self, doc):
-        html_parser.HTMLParser.__init__(self)
+        self.tree = None
         self.doc = doc
-        self.unhandled_tags = []
+        six.moves.html_parser.HTMLParser.__init__(self)
+
+    def reset(self):
+        six.moves.html_parser.HTMLParser.reset(self)
+        self.tree = HTMLTree(self.doc)
+
+    def feed(self, data):
+        # HTMLParser is an old style class, so the super() method will not work.
+        six.moves.html_parser.HTMLParser.feed(self, data)
+        self.tree.write()
+        self.tree = HTMLTree(self.doc)
+
+    def close(self):
+        six.moves.html_parser.HTMLParser.close(self)
+        # Write if there is anything remaining.
+        self.tree.write()
+        self.tree = HTMLTree(self.doc)
 
     def handle_starttag(self, tag, attrs):
-        handler_name = 'start_%s' % tag
-        if hasattr(self.doc.style, handler_name):
-            getattr(self.doc.style, handler_name)(attrs)
-        else:
-            self.unhandled_tags.append(tag)
+        self.tree.add_tag(tag, attrs=attrs)
 
     def handle_endtag(self, tag):
-        handler_name = 'end_%s' % tag
-        if hasattr(self.doc.style, handler_name):
-            getattr(self.doc.style, handler_name)()
+        self.tree.add_tag(tag, is_start=False)
 
     def handle_data(self, data):
-        if data.isspace():
-            data = ' '
+        self.tree.add_data(data)
+
+
+class HTMLTree(object):
+    """
+    A tree which handles HTML nodes. Designed to work with a python HTML parser,
+    meaning that the current_node will be the most recently opened tag. When
+    a tag is closed, the current_node moves up to the parent node.
+    """
+    def __init__(self, doc):
+        self.doc = doc
+        self.head = StemNode()
+        self.current_node = self.head
+        self.unhandled_tags = []
+
+    def add_tag(self, tag, attrs=None, is_start=True):
+        if not self._doc_has_handler(tag, is_start):
+            self.unhandled_tags.append(tag)
+            return
+
+        if is_start:
+            if tag == 'li':
+                node = LineItemNode(attrs)
+            else:
+                node = TagNode(tag, attrs)
+            self.current_node.add_child(node)
+            self.current_node = node
+        else:
+            self.current_node = self.current_node.parent
+
+    def _doc_has_handler(self, tag, is_start):
+        if is_start:
+            handler_name = 'start_%s' % tag
+        else:
+            handler_name = 'end_%s' % tag
+
+        return hasattr(self.doc.style, handler_name)
+
+    def add_data(self, data):
+        self.current_node.add_child(DataNode(data))
+
+    def write(self):
+        self.head.write(self.doc)
+
+
+class Node(object):
+    def __init__(self, parent=None):
+        self.parent = parent
+
+    def write(self, doc):
+        raise NotImplementedError
+
+
+class StemNode(Node):
+    def __init__(self, parent=None):
+        super(StemNode, self).__init__(parent)
+        self.children = []
+
+    def add_child(self, child):
+        child.parent = self
+        self.children.append(child)
+
+    def write(self, doc):
+        self._write_children(doc)
+
+    def _write_children(self, doc):
+        for child in self.children:
+            child.write(doc)
+
+
+class TagNode(StemNode):
+    """
+    A generic Tag node. It will verify that handlers exist before writing.
+    """
+    def __init__(self, tag, attrs=None, parent=None):
+        super(TagNode, self).__init__(parent)
+        self.attrs = attrs
+        self.tag = tag
+
+    def write(self, doc):
+        self._write_start(doc)
+        self._write_children(doc)
+        self._write_end(doc)
+
+    def _write_start(self, doc):
+        handler_name = 'start_%s' % self.tag
+        if hasattr(doc.style, handler_name):
+            getattr(doc.style, handler_name)(self.attrs)
+
+    def _write_end(self, doc):
+        handler_name = 'end_%s' % self.tag
+        if hasattr(doc.style, handler_name):
+            getattr(doc.style, handler_name)()
+
+
+class LineItemNode(TagNode):
+    def __init__(self, attrs=None, parent=None):
+        super(LineItemNode, self).__init__('li', attrs, parent)
+
+    def write(self, doc):
+        self._lstrip(self)
+        super(LineItemNode, self).write(doc)
+
+    def _lstrip(self, node):
+        """
+        Traverses the tree, stripping out whitespace until text data is found
+        :param node: The node to strip
+        :return: True if non-whitespace data was found, False otherwise
+        """
+        for child in node.children:
+            if isinstance(child, DataNode):
+                child.lstrip()
+                if child.data:
+                    return True
+            else:
+                found = self._lstrip(child)
+                if found:
+                    return True
+
+        return False
+
+
+class DataNode(Node):
+    """
+    A Node that contains only string data.
+    """
+    def __init__(self, data, parent=None):
+        super(DataNode, self).__init__(parent)
+        if not isinstance(data, six.string_types):
+            raise ValueError("Expecting string type, %s given." % type(data))
+        self.data = data
+
+    def lstrip(self):
+        self.data = self.data.lstrip()
+
+    def write(self, doc):
+        if not self.data:
+            return
+
+        if self.data.isspace():
+            str_data = ' '
         else:
-            end_space = data[-1].isspace()
-            words = data.split()
-            words = self.doc.translate_words(words)
-            data = ' '.join(words)
+            end_space = self.data[-1].isspace()
+            words = self.data.split()
+            words = doc.translate_words(words)
+            str_data = ' '.join(words)
             if end_space:
-                data += ' '
-        self.doc.handle_data(data)
+                str_data += ' '
+
+        doc.handle_data(str_data)
diff --git a/botocore/docs/bcdoc/style.py b/botocore/docs/bcdoc/style.py
index 83563f376..fd5570d4d 100644
--- a/botocore/docs/bcdoc/style.py
+++ b/botocore/docs/bcdoc/style.py
@@ -70,14 +70,13 @@ class ReSTStyle(BaseStyle):
         BaseStyle.__init__(self, doc, indent_width)
         self.do_p = True
         self.a_href = None
+        self.list_depth = 0
 
     def new_paragraph(self):
-        if self.do_p:
-            self.doc.write('\n\n%s' % self.spaces())
+        self.doc.write('\n\n%s' % self.spaces())
 
     def new_line(self):
-        if self.do_p:
-            self.doc.write('\n%s' % self.spaces())
+        self.doc.write('\n%s' % self.spaces())
 
     def _start_inline(self, markup):
         self.doc.write(markup)
@@ -261,16 +260,28 @@ class ReSTStyle(BaseStyle):
             self.end_li()
 
     def start_ul(self, attrs=None):
+        if self.list_depth != 0:
+            self.indent()
+        self.list_depth += 1
         self.new_paragraph()
 
     def end_ul(self):
+        self.list_depth -= 1
+        if self.list_depth != 0:
+            self.dedent()
         self.new_paragraph()
 
     def start_ol(self, attrs=None):
         # TODO: Need to control the bullets used for LI items
+        if self.list_depth != 0:
+            self.indent()
+        self.list_depth += 1
         self.new_paragraph()
 
     def end_ol(self):
+        self.list_depth -= 1
+        if self.list_depth != 0:
+            self.dedent()
         self.new_paragraph()
 
     def start_examples(self, attrs=None):
diff --git a/botocore/exceptions.py b/botocore/exceptions.py
index c4f1404c9..7ccc89265 100644
--- a/botocore/exceptions.py
+++ b/botocore/exceptions.py
@@ -367,7 +367,3 @@ class InvalidConfigError(BotoCoreError):
 
 class RefreshWithMFAUnsupportedError(BotoCoreError):
     fmt = 'Cannot refresh credentials: MFA token required.'
-
-
-class MD5UnavailableError(BotoCoreError):
-    fmt = "This system does not support MD5 generation."
diff --git a/botocore/handlers.py b/botocore/handlers.py
index 7a384e64f..1b8f07b9a 100644
--- a/botocore/handlers.py
+++ b/botocore/handlers.py
@@ -17,6 +17,7 @@ This module contains builtin handlers for events emitted by botocore.
 """
 
 import base64
+import hashlib
 import logging
 import xml.etree.cElementTree
 import copy
@@ -24,7 +25,7 @@ import re
 import warnings
 
 from botocore.compat import urlsplit, urlunsplit, unquote, \
-    json, quote, six, unquote_str, ensure_bytes, get_md5, MD5_AVAILABLE
+    json, quote, six, unquote_str, ensure_bytes
 from botocore.docs.utils import AutoPopulatedParam
 from botocore.docs.utils import HideParamFromOperations
 from botocore.docs.utils import AppendParamDocumentation
@@ -133,13 +134,13 @@ def calculate_md5(params, **kwargs):
 
 
 def _calculate_md5_from_bytes(body_bytes):
-    md5 = get_md5(body_bytes)
+    md5 = hashlib.md5(body_bytes)
     return md5.digest()
 
 
 def _calculate_md5_from_file(fileobj):
     start_position = fileobj.tell()
-    md5 = get_md5()
+    md5 = hashlib.md5()
     for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
         md5.update(chunk)
     fileobj.seek(start_position)
@@ -149,7 +150,7 @@ def _calculate_md5_from_file(fileobj):
 def conditionally_calculate_md5(params, **kwargs):
     """Only add a Content-MD5 when not using sigv4"""
     signer = kwargs['request_signer']
-    if signer.signature_version not in ['v4', 's3v4'] and MD5_AVAILABLE:
+    if signer.signature_version not in ['v4', 's3v4']:
         calculate_md5(params, **kwargs)
 
 
@@ -187,14 +188,13 @@ def copy_source_sse_md5(params, **kwargs):
 def _sse_md5(params, sse_member_prefix='SSECustomer'):
     if not _needs_s3_sse_customization(params, sse_member_prefix):
         return
-
     sse_key_member = sse_member_prefix + 'Key'
     sse_md5_member = sse_member_prefix + 'KeyMD5'
     key_as_bytes = params[sse_key_member]
     if isinstance(key_as_bytes, six.text_type):
         key_as_bytes = key_as_bytes.encode('utf-8')
     key_md5_str = base64.b64encode(
-        get_md5(key_as_bytes).digest()).decode('utf-8')
+        hashlib.md5(key_as_bytes).digest()).decode('utf-8')
     key_b64_encoded = base64.b64encode(key_as_bytes).decode('utf-8')
     params[sse_key_member] = key_b64_encoded
     params[sse_md5_member] = key_md5_str
diff --git a/botocore/serialize.py b/botocore/serialize.py
index 4917264cd..9873c561e 100644
--- a/botocore/serialize.py
+++ b/botocore/serialize.py
@@ -62,8 +62,7 @@ def create_serializer(protocol_name, include_validation=True):
     serializer = SERIALIZERS[protocol_name]()
     if include_validation:
         validator = validate.ParamValidator()
-        serializer = validate.ParamValidationDecorator(validator, serializer)
-    return serializer
+    return validate.ParamValidationDecorator(validator, serializer)
 
 
 class Serializer(object):
diff --git a/botocore/signers.py b/botocore/signers.py
index d586b8d3f..846dfe871 100644
--- a/botocore/signers.py
+++ b/botocore/signers.py
@@ -150,14 +150,7 @@ class RequestSigner(object):
         if cls is None:
             raise UnknownSignatureVersionError(
                 signature_version=signature_version)
-        # If there's no credentials provided (i.e credentials is None),
-        # then we'll pass a value of "None" over to the auth classes,
-        # which already handle the cases where no credentials have
-        # been provided.
-        frozen_credentials = None
-        if self._credentials is not None:
-            frozen_credentials = self._credentials.get_frozen_credentials()
-        kwargs['credentials'] = frozen_credentials
+        kwargs['credentials'] = self._credentials.get_frozen_credentials()
         if cls.REQUIRES_REGION:
             if self._region_name is None:
                 raise botocore.exceptions.NoRegionError()


Test patch:
diff --git a/tests/unit/docs/bcdoc/test_docstringparser.py b/tests/unit/docs/bcdoc/test_docstringparser.py
new file mode 100644
index 000000000..23acdea1d
--- /dev/null
+++ b/tests/unit/docs/bcdoc/test_docstringparser.py
@@ -0,0 +1,212 @@
+# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+import mock
+from tests import unittest
+
+import botocore.docs.bcdoc.docstringparser as parser
+from botocore.docs.bcdoc.restdoc import ReSTDocument
+
+
+class TestDocStringParser(unittest.TestCase):
+    def parse(self, html):
+        docstring_parser = parser.DocStringParser(ReSTDocument())
+        docstring_parser.feed(html)
+        docstring_parser.close()
+        return docstring_parser.doc.getvalue()
+
+    def assert_contains_exact_lines_in_order(self, actual, expected):
+        # Get each line and filter out empty lines
+        contents = actual.split(b'\n')
+        contents = [line for line in contents if line and not line.isspace()]
+
+        for line in expected:
+            self.assertIn(line, contents)
+            beginning = contents.index(line)
+            contents = contents[beginning:]
+
+    def test_nested_lists(self):
+        html = "<ul><li>Wello</li><ul><li>Horld</li></ul></ul>"
+        result = self.parse(html)
+        self.assert_contains_exact_lines_in_order(result, [
+            b'* Wello',
+            b'  * Horld'
+        ])
+
+    def test_nested_lists_with_extra_white_space(self):
+        html = "<ul> <li> Wello</li><ul> <li> Horld</li></ul></ul>"
+        result = self.parse(html)
+        self.assert_contains_exact_lines_in_order(result, [
+            b'* Wello',
+            b'  * Horld'
+        ])
+
+
+class TestHTMLTree(unittest.TestCase):
+    def setUp(self):
+        self.style = mock.Mock()
+        self.doc = mock.Mock()
+        self.doc.style = self.style
+        self.tree = parser.HTMLTree(self.doc)
+
+    def test_add_tag(self):
+        self.tree.add_tag('foo')
+        self.assertIsInstance(self.tree.current_node, parser.TagNode)
+        self.assertEqual(self.tree.current_node.tag, 'foo')
+
+    def test_add_unsupported_tag(self):
+        del self.style.start_foo
+        del self.style.end_foo
+        self.tree.add_tag('foo')
+        self.assertIn('foo', self.tree.unhandled_tags)
+
+    def test_add_data(self):
+        self.tree.add_data('foo')
+        self.assertNotIsInstance(self.tree.current_node, parser.DataNode)
+        node = self.tree.head.children[0]
+        self.assertIsInstance(node, parser.DataNode)
+        self.assertEqual(node.data, 'foo')
+
+
+class TestStemNode(unittest.TestCase):
+    def setUp(self):
+        self.style = mock.Mock()
+        self.doc = mock.Mock()
+        self.doc.style = self.style
+        self.node = parser.StemNode()
+
+    def test_add_child(self):
+        child = parser.StemNode()
+        self.node.add_child(child)
+        self.assertIn(child, self.node.children)
+        self.assertEqual(child.parent, self.node)
+
+    def test_write(self):
+        self.node.add_child(mock.Mock())
+        self.node.add_child(mock.Mock())
+
+        self.node.write(mock.Mock())
+        for child in self.node.children:
+            self.assertTrue(child.write.called)
+
+
+class TestTagNode(unittest.TestCase):
+    def setUp(self):
+        self.style = mock.Mock()
+        self.doc = mock.Mock()
+        self.doc.style = self.style
+        self.tag = 'foo'
+        self.node = parser.TagNode(self.tag)
+
+    def test_write_calls_style(self):
+        self.node.write(self.doc)
+        self.assertTrue(self.style.start_foo.called)
+        self.assertTrue(self.style.end_foo.called)
+
+    def test_write_unsupported_tag(self):
+        del self.style.start_foo
+        del self.style.end_foo
+
+        try:
+            self.node.write(self.doc)
+        except AttributeError as e:
+            self.fail(str(e))
+
+
+class TestDataNode(unittest.TestCase):
+    def setUp(self):
+        self.style = mock.Mock()
+        self.doc = mock.Mock()
+        self.doc.style = self.style
+
+    def test_string_data(self):
+        node = parser.DataNode('foo')
+        self.assertEqual(node.data, 'foo')
+
+    def test_non_string_data_raises_error(self):
+        with self.assertRaises(ValueError):
+            parser.DataNode(5)
+
+    def test_lstrip(self):
+        node = parser.DataNode(' foo')
+        node.lstrip()
+        self.assertEqual(node.data, 'foo')
+
+    def test_write(self):
+        node = parser.DataNode('foo   bar baz')
+        self.doc.translate_words.return_value = ['foo', 'bar', 'baz']
+        node.write(self.doc)
+        self.doc.handle_data.assert_called_once_with('foo bar baz')
+
+    def test_write_space(self):
+        node = parser.DataNode(' ')
+        node.write(self.doc)
+        self.doc.handle_data.assert_called_once_with(' ')
+        self.doc.handle_data.reset_mock()
+
+        node = parser.DataNode('              ')
+        node.write(self.doc)
+        self.doc.handle_data.assert_called_once_with(' ')
+
+    def test_write_empty_string(self):
+        node = parser.DataNode('')
+        node.write(self.doc)
+        self.assertFalse(self.doc.handle_data.called)
+
+
+class TestLineItemNode(unittest.TestCase):
+    def setUp(self):
+        self.style = mock.Mock()
+        self.doc = mock.Mock()
+        self.doc.style = self.style
+        self.doc.translate_words.return_value = ['foo']
+        self.node = parser.LineItemNode()
+
+    def test_write_strips_white_space(self):
+        self.node.add_child(parser.DataNode('  foo'))
+        self.node.write(self.doc)
+        self.doc.handle_data.assert_called_once_with('foo')
+
+    def test_write_strips_nested_white_space(self):
+        self.node.add_child(parser.DataNode('  '))
+        tag_child = parser.TagNode('foo')
+        tag_child.add_child(parser.DataNode('  '))
+        tag_child_2 = parser.TagNode('foo')
+        tag_child_2.add_child(parser.DataNode(' foo'))
+        tag_child.add_child(tag_child_2)
+        self.node.add_child(tag_child)
+
+        self.node.write(self.doc)
+        self.doc.handle_data.assert_called_once_with('foo')
+
+    def test_write_only_strips_until_text_is_found(self):
+        self.node.add_child(parser.DataNode('  '))
+        tag_child = parser.TagNode('foo')
+        tag_child.add_child(parser.DataNode('  '))
+        tag_child_2 = parser.TagNode('foo')
+        tag_child_2.add_child(parser.DataNode(' foo'))
+        tag_child_2.add_child(parser.DataNode(' '))
+        tag_child.add_child(tag_child_2)
+        self.node.add_child(tag_child)
+
+        self.node.write(self.doc)
+
+        calls = [mock.call('foo'), mock.call(' ')]
+        self.doc.handle_data.assert_has_calls(calls)
diff --git a/tests/unit/docs/bcdoc/test_style.py b/tests/unit/docs/bcdoc/test_style.py
index 963b9d35f..b89e0d310 100644
--- a/tests/unit/docs/bcdoc/test_style.py
+++ b/tests/unit/docs/bcdoc/test_style.py
@@ -241,3 +241,41 @@ class TestStyle(unittest.TestCase):
         )
         style.write_py_doc_string(docstring)
         self.assertEqual(style.doc.getvalue(), six.b(docstring + '\n'))
+
+    def test_new_line(self):
+        style = ReSTStyle(ReSTDocument())
+        style.new_line()
+        self.assertEqual(style.doc.getvalue(), six.b('\n'))
+
+        style.do_p = False
+        style.new_line()
+        self.assertEqual(style.doc.getvalue(), six.b('\n\n'))
+
+    def test_list(self):
+        style = ReSTStyle(ReSTDocument())
+        style.li('foo')
+        self.assertEqual(style.doc.getvalue(), six.b('\n* foo\n\n'))
+
+    def test_non_top_level_lists_are_indented(self):
+        style = ReSTStyle(ReSTDocument())
+
+        # Start the top level list
+        style.start_ul()
+
+        # Write one list element
+        style.start_li()
+        style.doc.handle_data('foo')
+        style.end_li()
+
+        self.assertEqual(style.doc.getvalue(), six.b("\n\n\n* foo\n"))
+
+        # Start the nested list
+        style.start_ul()
+
+        # Write an element to the nested list
+        style.start_li()
+        style.doc.handle_data('bar')
+        style.end_li()
+
+        self.assertEqual(style.doc.getvalue(),
+                         six.b("\n\n\n* foo\n\n\n  \n  * bar\n  "))
diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py
index 1683ed5b2..fbaae3369 100644
--- a/tests/unit/test_compat.py
+++ b/tests/unit/test_compat.py
@@ -12,12 +12,8 @@
 # language governing permissions and limitations under the License.
 
 import datetime
-import mock
 
-from botocore.exceptions import MD5UnavailableError
-from botocore.compat import (
-    total_seconds, unquote_str, six, ensure_bytes, get_md5
-)
+from botocore.compat import total_seconds, unquote_str, six, ensure_bytes
 
 from tests import BaseEnvVar, unittest
 
@@ -82,16 +78,3 @@ class TestEnsureBytes(unittest.TestCase):
         value = 500
         with self.assertRaises(ValueError):
             ensure_bytes(value)
-
-
-class TestGetMD5(unittest.TestCase):
-    def test_available(self):
-        md5 = mock.Mock()
-        with mock.patch('botocore.compat.MD5_AVAILABLE', True):
-            with mock.patch('hashlib.md5', mock.Mock(return_value=md5)):
-                self.assertEqual(get_md5(), md5)
-
-    def test_unavailable_raises_error(self):
-        with mock.patch('botocore.compat.MD5_AVAILABLE', False):
-            with self.assertRaises(MD5UnavailableError):
-                get_md5()
diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py
index 9d56765e2..bcbbf7668 100644
--- a/tests/unit/test_handlers.py
+++ b/tests/unit/test_handlers.py
@@ -20,7 +20,7 @@ import os
 
 import botocore
 import botocore.session
-from botocore.exceptions import ParamValidationError, MD5UnavailableError
+from botocore.exceptions import ParamValidationError
 from botocore.awsrequest import AWSRequest
 from botocore.compat import quote, six
 from botocore.model import OperationModel, ServiceModel
@@ -199,6 +199,45 @@ class TestHandlers(BaseSessionTest):
         # object is None.  We need to handle this case.
         handlers.check_for_200_error(None)
 
+    def test_sse_params(self):
+        for op in ('HeadObject', 'GetObject', 'PutObject', 'CopyObject',
+                   'CreateMultipartUpload', 'UploadPart', 'UploadPartCopy'):
+            event = 'before-parameter-build.s3.%s' % op
+            params = {'SSECustomerKey': b'bar',
+                      'SSECustomerAlgorithm': 'AES256'}
+            self.session.emit(event, params=params, model=mock.Mock())
+            self.assertEqual(params['SSECustomerKey'], 'YmFy')
+            self.assertEqual(params['SSECustomerKeyMD5'],
+                             'N7UdGUp1E+RbVvZSTy1R8g==')
+
+    def test_sse_params_as_str(self):
+        event = 'before-parameter-build.s3.PutObject'
+        params = {'SSECustomerKey': 'bar',
+                  'SSECustomerAlgorithm': 'AES256'}
+        self.session.emit(event, params=params, model=mock.Mock())
+        self.assertEqual(params['SSECustomerKey'], 'YmFy')
+        self.assertEqual(params['SSECustomerKeyMD5'],
+                         'N7UdGUp1E+RbVvZSTy1R8g==')
+
+    def test_copy_source_sse_params(self):
+        for op in ['CopyObject', 'UploadPartCopy']:
+            event = 'before-parameter-build.s3.%s' % op
+            params = {'CopySourceSSECustomerKey': b'bar',
+                      'CopySourceSSECustomerAlgorithm': 'AES256'}
+            self.session.emit(event, params=params, model=mock.Mock())
+            self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
+            self.assertEqual(params['CopySourceSSECustomerKeyMD5'],
+                             'N7UdGUp1E+RbVvZSTy1R8g==')
+
+    def test_copy_source_sse_params_as_str(self):
+        event = 'before-parameter-build.s3.CopyObject'
+        params = {'CopySourceSSECustomerKey': 'bar',
+                  'CopySourceSSECustomerAlgorithm': 'AES256'}
+        self.session.emit(event, params=params, model=mock.Mock())
+        self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
+        self.assertEqual(params['CopySourceSSECustomerKeyMD5'],
+                         'N7UdGUp1E+RbVvZSTy1R8g==')
+
     def test_route53_resource_id(self):
         event = 'before-parameter-build.route53.GetHostedZone'
         params = {'Id': '/hostedzone/ABC123',
@@ -470,7 +509,60 @@ class TestHandlers(BaseSessionTest):
         handlers.switch_host_with_param(request, 'PredictEndpoint')
         self.assertEqual(request.url, new_endpoint)
 
+    def test_does_not_add_md5_when_v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' not in request_dict['headers'])
 
+    def test_does_not_add_md5_when_s3v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' not in request_dict['headers'])
+
+    def test_adds_md5_when_not_v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' in request_dict['headers'])
+
+    def test_adds_md5_with_file_like_body(self):
+        request_dict = {
+            'body': six.BytesIO(b'foobar'),
+            'headers': {}
+        }
+        handlers.calculate_md5(request_dict)
+        self.assertEqual(request_dict['headers']['Content-MD5'],
+                         'OFj2IjCsPJFfMAxmQxLGPw==')
+
+    def test_adds_md5_with_bytes_object(self):
+        request_dict = {
+            'body': b'foobar',
+            'headers': {}
+        }
+        handlers.calculate_md5(request_dict)
+        self.assertEqual(
+            request_dict['headers']['Content-MD5'],
+            'OFj2IjCsPJFfMAxmQxLGPw==')
 
     def test_invalid_char_in_bucket_raises_exception(self):
         params = {
@@ -643,164 +735,3 @@ class TestRetryHandlerOrder(BaseSessionTest):
         self.assertTrue(s3_200_handler < general_retry_handler,
                         "S3 200 error handler was supposed to be before "
                         "the general retry handler, but it was not.")
-
-
-class BaseMD5Test(BaseSessionTest):
-    def setUp(self, **environ):
-        super(BaseMD5Test, self).setUp(**environ)
-        self.md5_object = mock.Mock()
-        self.md5_digest = mock.Mock(return_value=b'foo')
-        self.md5_object.digest = self.md5_digest
-        md5_builder = mock.Mock(return_value=self.md5_object)
-        self.md5_patch = mock.patch('hashlib.md5', md5_builder)
-        self.md5_patch.start()
-        self._md5_available_patch = None
-        self.set_md5_available()
-
-    def tearDown(self):
-        super(BaseMD5Test, self).tearDown()
-        self.md5_patch.stop()
-        if self._md5_available_patch:
-            self._md5_available_patch.stop()
-
-    def set_md5_available(self, is_available=True):
-        if self._md5_available_patch:
-            self._md5_available_patch.stop()
-
-        self._md5_available_patch = \
-            mock.patch('botocore.compat.MD5_AVAILABLE', is_available)
-        self._md5_available_patch.start()
-
-
-class TestSSEMD5(BaseMD5Test):
-    def test_raises_error_when_md5_unavailable(self):
-        self.set_md5_available(False)
-
-        with self.assertRaises(MD5UnavailableError):
-            handlers.sse_md5({'SSECustomerKey': b'foo'})
-
-        with self.assertRaises(MD5UnavailableError):
-            handlers.copy_source_sse_md5({'CopySourceSSECustomerKey': b'foo'})
-
-    def test_sse_params(self):
-        for op in ('HeadObject', 'GetObject', 'PutObject', 'CopyObject',
-                   'CreateMultipartUpload', 'UploadPart', 'UploadPartCopy'):
-            event = 'before-parameter-build.s3.%s' % op
-            params = {'SSECustomerKey': b'bar',
-                      'SSECustomerAlgorithm': 'AES256'}
-            self.session.emit(event, params=params, model=mock.Mock())
-            self.assertEqual(params['SSECustomerKey'], 'YmFy')
-            self.assertEqual(params['SSECustomerKeyMD5'], 'Zm9v')
-
-    def test_sse_params_as_str(self):
-        event = 'before-parameter-build.s3.PutObject'
-        params = {'SSECustomerKey': 'bar',
-                  'SSECustomerAlgorithm': 'AES256'}
-        self.session.emit(event, params=params, model=mock.Mock())
-        self.assertEqual(params['SSECustomerKey'], 'YmFy')
-        self.assertEqual(params['SSECustomerKeyMD5'], 'Zm9v')
-
-    def test_copy_source_sse_params(self):
-        for op in ['CopyObject', 'UploadPartCopy']:
-            event = 'before-parameter-build.s3.%s' % op
-            params = {'CopySourceSSECustomerKey': b'bar',
-                      'CopySourceSSECustomerAlgorithm': 'AES256'}
-            self.session.emit(event, params=params, model=mock.Mock())
-            self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
-            self.assertEqual(params['CopySourceSSECustomerKeyMD5'], 'Zm9v')
-
-    def test_copy_source_sse_params_as_str(self):
-        event = 'before-parameter-build.s3.CopyObject'
-        params = {'CopySourceSSECustomerKey': 'bar',
-                  'CopySourceSSECustomerAlgorithm': 'AES256'}
-        self.session.emit(event, params=params, model=mock.Mock())
-        self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
-        self.assertEqual(params['CopySourceSSECustomerKeyMD5'], 'Zm9v')
-
-
-class TestAddMD5(BaseMD5Test):
-    def test_does_not_add_md5_when_v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' not in request_dict['headers'])
-
-    def test_does_not_add_md5_when_s3v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' not in request_dict['headers'])
-
-    def test_conditional_does_not_add_when_md5_unavailable(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-
-        self.set_md5_available(False)
-        with mock.patch('botocore.handlers.MD5_AVAILABLE', False):
-            handlers.conditionally_calculate_md5(
-                request_dict, request_signer=request_signer)
-            self.assertFalse('Content-MD5' in request_dict['headers'])
-
-    def test_add_md5_raises_error_when_md5_unavailable(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-
-        self.set_md5_available(False)
-        with self.assertRaises(MD5UnavailableError):
-            handlers.calculate_md5(
-                request_dict, request_signer=request_signer)
-
-    def test_adds_md5_when_not_v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' in request_dict['headers'])
-
-    def test_add_md5_with_file_like_body(self):
-        request_dict = {
-            'body': six.BytesIO(b'foobar'),
-            'headers': {}
-        }
-        self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
-        handlers.calculate_md5(request_dict)
-        self.assertEqual(request_dict['headers']['Content-MD5'],
-                         'OFj2IjCsPJFfMAxmQxLGPw==')
-
-    def test_add_md5_with_bytes_object(self):
-        request_dict = {
-            'body': b'foobar',
-            'headers': {}
-        }
-        self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
-        handlers.calculate_md5(request_dict)
-        self.assertEqual(
-            request_dict['headers']['Content-MD5'],
-            'OFj2IjCsPJFfMAxmQxLGPw==')
diff --git a/tests/unit/test_serialize.py b/tests/unit/test_serialize.py
index 97cfb0b53..800c96237 100644
--- a/tests/unit/test_serialize.py
+++ b/tests/unit/test_serialize.py
@@ -20,9 +20,6 @@ from tests import unittest
 from botocore.model import ServiceModel
 from botocore import serialize
 from botocore.compat import six
-from botocore.exceptions import ParamValidationError
-from botocore.serialize import Serializer
-from botocore.validate import ParamValidationDecorator
 
 
 class BaseModelWithBlob(unittest.TestCase):
@@ -330,75 +327,3 @@ class TestJSONTimestampSerialization(unittest.TestCase):
         body = json.loads(self.serialize_to_request(
             {'Timestamp': '1970-01-01'})['body'].decode('utf-8'))
         self.assertEqual(body['Timestamp'], 0)
-
-
-class TestInstanceCreation(unittest.TestCase):
-    def setUp(self):
-        self.model = {
-            'metadata': {'protocol': 'query', 'apiVersion': '2014-01-01'},
-            'documentation': '',
-            'operations': {
-                'TestOperation': {
-                    'name': 'TestOperation',
-                    'http': {
-                        'method': 'POST',
-                        'requestUri': '/',
-                    },
-                    'input': {'shape': 'InputShape'},
-                }
-            },
-            'shapes': {
-                'InputShape': {
-                    'type': 'structure',
-                    'members': {
-                        'Timestamp': {'shape': 'StringTestType'},
-                    }
-                },
-                'StringTestType': {
-                    'type': 'string',
-                    'min': 15
-                }
-            }
-        }
-        self.service_model = ServiceModel(self.model)
-
-    def assert_serialize_valid_parameter(self, request_serializer):
-        valid_string = 'valid_string_with_min_15_chars'
-        request = request_serializer.serialize_to_request(
-            {'Timestamp': valid_string},
-            self.service_model.operation_model('TestOperation'))
-
-        self.assertEqual(request['body']['Timestamp'], valid_string)
-
-    def assert_serialize_invalid_parameter(self, request_serializer):
-        invalid_string = 'short string'
-        request = request_serializer.serialize_to_request(
-            {'Timestamp': invalid_string},
-            self.service_model.operation_model('TestOperation'))
-
-        self.assertEqual(request['body']['Timestamp'], invalid_string)
-
-    def test_instantiate_without_validation(self):
-        request_serializer = serialize.create_serializer(
-            self.service_model.metadata['protocol'], False)
-
-        try:
-            self.assert_serialize_valid_parameter(request_serializer)
-        except ParamValidationError as e:
-            self.fail("Shouldn't fail serializing valid parameter without validation")
-
-        try:
-            self.assert_serialize_invalid_parameter(request_serializer)
-        except ParamValidationError as e:
-            self.fail("Shouldn't fail serializing invalid parameter without validation")
-
-    def test_instantiate_with_validation(self):
-        request_serializer = serialize.create_serializer(
-            self.service_model.metadata['protocol'], True)
-        try:
-            self.assert_serialize_valid_parameter(request_serializer)
-        except ParamValidationError as e:
-            self.fail("Shouldn't fail serializing valid parameter with validation")
-
-        with self.assertRaises(ParamValidationError):
-            self.assert_serialize_invalid_parameter(request_serializer)
diff --git a/tests/unit/test_signers.py b/tests/unit/test_signers.py
index 3760bad60..ec9bde350 100644
--- a/tests/unit/test_signers.py
+++ b/tests/unit/test_signers.py
@@ -254,25 +254,6 @@ class TestSigner(BaseSignerTest):
                 service_name='service_name',
                 region_name='region_name')
 
-    def test_no_credentials_case_is_forwarded_to_signer(self):
-        # If no credentials are given to the RequestSigner, we should
-        # forward that fact on to the Auth class and let them handle
-        # the error (which they already do).
-        self.credentials = None
-        self.signer = RequestSigner(
-            'service_name', 'region_name', 'signing_name',
-            'v4', self.credentials, self.emitter)
-        auth_cls = mock.Mock()
-        with mock.patch.dict(botocore.auth.AUTH_TYPE_MAPS,
-                             {'v4': auth_cls}):
-            auth = self.signer.get_auth_instance(
-                'service_name', 'region_name', 'v4')
-            auth_cls.assert_called_with(
-                service_name='service_name',
-                region_name='region_name',
-                credentials=None,
-            )
-
 
 class TestCloudfrontSigner(unittest.TestCase):
     def setUp(self):
@@ -296,11 +277,11 @@ class TestCloudfrontSigner(unittest.TestCase):
             ip_address='12.34.56.78/9')
         expected = {
             "Statement": [{
-                "Resource": "foo",
-                "Condition": {
-                    "DateGreaterThan": {"AWS:EpochTime": 1448928000},
-                    "DateLessThan": {"AWS:EpochTime": 1451606400},
-                    "IpAddress": {"AWS:SourceIp": "12.34.56.78/9"}
+                "Resource":"foo",
+                "Condition":{
+                    "DateGreaterThan":{"AWS:EpochTime":1448928000},
+                    "DateLessThan":{"AWS:EpochTime":1451606400},
+                    "IpAddress":{"AWS:SourceIp":"12.34.56.78/9"}
                 },
             }]
         }
@@ -339,10 +320,10 @@ class TestCloudfrontSigner(unittest.TestCase):
         expected = (
             'http://test.com/index.html?foo=bar'
             '&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiZm9vIiwiQ29uZ'
-            'Gl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIj'
-            'oxNDUxNjA2NDAwfSwiSXBBZGRyZXNzIjp7IkFXUzpTb3VyY2VJcCI'
-            '6IjEyLjM0LjU2Ljc4LzkifSwiRGF0ZUdyZWF0ZXJUaGFuIjp7IkFX'
-            'UzpFcG9jaFRpbWUiOjE0NDg5MjgwMDB9fX1dfQ__'
+                'Gl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIj'
+                'oxNDUxNjA2NDAwfSwiSXBBZGRyZXNzIjp7IkFXUzpTb3VyY2VJcCI'
+                '6IjEyLjM0LjU2Ljc4LzkifSwiRGF0ZUdyZWF0ZXJUaGFuIjp7IkFX'
+                'UzpFcG9jaFRpbWUiOjE0NDg5MjgwMDB9fX1dfQ__'
             '&Signature=c2lnbmVk&Key-Pair-Id=MY_KEY_ID')
         self.assertEqualUrl(signed_url, expected)