Repository: boto/botocore
Pull request #582 (2015-06-19 18:13:07)

Boto3 doc update
This pull request is needed for the impending boto3 pull request. It pretty much involves adding a few more methods/attributes to the ``DocumentStructure`` class. The functionality added includes:

* a ``remove_all_sections()`` method that removes all subsections
* a ``clear_text()`` method that clears all of the text for a section, but does not erase the text in subsections.
* a ``meta`` attribute that allows you to store extra data about a section.
* a way to document python attributes in bcdoc.

There is also some logic in recording the shape that represents a section via the new ``meta`` attribute. This is useful for documenting the dynamodb customizations in boto3.

I tried to extract out the commits the best I can to give you a better sense of every change that is made. Might be better to review each commit one at a time.

cc @jamesls @mtdowling 

jamesls: :shipit: Looks good.

Patch:
diff --git a/botocore/docs/bcdoc/restdoc.py b/botocore/docs/bcdoc/restdoc.py
index 2cbd29ef8..9d67ec74a 100644
--- a/botocore/docs/bcdoc/restdoc.py
+++ b/botocore/docs/bcdoc/restdoc.py
@@ -103,7 +103,7 @@ class ReSTDocument(object):
 
 
 class DocumentStructure(ReSTDocument):
-    def __init__(self, name, section_names=None, target='man'):
+    def __init__(self, name, section_names=None, target='man', context=None):
         """Provides a Hierarichial structure to a ReSTDocument
 
         You can write to it similiar to as you can to a ReSTDocument but
@@ -112,12 +112,17 @@ class DocumentStructure(ReSTDocument):
         :param name: The name of the document
         :param section_names: A list of sections to be included
             in the document.
-        :parma target: The target documentation of the Document structure
+        :param target: The target documentation of the Document structure
+        :param context: A dictionary of data to store with the strucuture. These
+            are only stored per section not the entire structure.
         """
         super(DocumentStructure, self).__init__(target=target)
         self._name = name
         self._structure = OrderedDict()
         self._path = [self._name]
+        self._context = {}
+        if context is not None:
+            self._context = context
         if section_names is not None:
             self._generate_structure(section_names)
 
@@ -142,11 +147,15 @@ class DocumentStructure(ReSTDocument):
     def available_sections(self):
         return list(self._structure)
 
+    @property
+    def context(self):
+        return self._context
+
     def _generate_structure(self, section_names):
         for section_name in section_names:
             self.add_new_section(section_name)
 
-    def add_new_section(self, name):
+    def add_new_section(self, name, context=None):
         """Adds a new section to the current document structure
 
         This document structure will be considered a section to the
@@ -155,12 +164,15 @@ class DocumentStructure(ReSTDocument):
         as well
 
         :param name: The name of the section.
+        :param context: A dictionary of data to store with the strucuture. These
+            are only stored per section not the entire structure.
         :rtype: DocumentStructure
         :returns: A new document structure to add to but lives as a section
             to the document structure it was instantiated from.
         """
         # Add a new section
-        section = self.__class__(name=name, target=self.target)
+        section = self.__class__(name=name, target=self.target,
+                                 context=context)
         section.path = self.path + [name]
         # Indent the section apporpriately as well
         section.style.indentation = self.style.indentation
@@ -198,5 +210,8 @@ class DocumentStructure(ReSTDocument):
     def getvalue(self):
         return ''.join(self._writes).encode('utf-8')
 
+    def remove_all_sections(self):
+        self._structure = OrderedDict()
+
     def clear_text(self):
         self._writes = []
diff --git a/botocore/docs/bcdoc/style.py b/botocore/docs/bcdoc/style.py
index deefb136d..83563f376 100644
--- a/botocore/docs/bcdoc/style.py
+++ b/botocore/docs/bcdoc/style.py
@@ -362,6 +362,16 @@ class ReSTStyle(BaseStyle):
         self.dedent()
         self.new_paragraph()
 
+    def start_sphinx_py_attr(self, attr_name):
+        self.new_paragraph()
+        self.doc.write('.. py:attribute:: %s' % attr_name)
+        self.indent()
+        self.new_paragraph()
+
+    def end_sphinx_py_attr(self):
+        self.dedent()
+        self.new_paragraph()
+
     def write_py_doc_string(self, docstring):
         docstring_lines = docstring.splitlines()
         for docstring_line in docstring_lines:
diff --git a/botocore/docs/client.py b/botocore/docs/client.py
index e435a9f2d..a887f739e 100644
--- a/botocore/docs/client.py
+++ b/botocore/docs/client.py
@@ -46,13 +46,7 @@ class ClientDocumenter(object):
             'A low-level client representing %s' % official_service_name)
 
         # Write out the client example instantiation.
-        section.style.start_codeblock()
-        section.style.new_line()
-        section.write(
-            'client = session.create_client(\'{service}\')'.format(
-                service=self._service_name)
-        )
-        section.style.end_codeblock()
+        self._add_client_creation_example(section)
 
         # List out all of the possible client methods.
         section.style.new_line()
@@ -65,6 +59,15 @@ class ClientDocumenter(object):
         section.style.start_sphinx_py_class(
             class_name='%s.Client' % self._client.__class__.__name__)
 
+    def _add_client_creation_example(self, section):
+        section.style.start_codeblock()
+        section.style.new_line()
+        section.write(
+            'client = session.create_client(\'{service}\')'.format(
+                service=self._service_name)
+        )
+        section.style.end_codeblock()
+
     def _add_client_methods(self, section, client_methods):
         section = section.add_new_section('methods')
         for method_name in sorted(client_methods):
diff --git a/botocore/docs/example.py b/botocore/docs/example.py
index 7d6c0a0e5..321e50489 100644
--- a/botocore/docs/example.py
+++ b/botocore/docs/example.py
@@ -62,11 +62,13 @@ class BaseExampleDocumenter(ShapeDocumenter):
 
     def document_shape_type_list(self, section, shape, history, include=None,
                                  exclude=None, **kwargs):
+        param_shape = shape.member
         list_section = section.add_new_section('list-value')
         self._start_nested_param(list_section, '[')
-        param_shape = shape.member
+        param_section = list_section.add_new_section(
+            'member', context={'shape': param_shape.name})
         self.traverse_and_document_shape(
-            section=list_section, shape=param_shape, history=history)
+            section=param_section, shape=param_shape, history=history)
         ending_comma_section = list_section.add_new_section('ending-comma')
         ending_comma_section.write(',')
         ending_bracket_section = list_section.add_new_section(
@@ -86,8 +88,10 @@ class BaseExampleDocumenter(ShapeDocumenter):
             param_section = section.add_new_section(param)
             param_section.write('\'%s\': ' % param)
             param_shape = input_members[param]
+            param_value_section = param_section.add_new_section(
+                'member-value', context={'shape': param_shape.name})
             self.traverse_and_document_shape(
-                section=param_section, shape=param_shape,
+                section=param_value_section, shape=param_shape,
                 history=history, name=param)
             if i < len(input_members) - 1:
                 ending_comma_section = param_section.add_new_section(
@@ -101,9 +105,11 @@ class BaseExampleDocumenter(ShapeDocumenter):
         map_section = section.add_new_section('map-value')
         self._start_nested_param(map_section, '{')
         value_shape = shape.value
-        key_section = map_section.add_new_section('key')
+        key_section = map_section.add_new_section(
+            'key', context={'shape': shape.key.name})
         key_section.write('\'string\': ')
-        value_section = map_section.add_new_section('value')
+        value_section = map_section.add_new_section(
+            'value', context={'shape': value_shape.name})
         self.traverse_and_document_shape(
             section=value_section, shape=value_shape, history=history)
         end_bracket_section = map_section.add_new_section('ending-bracket')
@@ -174,8 +180,10 @@ class RequestExampleDocumenter(BaseExampleDocumenter):
             param_section.write(param_format % param)
             param_section.write(operator)
             param_shape = input_members[param]
+            param_value_section = param_section.add_new_section(
+                'member-value', context={'shape': param_shape.name})
             self.traverse_and_document_shape(
-                section=param_section, shape=param_shape,
+                section=param_value_section, shape=param_shape,
                 history=history, name=param)
             if i < len(input_members) - 1:
                 ending_comma_section = param_section.add_new_section(
diff --git a/botocore/docs/params.py b/botocore/docs/params.py
index 32d128111..191fb6b3e 100644
--- a/botocore/docs/params.py
+++ b/botocore/docs/params.py
@@ -46,7 +46,8 @@ class BaseParamsDocumenter(ShapeDocumenter):
                                  exclude=None, **kwargs):
         self._add_member_documentation(section, shape, **kwargs)
         param_shape = shape.member
-        param_section = section.add_new_section(param_shape.name)
+        param_section = section.add_new_section(
+            param_shape.name, context={'shape': shape.member.name})
         self._start_nested_param(param_section)
         self.traverse_and_document_shape(
             section=param_section, shape=param_shape,
@@ -58,11 +59,13 @@ class BaseParamsDocumenter(ShapeDocumenter):
                                 exclude=None, **kwargs):
         self._add_member_documentation(section, shape, **kwargs)
 
-        key_section = section.add_new_section('key')
+        key_section = section.add_new_section(
+            'key', context={'shape': shape.key.name})
         self._start_nested_param(key_section)
         self._add_member_documentation(key_section, shape.key)
 
-        param_section = section.add_new_section(shape.value.name)
+        param_section = section.add_new_section(
+            shape.value.name, context={'shape': shape.value.name})
         param_section.style.indent()
         self._start_nested_param(param_section)
         self.traverse_and_document_shape(
@@ -81,9 +84,10 @@ class BaseParamsDocumenter(ShapeDocumenter):
         for param in members:
             if exclude and param in exclude:
                 continue
-            param_section = section.add_new_section(param)
-            self._start_nested_param(param_section)
             param_shape = members[param]
+            param_section = section.add_new_section(
+                param, context={'shape': param_shape.name})
+            self._start_nested_param(param_section)
             self.traverse_and_document_shape(
                 section=param_section, shape=param_shape,
                 history=history, name=param)
@@ -121,7 +125,8 @@ class ResponseParamsDocumenter(BaseParamsDocumenter):
         name_section.write('- ')
         if name is not None:
             name_section.style.bold('%s ' % name)
-        name_section.style.italics('(%s) -- ' % py_type)
+        type_section = section.add_new_section('param-type')
+        type_section.style.italics('(%s) -- ' % py_type)
 
         documentation_section = section.add_new_section('param-documentation')
         if shape.documentation:
@@ -145,9 +150,10 @@ class RequestParamsDocumenter(BaseParamsDocumenter):
         for i, param in enumerate(members):
             if exclude and param in exclude:
                 continue
-            param_section = section.add_new_section(param)
-            param_section.style.new_line()
             param_shape = members[param]
+            param_section = section.add_new_section(
+                param, context={'shape': param_shape.name})
+            param_section.style.new_line()
             is_required = param in shape.required_members
             self.traverse_and_document_shape(
                 section=param_section, shape=param_shape,
@@ -174,7 +180,8 @@ class RequestParamsDocumenter(BaseParamsDocumenter):
             name_section.write('- ')
             if name is not None:
                 name_section.style.bold('%s ' % name)
-            name_section.style.italics('(%s) -- ' % py_type)
+            type_section = section.add_new_section('param-type')
+            type_section.style.italics('(%s) -- ' % py_type)
 
         if is_required:
             is_required_section = section.add_new_section('is-required')


Test patch:
diff --git a/tests/integration/test_credentials.py b/tests/integration/test_credentials.py
index 2c51a5979..1a18390b8 100644
--- a/tests/integration/test_credentials.py
+++ b/tests/integration/test_credentials.py
@@ -46,7 +46,7 @@ class TestCredentialPrecedence(BaseEnvVar):
         # take precedence.
         os.environ['AWS_ACCESS_KEY_ID'] = 'env'
         os.environ['AWS_SECRET_ACCESS_KEY'] = 'env-secret'
-        os.environ['AWS_DEFAULT_PROFILE'] = 'test'
+        os.environ['BOTO_DEFAULT_PROFILE'] = 'test'
 
         s = self.create_session()
         credentials = s.get_credentials()
@@ -58,7 +58,8 @@ class TestCredentialPrecedence(BaseEnvVar):
     def test_access_secret_vs_profile_code(self, credentials_cls):
         # If all three are given, then the access/secret keys should
         # take precedence.
-        s = self.create_session(profile='test')
+        s = self.create_session()
+        s.profile = 'test'
 
         client = s.create_client('s3', aws_access_key_id='code',
                                  aws_secret_access_key='code-secret')
@@ -69,8 +70,9 @@ class TestCredentialPrecedence(BaseEnvVar):
     def test_profile_env_vs_code(self):
         # If the profile is set both by the env var and by code,
         # then the one set by code should take precedence.
-        os.environ['AWS_DEFAULT_PROFILE'] = 'test'
-        s = self.create_session(profile='default')
+        os.environ['BOTO_DEFAULT_PROFILE'] = 'test'
+        s = self.create_session()
+        s.profile = 'default'
 
         credentials = s.get_credentials()
 
@@ -101,7 +103,8 @@ class TestCredentialPrecedence(BaseEnvVar):
         #
         os.environ['AWS_ACCESS_KEY_ID'] = 'env'
         os.environ['AWS_SECRET_ACCESS_KEY'] = 'env-secret'
-        s = self.create_session(profile='test')
+        s = self.create_session()
+        s.profile = 'test'
 
         credentials = s.get_credentials()
 
diff --git a/tests/unit/docs/bcdoc/test_document.py b/tests/unit/docs/bcdoc/test_document.py
index 617828f26..dfd23c6f5 100644
--- a/tests/unit/docs/bcdoc/test_document.py
+++ b/tests/unit/docs/bcdoc/test_document.py
@@ -132,6 +132,21 @@ class TestDocumentStructure(unittest.TestCase):
             ['mysection', 'mysection2']
         )
 
+    def test_context(self):
+        context = {'Foo': 'Bar'}
+        section = self.doc_structure.add_new_section(
+            'mysection', context=context)
+        self.assertEqual(section.context, context)
+
+        # Make sure if context is not specified it is empty.
+        section = self.doc_structure.add_new_section('mysection2')
+        self.assertEqual(section.context, {})
+
+    def test_remove_all_sections(self):
+        self.doc_structure.add_new_section('mysection2')
+        self.doc_structure.remove_all_sections()
+        self.assertEqual(self.doc_structure.available_sections, [])
+
     def test_clear_text(self):
         self.doc_structure.write('Foo')
         self.doc_structure.clear_text()
diff --git a/tests/unit/docs/bcdoc/test_style.py b/tests/unit/docs/bcdoc/test_style.py
index e66c2a500..963b9d35f 100644
--- a/tests/unit/docs/bcdoc/test_style.py
+++ b/tests/unit/docs/bcdoc/test_style.py
@@ -225,6 +225,13 @@ class TestStyle(unittest.TestCase):
             style.doc.getvalue(),
             six.b('\n\n.. py:method:: method(foo=None)\n\n  \n\n'))
 
+    def test_sphinx_py_attr(self):
+        style = ReSTStyle(ReSTDocument())
+        style.start_sphinx_py_attr('Foo')
+        style.end_sphinx_py_attr()
+        self.assertEqual(style.doc.getvalue(),
+                         six.b('\n\n.. py:attribute:: Foo\n\n  \n\n'))
+
     def test_write_py_doc_string(self):
         style = ReSTStyle(ReSTDocument())
         docstring = (
diff --git a/tests/unit/docs/test_example.py b/tests/unit/docs/test_example.py
index e17afe257..04e4a111a 100644
--- a/tests/unit/docs/test_example.py
+++ b/tests/unit/docs/test_example.py
@@ -102,10 +102,12 @@ class TestTraverseAndDocumentShape(BaseExampleDocumenterTest):
             shape=self.operation_model.input_shape, history=[]
         )
         structure_section = self.doc_structure.get_section('structure-value')
+        print(self.event_emitter.emit.call_args_list[0][1]['section'].name)
         self.assertEqual(
             self.event_emitter.emit.call_args_list,
             [mock.call('docs.response-example.myservice.SampleOperation.Foo',
-                       section=structure_section.get_section('Foo')),
+                       section=structure_section.get_section(
+                           'Foo').get_section('member-value')),
              mock.call(('docs.response-example.myservice.SampleOperation'
                         '.complete-section'), section=self.doc_structure)]
         )
@@ -119,7 +121,8 @@ class TestTraverseAndDocumentShape(BaseExampleDocumenterTest):
         self.assertEqual(
             self.event_emitter.emit.call_args_list,
             [mock.call('docs.request-example.myservice.SampleOperation.Foo',
-                       section=structure_section.get_section('Foo')),
+                       section=structure_section.get_section(
+                           'Foo').get_section('member-value')),
              mock.call(('docs.request-example.myservice.SampleOperation'
                         '.complete-section'), section=self.doc_structure)]
         )