Repository: boto/botocore
Pull request #550 (2015-06-04 23:03:09)

Add the ability to customize botocore docs
I added the following in this PR:

* Two types of events: ``'docs.{section-in-documentation}.{service-name}.{operation-name}.{param-name}' `` that returns a section relevant to a specific parameter in the documentation and ``'docs.{section-in-documentation}.{service-name}.{operation-name}.complete-section' `` that returns a particular section for an entire method. 

* Add a customization that allows us to document auto-populated parameters. I tested this out with Glacier's accountId and checksum parameters. The auto-populated parameters are removed from the example, any required label is removed from them, and a note about the parameter being auto-populated is added to the parameter description.

* Add a functional test section to functionally test the docs for specific services. Pretty much adopted boto3's ci scripts.

I tried to make the section structures as consistent as possible, but there are some inconsistencies in structures due to formatting issues. Any suggestions on how to make these more consistent would be more than welcome.

Also this PR will not pass till this PR is merged: https://github.com/boto/bcdoc/pull/37

cc @jamesls @mtdowling


Patch:
diff --git a/.travis.yml b/.travis.yml
index 2fb4feb9b..b4affcc64 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,9 +4,9 @@ python:
   - "2.7"
   - "3.3"
   - "3.4"
+sudo: false
 install:
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install -r requirements26.txt; fi
-  - pip install -r requirements.txt
-  - pip install coverage python-coveralls
-script: nosetests tests/unit --cover-erase --with-coverage --cover-package botocore
-after_success: coveralls
+  - python scripts/ci/install
+script: python scripts/ci/run-tests
+after_success:
+  - pip install python-coveralls && cd tests && coveralls
diff --git a/botocore/docs/client.py b/botocore/docs/client.py
index ac16bfb9c..748a39ee3 100644
--- a/botocore/docs/client.py
+++ b/botocore/docs/client.py
@@ -97,6 +97,7 @@ class ClientDocumenter(object):
 
         document_model_driven_method(
             section, method_name, operation_model,
+            event_emitter=self._client.meta.events,
             method_description=operation_model.documentation,
             example_prefix='response = client.%s' % method_name
         )
diff --git a/botocore/docs/example.py b/botocore/docs/example.py
index 7a66c25b8..b4179f710 100644
--- a/botocore/docs/example.py
+++ b/botocore/docs/example.py
@@ -10,11 +10,11 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-from botocore.docs.utils import traverse_and_document_shape
+from botocore.docs.shape import ShapeDocumenter
 from botocore.docs.utils import py_default
 
 
-class BaseExampleDocumenter(object):
+class BaseExampleDocumenter(ShapeDocumenter):
     def document_example(self, section, shape, prefix=None, include=None,
                          exclude=None):
         """Generates an example based on a shape
@@ -38,8 +38,8 @@ class BaseExampleDocumenter(object):
         section.style.start_codeblock()
         if prefix is not None:
             section.write(prefix)
-        traverse_and_document_shape(
-            documenter=self, section=section, shape=shape, history=history,
+        self.traverse_and_document_shape(
+            section=section, shape=shape, history=history,
             include=include, exclude=exclude)
 
     def document_recursive_shape(self, section, shape, **kwargs):
@@ -62,16 +62,20 @@ class BaseExampleDocumenter(object):
 
     def document_shape_type_list(self, section, shape, history, include=None,
                                  exclude=None, **kwargs):
-        self._start_nested_param(section, '[')
+        list_section = section.add_new_section('list-value')
+        self._start_nested_param(list_section, '[')
         param_shape = shape.member
-        traverse_and_document_shape(
-            documenter=self, section=section, shape=param_shape,
-            history=history)
-        section.write(',')
-        self._end_nested_param(section, ']')
+        self.traverse_and_document_shape(
+            section=list_section, shape=param_shape, history=history)
+        ending_comma_section = list_section.add_new_section('ending-comma')
+        ending_comma_section.write(',')
+        ending_bracket_section = list_section.add_new_section(
+            'ending-bracket')
+        self._end_nested_param(ending_bracket_section, ']')
 
     def document_shape_type_structure(self, section, shape, history,
                                       include=None, exclude=None, **kwargs):
+        section = section.add_new_section('structure-value')
         self._start_nested_param(section, '{')
 
         input_members = self._add_members_to_shape(shape.members, include)
@@ -79,26 +83,32 @@ class BaseExampleDocumenter(object):
         for i, param in enumerate(input_members):
             if exclude and param in exclude:
                 continue
-            section.write('\'%s\': ' % param)
+            param_section = section.add_new_section(param)
+            param_section.write('\'%s\': ' % param)
             param_shape = input_members[param]
-            traverse_and_document_shape(
-                documenter=self, section=section, shape=param_shape,
-                history=history)
+            self.traverse_and_document_shape(
+                section=param_section, shape=param_shape,
+                history=history, name=param)
             if i < len(input_members) - 1:
-                section.write(',')
-                section.style.new_line()
-
-        self._end_nested_param(section, '}')
+                ending_comma_section = param_section.add_new_section(
+                    'ending-comma')
+                ending_comma_section.write(',')
+                ending_comma_section.style.new_line()
+        ending_bracket_section = section.add_new_section('ending-bracket')
+        self._end_nested_param(ending_bracket_section, '}')
 
     def document_shape_type_map(self, section, shape, history,
                                 include=None, exclude=None, **kwargs):
-        self._start_nested_param(section, '{')
+        map_section = section.add_new_section('map-value')
+        self._start_nested_param(map_section, '{')
         value_shape = shape.value
-        section.write('\'string\': ')
-        traverse_and_document_shape(
-            documenter=self, section=section, shape=value_shape,
-            history=history)
-        self._end_nested_param(section, '}')
+        key_section = map_section.add_new_section('key')
+        key_section.write('\'string\': ')
+        value_section = map_section.add_new_section('value')
+        self.traverse_and_document_shape(
+            section=value_section, shape=value_shape, history=history)
+        end_bracket_section = map_section.add_new_section('ending-bracket')
+        self._end_nested_param(end_bracket_section, '}')
 
     def _add_members_to_shape(self, members, include):
         if include:
@@ -123,10 +133,13 @@ class BaseExampleDocumenter(object):
 
 
 class ResponseExampleDocumenter(BaseExampleDocumenter):
-    pass
+    EVENT_NAME = 'response-example'
 
 
 class RequestExampleDocumenter(BaseExampleDocumenter):
+    EVENT_NAME = 'request-example'
+
+
     def document_shape_type_structure(self, section, shape, history,
                                       include=None, exclude=None, **kwargs):
         param_format = '\'%s\''
@@ -139,20 +152,24 @@ class RequestExampleDocumenter(BaseExampleDocumenter):
             start = '('
             end = ')'
             param_format = '%s'
-
+        section = section.add_new_section('structure-value')
         self._start_nested_param(section, start)
         input_members = self._add_members_to_shape(shape.members, include)
 
         for i, param in enumerate(input_members):
             if exclude and param in exclude:
                 continue
-            section.write(param_format % param)
-            section.write(operator)
+            param_section = section.add_new_section(param)
+            param_section.write(param_format % param)
+            param_section.write(operator)
             param_shape = input_members[param]
-            traverse_and_document_shape(
-                documenter=self, section=section, shape=param_shape,
-                history=history)
+            self.traverse_and_document_shape(
+                section=param_section, shape=param_shape,
+                history=history, name=param)
             if i < len(input_members) - 1:
-                section.write(',')
-                section.style.new_line()
-        self._end_nested_param(section, end)
+                ending_comma_section = param_section.add_new_section(
+                    'ending-comma')
+                ending_comma_section.write(',')
+                ending_comma_section.style.new_line()
+        end_bracket_section = section.add_new_section('ending-bracket')
+        self._end_nested_param(end_bracket_section, end)
diff --git a/botocore/docs/method.py b/botocore/docs/method.py
index 834648d6e..43724abbc 100644
--- a/botocore/docs/method.py
+++ b/botocore/docs/method.py
@@ -109,9 +109,10 @@ def document_custom_signature(section, name, method,
 
 
 def document_model_driven_method(section, method_name, operation_model,
-                                 method_description=None, example_prefix=None,
-                                 include_input=None, include_output=None,
-                                 exclude_input=None, exclude_output=None,
+                                 event_emitter, method_description=None,
+                                 example_prefix=None, include_input=None,
+                                 include_output=None, exclude_input=None,
+                                 exclude_output=None,
                                  document_output=True):
     """Documents an individual method
 
@@ -121,6 +122,8 @@ def document_model_driven_method(section, method_name, operation_model,
 
     :param operation_model: The model of the operation
 
+    :param event_emitter: The event emitter to use to emit events
+
     :param example_prefix: The prefix to use in the method example.
 
     :type include_input: Dictionary where keys are parameter names and
@@ -158,10 +161,13 @@ def document_model_driven_method(section, method_name, operation_model,
     example_section.style.new_paragraph()
     example_section.style.bold('Example')
     if operation_model.input_shape:
-        RequestExampleDocumenter().document_example(
-            example_section, operation_model.input_shape,
-            prefix=example_prefix, include=include_input,
-            exclude=exclude_input)
+        RequestExampleDocumenter(
+            service_name=operation_model.service_model.service_name,
+            operation_name=operation_model.name,
+            event_emitter=event_emitter).document_example(
+                example_section, operation_model.input_shape,
+                prefix=example_prefix, include=include_input,
+                exclude=exclude_input)
     else:
         example_section.style.new_paragraph()
         example_section.style.start_codeblock()
@@ -170,9 +176,12 @@ def document_model_driven_method(section, method_name, operation_model,
     # Add the request parameter documentation.
     request_params_section = section.add_new_section('request-params')
     if operation_model.input_shape:
-        RequestParamsDocumenter().document_params(
-            request_params_section, operation_model.input_shape,
-            include=include_input, exclude=exclude_input)
+        RequestParamsDocumenter(
+            service_name=operation_model.service_model.service_name,
+            operation_name=operation_model.name,
+            event_emitter=event_emitter).document_params(
+                request_params_section, operation_model.input_shape,
+                include=include_input, exclude=exclude_input)
 
     # Add the return value documentation
     return_section = section.add_new_section('return')
@@ -189,10 +198,12 @@ def document_model_driven_method(section, method_name, operation_model,
         return_example_section.style.new_line()
         return_example_section.style.bold('Response Example')
         return_example_section.style.new_paragraph()
-        ResponseExampleDocumenter().document_example(
-            return_example_section, operation_model.output_shape,
-            include=include_output, exclude=exclude_output)
-        return_example_section.style.new_paragraph()
+        ResponseExampleDocumenter(
+            service_name=operation_model.service_model.service_name,
+            operation_name=operation_model.name,
+            event_emitter=event_emitter).document_example(
+                return_example_section, operation_model.output_shape,
+                include=include_output, exclude=exclude_output)
 
         # Add a description for the return value
         return_description_section = return_section.add_new_section(
@@ -200,8 +211,11 @@ def document_model_driven_method(section, method_name, operation_model,
         return_description_section.style.new_line()
         return_description_section.style.bold('Response Structure')
         return_description_section.style.new_paragraph()
-        ResponseParamsDocumenter().document_params(
-            return_description_section, operation_model.output_shape,
-            include=include_output, exclude=exclude_output)
+        ResponseParamsDocumenter(
+            service_name=operation_model.service_model.service_name,
+            operation_name=operation_model.name,
+            event_emitter=event_emitter).document_params(
+                return_description_section, operation_model.output_shape,
+                include=include_output, exclude=exclude_output)
     else:
         return_section.write(':returns: None')
diff --git a/botocore/docs/paginator.py b/botocore/docs/paginator.py
index fa4d86e51..3edf76f45 100644
--- a/botocore/docs/paginator.py
+++ b/botocore/docs/paginator.py
@@ -140,6 +140,7 @@ class PaginatorDocumenter(object):
 
         document_model_driven_method(
             section, 'paginate', operation_model,
+            event_emitter=self._client.meta.events,
             method_description=paginate_description,
             example_prefix='response_iterator = paginator.paginate',
             include_input=botocore_pagination_params,
diff --git a/botocore/docs/params.py b/botocore/docs/params.py
index 105fe273b..32d128111 100644
--- a/botocore/docs/params.py
+++ b/botocore/docs/params.py
@@ -10,11 +10,11 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-from botocore.docs.utils import traverse_and_document_shape
+from botocore.docs.shape import ShapeDocumenter
 from botocore.docs.utils import py_type_name
 
 
-class BaseParamsDocumenter(object):
+class BaseParamsDocumenter(ShapeDocumenter):
     def document_params(self, section, shape, include=None, exclude=None):
         """Fills out the documentation for a section given a model shape.
 
@@ -31,8 +31,8 @@ class BaseParamsDocumenter(object):
             documentation.
         """
         history = []
-        traverse_and_document_shape(
-            documenter=self, section=section, shape=shape, history=history,
+        self.traverse_and_document_shape(
+            section=section, shape=shape, history=history,
             name=None, include=include, exclude=exclude)
 
     def document_recursive_shape(self, section, shape, **kwargs):
@@ -46,25 +46,32 @@ class BaseParamsDocumenter(object):
                                  exclude=None, **kwargs):
         self._add_member_documentation(section, shape, **kwargs)
         param_shape = shape.member
-        self._start_nested_param(section)
-        traverse_and_document_shape(
-            documenter=self, section=section, shape=param_shape,
+        param_section = section.add_new_section(param_shape.name)
+        self._start_nested_param(param_section)
+        self.traverse_and_document_shape(
+            section=param_section, shape=param_shape,
             history=history, name=None)
+        section = section.add_new_section('end-list')
         self._end_nested_param(section)
 
     def document_shape_type_map(self, section, shape, history, include=None,
                                 exclude=None, **kwargs):
         self._add_member_documentation(section, shape, **kwargs)
 
-        self._start_nested_param(section)
-        self._add_member_documentation(section, shape.key)
+        key_section = section.add_new_section('key')
+        self._start_nested_param(key_section)
+        self._add_member_documentation(key_section, shape.key)
 
-        self._start_nested_param(section)
-        traverse_and_document_shape(
-            documenter=self, section=section, shape=shape.value,
+        param_section = section.add_new_section(shape.value.name)
+        param_section.style.indent()
+        self._start_nested_param(param_section)
+        self.traverse_and_document_shape(
+            section=param_section, shape=shape.value,
             history=history, name=None)
-        self._end_nested_param(section)
-        self._end_nested_param(section)
+
+        end_section = section.add_new_section('end-map')
+        self._end_nested_param(end_section)
+        self._end_nested_param(end_section)
 
     def document_shape_type_structure(self, section, shape, history,
                                       include=None, exclude=None,
@@ -74,12 +81,14 @@ class BaseParamsDocumenter(object):
         for param in members:
             if exclude and param in exclude:
                 continue
-            self._start_nested_param(section)
+            param_section = section.add_new_section(param)
+            self._start_nested_param(param_section)
             param_shape = members[param]
-            traverse_and_document_shape(
-                documenter=self, section=section, shape=param_shape,
+            self.traverse_and_document_shape(
+                section=param_section, shape=param_shape,
                 history=history, name=param)
-            self._end_nested_param(section)
+        section = section.add_new_section('end-structure')
+        self._end_nested_param(section)
 
     def _add_member_documentation(self, section, shape, **kwargs):
         pass
@@ -103,64 +112,78 @@ class BaseParamsDocumenter(object):
 class ResponseParamsDocumenter(BaseParamsDocumenter):
     """Generates the description for the response parameters"""
 
+    EVENT_NAME = 'response-params'
+
+
     def _add_member_documentation(self, section, shape, name=None, **kwargs):
         py_type = py_type_name(shape.type_name)
-        section.write('- ')
+        name_section = section.add_new_section('param-name')
+        name_section.write('- ')
         if name is not None:
-            section.style.bold('%s ' % name)
-        section.style.italics('(%s) -- ' % py_type)
+            name_section.style.bold('%s ' % name)
+        name_section.style.italics('(%s) -- ' % py_type)
 
+        documentation_section = section.add_new_section('param-documentation')
         if shape.documentation:
-            section.style.indent()
-            section.include_doc_string(shape.documentation)
-            section.style.dedent()
+            documentation_section.style.indent()
+            documentation_section.include_doc_string(shape.documentation)
         section.style.new_paragraph()
 
 
 class RequestParamsDocumenter(BaseParamsDocumenter):
     """Generates the description for the request parameters"""
 
+    EVENT_NAME = 'request-params'
+
+
     def document_shape_type_structure(self, section, shape, history,
                                       include=None, exclude=None, **kwargs):
         if len(history) > 1:
             self._add_member_documentation(section, shape, **kwargs)
+            section.style.indent()
         members = self._add_members_to_shape(shape.members, include)
         for i, param in enumerate(members):
             if exclude and param in exclude:
                 continue
-            if len(history) > 1:
-                section.style.indent()
-            section.style.new_line()
+            param_section = section.add_new_section(param)
+            param_section.style.new_line()
             param_shape = members[param]
             is_required = param in shape.required_members
-            traverse_and_document_shape(
-                documenter=self, section=section, shape=param_shape,
+            self.traverse_and_document_shape(
+                section=param_section, shape=param_shape,
                 history=history, name=param, is_required=is_required)
-            if len(history) > 1:
-                section.style.dedent()
-            section.style.new_line()
+        section = section.add_new_section('end-structure')
+        if len(history) > 1:
+            section.style.dedent()
+        section.style.new_line()
 
     def _add_member_documentation(self, section, shape, name=None,
                                   is_top_level_param=False, is_required=False,
                                   **kwargs):
         py_type = py_type_name(shape.type_name)
         if is_top_level_param:
-            section.write(':type %s: %s' % (name, py_type))
-            section.style.new_line()
-            section.write(':param %s: ' % name)
+            type_section = section.add_new_section('param-type')
+            type_section.write(':type %s: %s' % (name, py_type))
+            end_type_section = type_section.add_new_section('end-param-type')
+            end_type_section.style.new_line()
+            name_section = section.add_new_section('param-name')
+            name_section.write(':param %s: ' % name)
 
         else:
-            section.write('- ')
+            name_section = section.add_new_section('param-name')
+            name_section.write('- ')
             if name is not None:
-                section.style.bold('%s ' % name)
-            section.style.italics('(%s) -- ' % py_type)
+                name_section.style.bold('%s ' % name)
+            name_section.style.italics('(%s) -- ' % py_type)
 
         if is_required:
-            section.style.indent()
-            section.style.bold('[REQUIRED] ')
-            section.style.dedent()
+            is_required_section = section.add_new_section('is-required')
+            is_required_section.style.indent()
+            is_required_section.style.bold('[REQUIRED] ')
         if shape.documentation:
-            section.style.indent()
-            section.include_doc_string(shape.documentation)
-            section.style.dedent()
-        section.style.new_paragraph()
+            documentation_section = section.add_new_section(
+                'param-documentation')
+            documentation_section.style.indent()
+            documentation_section.include_doc_string(shape.documentation)
+        end_param_section = section.add_new_section('end-param')
+        end_param_section.style.new_paragraph()
diff --git a/botocore/docs/shape.py b/botocore/docs/shape.py
new file mode 100644
index 000000000..f5965f650
--- /dev/null
+++ b/botocore/docs/shape.py
@@ -0,0 +1,78 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+
+# NOTE: This class should not be instantiated and its
+# ``traverse_and_document_shape`` method called directly. It should be
+# inherited from a Documenter class with the appropriate methods
+# and attributes.
+class ShapeDocumenter(object):
+    EVENT_NAME = ''
+
+    def __init__(self, service_name, operation_name, event_emitter):
+        self._service_name = service_name
+        self._operation_name = operation_name
+        self._event_emitter = event_emitter
+
+    def traverse_and_document_shape(self, section, shape, history,
+                                    include=None, exclude=None, name=None,
+                                    is_required=False):
+        """Traverses and documents a shape
+
+        Will take a self class and call its appropriate methods as a shape
+        is traversed.
+
+        :param section: The section to document.
+
+        :param history: A list of the names of the shapes that have been
+            traversed.
+
+        :type include: Dictionary where keys are parameter names and
+            values are the shapes of the parameter names.
+        :param include: The parameter shapes to include in the documentation.
+
+        :type exclude: List of the names of the parameters to exclude.
+        :param exclude: The names of the parameters to exclude from
+            documentation.
+
+        :param name: The name of the shape.
+
+        :param is_required: If the shape is a required member.
+        """
+        param_type = shape.type_name
+        if shape.name in history:
+            self.document_recursive_shape(section, shape, name=name)
+        else:
+            history.append(shape.name)
+            is_top_level_param = (len(history) == 2)
+            getattr(self, 'document_shape_type_%s' % param_type,
+                    self.document_shape_default)(
+                        section, shape, history=history, name=name,
+                        include=include, exclude=exclude,
+                        is_top_level_param=is_top_level_param,
+                        is_required=is_required)
+            if is_top_level_param:
+                self._event_emitter.emit(
+                    'docs.%s.%s.%s.%s' % (self.EVENT_NAME,
+                                          self._service_name,
+                                          self._operation_name,
+                                          name),
+                    section=section)
+            at_overlying_method_section = (len(history) == 1)
+            if at_overlying_method_section:
+                self._event_emitter.emit(
+                    'docs.%s.%s.%s.complete-section' % (self.EVENT_NAME,
+                                                        self._service_name,
+                                                        self._operation_name),
+                    section=section)
+            history.pop()
diff --git a/botocore/docs/utils.py b/botocore/docs/utils.py
index 489adb05e..cb976b6c4 100644
--- a/botocore/docs/utils.py
+++ b/botocore/docs/utils.py
@@ -96,43 +96,31 @@ class DocumentedShape (_DocumentedShape):
             required_members)
 
 
-def traverse_and_document_shape(documenter, section, shape, history,
-                                include=None, exclude=None, name=None,
-                                is_required=False):
-    """Traverses and documents a shape
-
-    Will take a documenter class and call its appropriate methods as a shape
-    is traversed.
-
-    :param documenter: The documenter class to dispatch to as the shape is
-        traversed.
-
-    :param section: The section to document.
-
-    :param history: A list of the names of the shapes that have been traversed.
-
-    :type include: Dictionary where keys are parameter names and
-        values are the shapes of the parameter names.
-    :param include: The parameter shapes to include in the documentation.
-
-    :type exclude: List of the names of the parameters to exclude.
-    :param exclude: The names of the parameters to exclude from
-        documentation.
-
-    :param name: The name of the shape.
-
-    :param is_required: If the shape is a required member.
-    """
-    param_type = shape.type_name
-    if shape.name in history:
-        documenter.document_recursive_shape(section, shape, name=name)
-    else:
-        history.append(shape.name)
-        is_top_level_param = (len(history) == 2)
-        getattr(documenter, 'document_shape_type_%s' % param_type,
-                documenter.document_shape_default)(
-                    section, shape, history=history, name=name,
-                    include=include, exclude=exclude,
-                    is_top_level_param=is_top_level_param,
-                    is_required=is_required)
-        history.pop()
+class AutoPopulatedParam(object):
+    def __init__(self, name, param_description=None):
+        self.name = name
+        self.param_description = param_description
+        if param_description is None:
+            self.param_description = (
+                'Note this parameter is autopopulated. There is no '
+                'need to include in method call.\n')
+
+    def document_auto_populated_param(self, event_name, section, **kwargs):
+        """This documents the auto populated parameters
+
+        It will remove any required marks for the parameter, remove the
+        parameter from the example, and add a snippet about the parameter
+        being autopopulated in the description.
+        """
+        if event_name.startswith('docs.request-params'):
+            if self.name in section.available_sections:
+                section = section.get_section(self.name)
+                if 'is-required' in section.available_sections:
+                    section.delete_section('is-required')
+                description_section = section.get_section(
+                    'param-documentation')
+                description_section.writeln(self.param_description)
+        elif event_name.startswith('docs.request-example'):
+            section = section.get_section('structure-value')
+            if self.name in section.available_sections:
+                section.delete_section(self.name)
diff --git a/botocore/docs/waiter.py b/botocore/docs/waiter.py
index 563fdf4f3..c92242162 100644
--- a/botocore/docs/waiter.py
+++ b/botocore/docs/waiter.py
@@ -67,6 +67,7 @@ class WaiterDocumenter(object):
 
         document_model_driven_method(
             section, 'wait', operation_model,
+            event_emitter=self._client.meta.events,
             method_description=wait_description,
             example_prefix='waiter.wait',
             document_output=False
diff --git a/botocore/handlers.py b/botocore/handlers.py
index 0982affcf..8d62f9dd7 100644
--- a/botocore/handlers.py
+++ b/botocore/handlers.py
@@ -25,6 +25,7 @@ import xml.etree.cElementTree
 import copy
 
 from botocore.compat import unquote, json, quote, six
+from botocore.docs.utils import AutoPopulatedParam
 from botocore.signers import add_generate_presigned_url
 from botocore.signers import add_generate_presigned_post
 
@@ -440,4 +441,8 @@ BUILTIN_HANDLERS = [
      base64_encode_user_data),
     ('before-parameter-build.route53', fix_route53_ids),
     ('before-parameter-build.glacier', inject_account_id),
+    ('docs.*.glacier.*.complete-section',
+     AutoPopulatedParam('accountId').document_auto_populated_param),
+    ('docs.*.glacier.*.complete-section',
+     AutoPopulatedParam('checksum').document_auto_populated_param)
 ]
diff --git a/scripts/ci/install b/scripts/ci/install
new file mode 100755
index 000000000..62846cb6c
--- /dev/null
+++ b/scripts/ci/install
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import os
+import sys
+from subprocess import check_call
+import shutil
+
+_dname = os.path.dirname
+
+REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__))))
+os.chdir(REPO_ROOT)
+
+
+def run(command):
+    return check_call(command, shell=True)
+
+
+try:
+    # Has the form "major.minor"
+    python_version = os.environ['PYTHON_VERSION']
+except KeyError:
+    python_version = '.'.join([str(i) for i in sys.version_info[:2]])
+
+if python_version == '2.6':
+    run('pip install -r requirements26.txt')
+
+run('pip install -r requirements.txt')
+run('pip install coverage')
+if os.path.isdir('dist') and os.listdir('dist'):
+    shutil.rmtree('dist')
+run('python setup.py bdist_wheel')
+wheel_dist = os.listdir('dist')[0]
+run('pip install %s' % (os.path.join('dist', wheel_dist)))


Test patch:
diff --git a/scripts/ci/run-integ-tests b/scripts/ci/run-integ-tests
new file mode 100755
index 000000000..bb829588d
--- /dev/null
+++ b/scripts/ci/run-integ-tests
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# Don't run tests from the root repo dir.
+# We want to ensure we're importing from the installed
+# binary package not from the CWD.
+
+import os
+from subprocess import check_call
+
+_dname = os.path.dirname
+
+REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__))))
+os.chdir(os.path.join(REPO_ROOT, 'tests'))
+
+
+def run(command):
+    return check_call(command, shell=True)
+
+
+run('nosetests --with-xunit --cover-erase --with-coverage '
+    '--cover-package botocore --cover-xml -v integration')
diff --git a/scripts/ci/run-tests b/scripts/ci/run-tests
new file mode 100755
index 000000000..564c79896
--- /dev/null
+++ b/scripts/ci/run-tests
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# Don't run tests from the root repo dir.
+# We want to ensure we're importing from the installed
+# binary package not from the CWD.
+
+import os
+from subprocess import check_call
+
+_dname = os.path.dirname
+
+REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__))))
+os.chdir(os.path.join(REPO_ROOT, 'tests'))
+
+
+def run(command):
+    return check_call(command, shell=True)
+
+
+run('nosetests --with-coverage --cover-erase --cover-package botocore '
+    '--with-xunit --cover-xml -v unit/ functional/')
diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py
new file mode 100644
index 000000000..c89416d7a
--- /dev/null
+++ b/tests/functional/__init__.py
@@ -0,0 +1,12 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
diff --git a/tests/functional/docs/__init__.py b/tests/functional/docs/__init__.py
new file mode 100644
index 000000000..fd84baa56
--- /dev/null
+++ b/tests/functional/docs/__init__.py
@@ -0,0 +1,79 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from tests import unittest
+from botocore.docs.service import ServiceDocumenter
+
+
+class BaseDocsFunctionalTest(unittest.TestCase):
+    def assert_contains_line(self, line, contents):
+        contents = contents.decode('utf-8')
+        self.assertIn(line, contents)
+
+    def assert_contains_lines_in_order(self, lines, contents):
+       contents = contents.decode('utf-8')
+       for line in lines:
+            self.assertIn(line, contents)
+            beginning = contents.find(line)
+            contents = contents[(beginning + len(line)):]
+
+    def assert_not_contains_line(self, line, contents):
+        contents = contents.decode('utf-8')
+        self.assertNotIn(line, contents)
+
+    def assert_not_contains_lines(self, lines, contents):
+        contents = contents.decode('utf-8')
+        for line in lines:
+            self.assertNotIn(line, contents)
+
+    def get_method_document_block(self, operation_name, contents):
+        contents = contents.decode('utf-8')
+        start_method_document = '  .. py:method:: %s(' % operation_name
+        start_index = contents.find(start_method_document)
+        self.assertNotEqual(start_index, -1, 'Method is not found in contents')
+        contents = contents[start_index:]
+        end_index = contents.find(
+            '  ..py:method::', len(start_method_document))
+        contents = contents[:end_index]
+        return contents.encode('utf-8')
+
+    def get_parameter_document_block(self, param_name, contents):
+        contents = contents.decode('utf-8')
+        start_param_document = '    :type %s:' % param_name
+        start_index = contents.find(start_param_document)
+        self.assertNotEqual(start_index, -1, 'Param is not found in contents')
+        contents = contents[start_index:]
+        end_index = contents.find('    :type', len(start_param_document))
+        contents = contents[:end_index]
+        return contents.encode('utf-8')
+
+    def assert_is_documented_as_autopopulated_param(
+            self, service_name, method_name, param_name):
+        contents = ServiceDocumenter(service_name).document_service()
+        # Pick an arbitrary method that uses AccountId.
+        method_contents = self.get_method_document_block(
+            method_name, contents)
+
+        # Ensure it is not in the example.
+        self.assert_not_contains_line('%s=\'string\'' % param_name,
+                                      method_contents)
+
+        # Ensure it is in the params.
+        param_contents = self.get_parameter_document_block(
+            param_name, method_contents)
+
+        # Ensure it is not labeled as required.
+        self.assert_not_contains_line('REQUIRED', param_contents)
+
+        # Ensure the note about autopopulation was added.
+        self.assert_contains_line(
+            'Note this parameter is autopopulated', param_contents) 
diff --git a/tests/functional/docs/test_glacier.py b/tests/functional/docs/test_glacier.py
new file mode 100644
index 000000000..62942a217
--- /dev/null
+++ b/tests/functional/docs/test_glacier.py
@@ -0,0 +1,26 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from tests.functional.docs import BaseDocsFunctionalTest
+
+class TestGlacierDocs(BaseDocsFunctionalTest):
+    def test_account_id(self):
+        self.assert_is_documented_as_autopopulated_param(
+            service_name='glacier',
+            method_name='abort_multipart_upload',
+            param_name='accountId')
+
+    def test_checksum(self):
+        self.assert_is_documented_as_autopopulated_param(
+            service_name='glacier',
+            method_name='upload_archive',
+            param_name='checksum')
diff --git a/tests/unit/docs/test_example.py b/tests/unit/docs/test_example.py
index 2c0242a62..41048936a 100644
--- a/tests/unit/docs/test_example.py
+++ b/tests/unit/docs/test_example.py
@@ -10,7 +10,10 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import mock
+
 from tests.unit.docs import BaseDocsTest
+from botocore.hooks import HierarchicalEmitter
 from botocore.docs.example import ResponseExampleDocumenter
 from botocore.docs.example import RequestExampleDocumenter
 from botocore.docs.utils import DocumentedShape
@@ -19,8 +22,13 @@ from botocore.docs.utils import DocumentedShape
 class BaseExampleDocumenterTest(BaseDocsTest):
     def setUp(self):
         super(BaseExampleDocumenterTest, self).setUp()
-        self.request_example = RequestExampleDocumenter()
-        self.response_example = ResponseExampleDocumenter()
+        self.event_emitter = HierarchicalEmitter()
+        self.request_example = RequestExampleDocumenter(
+            service_name='myservice', operation_name='SampleOperation',
+            event_emitter=self.event_emitter)
+        self.response_example = ResponseExampleDocumenter(
+            service_name='myservice', operation_name='SampleOperation',
+            event_emitter=self.event_emitter)
 
 
 class TestDocumentDefaultValue(BaseExampleDocumenterTest):
@@ -52,6 +60,47 @@ class TestDocumentDefaultValue(BaseExampleDocumenterTest):
         ])
 
 
+class TestTraverseAndDocumentShape(BaseExampleDocumenterTest):
+    def setUp(self):
+        super(TestTraverseAndDocumentShape, self).setUp()
+        self.add_shape_to_params('Foo', 'String', 'This describes foo.')
+        self.event_emitter = mock.Mock()
+        self.request_example = RequestExampleDocumenter(
+            service_name='myservice', operation_name='SampleOperation',
+            event_emitter=self.event_emitter)
+        self.response_example = ResponseExampleDocumenter(
+            service_name='myservice', operation_name='SampleOperation',
+            event_emitter=self.event_emitter)
+
+    def test_events_emitted_response_example(self):
+        self.response_example.traverse_and_document_shape(
+            section=self.doc_structure,
+            shape=self.operation_model.input_shape, history=[]
+        )
+        structure_section = self.doc_structure.get_section('structure-value')
+        self.assertEqual(
+            self.event_emitter.emit.call_args_list,
+            [mock.call('docs.response-example.myservice.SampleOperation.Foo',
+                       section=structure_section.get_section('Foo')),
+             mock.call(('docs.response-example.myservice.SampleOperation'
+                        '.complete-section'), section=self.doc_structure)]
+        )
+
+    def test_events_emitted_request_example(self):
+        self.request_example.traverse_and_document_shape(
+            section=self.doc_structure,
+            shape=self.operation_model.input_shape, history=[]
+        )
+        structure_section = self.doc_structure.get_section('structure-value')
+        self.assertEqual(
+            self.event_emitter.emit.call_args_list,
+            [mock.call('docs.request-example.myservice.SampleOperation.Foo',
+                       section=structure_section.get_section('Foo')),
+             mock.call(('docs.request-example.myservice.SampleOperation'
+                        '.complete-section'), section=self.doc_structure)]
+        )
+
+
 class TestDocumentEnumValue(BaseExampleDocumenterTest):
     def setUp(self):
         super(TestDocumentEnumValue, self).setUp()
diff --git a/tests/unit/docs/test_method.py b/tests/unit/docs/test_method.py
index e65b7efa8..0d7e10e90 100644
--- a/tests/unit/docs/test_method.py
+++ b/tests/unit/docs/test_method.py
@@ -12,6 +12,7 @@
 # language governing permissions and limitations under the License.
 from tests import unittest
 from tests.unit.docs import BaseDocsTest
+from botocore.hooks import HierarchicalEmitter
 from botocore.docs.method import document_model_driven_signature
 from botocore.docs.method import document_custom_signature
 from botocore.docs.method import document_model_driven_method
@@ -83,11 +84,13 @@ class TestDocumentCustomSignature(BaseDocsTest):
 class TestDocumentModelDrivenMethod(BaseDocsTest):
     def setUp(self):
         super(TestDocumentModelDrivenMethod, self).setUp()
+        self.event_emitter = HierarchicalEmitter()
         self.add_shape_to_params('Bar', 'String')
 
     def test_default(self):
         document_model_driven_method(
             self.doc_structure, 'foo', self.operation_model,
+            event_emitter=self.event_emitter,
             method_description='This describes the foo method.',
             example_prefix='response = client.foo'
         )
@@ -118,6 +121,7 @@ class TestDocumentModelDrivenMethod(BaseDocsTest):
         del self.json_model['operations']['SampleOperation']['output']
         document_model_driven_method(
             self.doc_structure, 'foo', self.operation_model,
+            event_emitter=self.event_emitter,
             method_description='This describes the foo method.',
             example_prefix='response = client.foo'
         )
@@ -137,6 +141,7 @@ class TestDocumentModelDrivenMethod(BaseDocsTest):
         ]
         document_model_driven_method(
             self.doc_structure, 'foo', self.operation_model,
+            event_emitter=self.event_emitter,
             method_description='This describes the foo method.',
             example_prefix='response = client.foo',
             include_input=include_params
@@ -173,6 +178,7 @@ class TestDocumentModelDrivenMethod(BaseDocsTest):
         ]
         document_model_driven_method(
             self.doc_structure, 'foo', self.operation_model,
+            event_emitter=self.event_emitter,
             method_description='This describes the foo method.',
             example_prefix='response = client.foo',
             include_output=include_params
@@ -205,6 +211,7 @@ class TestDocumentModelDrivenMethod(BaseDocsTest):
         self.add_shape_to_params('Biz', 'String')
         document_model_driven_method(
             self.doc_structure, 'foo', self.operation_model,
+            event_emitter=self.event_emitter,
             method_description='This describes the foo method.',
             example_prefix='response = client.foo',
             exclude_input=['Bar']
@@ -241,6 +248,7 @@ class TestDocumentModelDrivenMethod(BaseDocsTest):
         self.add_shape_to_params('Biz', 'String')
         document_model_driven_method(
             self.doc_structure, 'foo', self.operation_model,
+            event_emitter=self.event_emitter,
             method_description='This describes the foo method.',
             example_prefix='response = client.foo',
             exclude_output=['Bar']
diff --git a/tests/unit/docs/test_params.py b/tests/unit/docs/test_params.py
index 76e5fc598..8d49f2824 100644
--- a/tests/unit/docs/test_params.py
+++ b/tests/unit/docs/test_params.py
@@ -10,7 +10,10 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import mock
+
 from tests.unit.docs import BaseDocsTest
+from botocore.hooks import HierarchicalEmitter
 from botocore.docs.params import RequestParamsDocumenter
 from botocore.docs.params import ResponseParamsDocumenter
 from botocore.docs.utils import DocumentedShape
@@ -19,8 +22,13 @@ from botocore.docs.utils import DocumentedShape
 class BaseParamsDocumenterTest(BaseDocsTest):
     def setUp(self):
         super(BaseParamsDocumenterTest, self).setUp()
-        self.request_params = RequestParamsDocumenter()
-        self.response_params = ResponseParamsDocumenter()
+        self.event_emitter = HierarchicalEmitter()
+        self.request_params = RequestParamsDocumenter(
+            service_name='myservice', operation_name='SampleOperation',
+            event_emitter=self.event_emitter)
+        self.response_params = ResponseParamsDocumenter(
+            service_name='myservice', operation_name='SampleOperation',
+            event_emitter=self.event_emitter)
 
 
 class TestDocumentDefaultValue(BaseParamsDocumenterTest):
@@ -45,6 +53,45 @@ class TestDocumentDefaultValue(BaseParamsDocumenterTest):
         ])
 
 
+class TestTraverseAndDocumentShape(BaseParamsDocumenterTest):
+    def setUp(self):
+        super(TestTraverseAndDocumentShape, self).setUp()
+        self.add_shape_to_params('Foo', 'String', 'This describes foo.')
+        self.event_emitter = mock.Mock()
+        self.request_params = RequestParamsDocumenter(
+            service_name='myservice', operation_name='SampleOperation',
+            event_emitter=self.event_emitter)
+        self.response_params = ResponseParamsDocumenter(
+            service_name='myservice', operation_name='SampleOperation',
+            event_emitter=self.event_emitter)
+
+    def test_events_emitted_response_params(self):
+        self.response_params.traverse_and_document_shape(
+            section=self.doc_structure,
+            shape=self.operation_model.input_shape, history=[]
+        )
+        self.assertEqual(
+            self.event_emitter.emit.call_args_list,
+            [mock.call('docs.response-params.myservice.SampleOperation.Foo',
+                       section=self.doc_structure.get_section('Foo')),
+             mock.call(('docs.response-params.myservice.SampleOperation'
+                        '.complete-section'), section=self.doc_structure)]
+        )
+
+    def test_events_emitted_request_params(self):
+        self.request_params.traverse_and_document_shape(
+            section=self.doc_structure,
+            shape=self.operation_model.input_shape, history=[]
+        )
+        self.assertEqual(
+            self.event_emitter.emit.call_args_list,
+            [mock.call('docs.request-params.myservice.SampleOperation.Foo',
+                       section=self.doc_structure.get_section('Foo')),
+             mock.call(('docs.request-params.myservice.SampleOperation'
+                        '.complete-section'), section=self.doc_structure)]
+        )
+
+
 class TestDocumentMultipleDefaultValues(BaseParamsDocumenterTest):
     def setUp(self):
         super(TestDocumentMultipleDefaultValues, self).setUp()
diff --git a/tests/unit/docs/test_utils.py b/tests/unit/docs/test_utils.py
index 540341fb2..294dcb6d1 100644
--- a/tests/unit/docs/test_utils.py
+++ b/tests/unit/docs/test_utils.py
@@ -10,11 +10,14 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import mock
+
 from tests import unittest
 from tests.unit.docs import BaseDocsTest
 from botocore.docs.utils import py_type_name
 from botocore.docs.utils import py_default
 from botocore.docs.utils import get_official_service_name
+from botocore.docs.utils import AutoPopulatedParam
 
 
 class TestPythonTypeName(unittest.TestCase):
@@ -107,3 +110,67 @@ class TestGetOfficialServiceName(BaseDocsTest):
         self.service_model.metadata['serviceAbbreviation'] = 'Amazon Foo'
         self.assertEqual('The Foo Service',
                          get_official_service_name(self.service_model))
+
+
+class TestAutopopulatedParam(BaseDocsTest):
+    def setUp(self):
+        super(TestAutopopulatedParam, self).setUp()
+        self.name = 'MyMember'
+        self.param = AutoPopulatedParam(self.name)
+
+    def test_request_param_not_required(self):
+        section = self.doc_structure.add_new_section(self.name)
+        section.add_new_section('param-documentation')
+        self.param.document_auto_populated_param(
+            'docs.request-params', self.doc_structure)
+        self.assert_contains_line(
+            ('Note this parameter is autopopulated. There is no need '
+             'to include in method call'))
+
+    def test_request_param_required(self):
+        section = self.doc_structure.add_new_section(self.name)
+        is_required_section = section.add_new_section('is-required')
+        section.add_new_section('param-documentation')
+        is_required_section.write('**[REQUIRED]**')
+        self.param.document_auto_populated_param(
+            'docs.request-params', self.doc_structure)
+        self.assert_not_contains_line('**[REQUIRED]**')
+        self.assert_contains_line(
+            ('Note this parameter is autopopulated. There is no need '
+             'to include in method call'))
+
+    def test_non_default_param_description(self):
+        description = 'This is a custom description'
+        self.param = AutoPopulatedParam(self.name, description)
+        section = self.doc_structure.add_new_section(self.name)
+        section.add_new_section('param-documentation')
+        self.param.document_auto_populated_param(
+            'docs.request-params', self.doc_structure)
+        self.assert_contains_line(description)
+
+    def test_request_example(self):
+        top_section = self.doc_structure.add_new_section('structure-value')
+        section = top_section.add_new_section(self.name)
+        example = 'MyMember: \'string\''
+        section.write(example)
+        self.assert_contains_line(example)
+        self.param.document_auto_populated_param(
+            'docs.request-example', self.doc_structure)
+        self.assert_not_contains_line(example)
+
+    def test_param_not_in_section_request_param(self):
+        self.doc_structure.add_new_section('Foo')
+        self.param.document_auto_populated_param(
+            'docs.request-params', self.doc_structure)
+        self.assertEqual(
+            '', self.doc_structure.flush_structure().decode('utf-8'))
+
+    def test_param_not_in_section_request_example(self):
+        top_section = self.doc_structure.add_new_section('structure-value')
+        section = top_section.add_new_section('Foo')
+        example = 'Foo: \'string\''
+        section.write(example)
+        self.assert_contains_line(example)
+        self.param.document_auto_populated_param(
+            'docs.request-example', self.doc_structure)
+        self.assert_contains_line(example)