Repository: boto/botocore
Pull request #633 (2015-08-20 21:21:00)

Dynamically add docstrings to client methods
So now if you are in the python repl you can access a client's method docstrings with ``__doc__`` and ``help()``. These docstrings are lazily loaded so they do not actually get generated until they are accessed, as it is really slow to generate all of docstrings for a client upfront. So they should not affect the speed to generate clients or make method calls. I did some performance testing to make sure. Also note that these automatically get picked up by boto3 clients which is nice. Here is the script I used:

```py
import boto3
import time

for i in range(5):
    before = time.time()
    boto3.client('ec2')
    after = time.time()
    delta = after - before
    print('For ec2 client %s, it took %s seconds to create' % (i, delta))

print('')
client = boto3.client('ec2')
for i in range(5):
    before = time.time()
    response = client.describe_regions()
    after = time.time()
    delta = after - before
    print('For ec2 client %s, it took %s seconds to DescribeRegions' % (
        i, delta))
```
Here is the current speed on the develop branch:
```
For ec2 client 0, it took 0.10999083519 seconds to create
For ec2 client 1, it took 0.00277781486511 seconds to create
For ec2 client 2, it took 0.00267887115479 seconds to create
For ec2 client 3, it took 0.00264096260071 seconds to create
For ec2 client 4, it took 0.00269412994385 seconds to create

For ec2 client 0, it took 0.341856002808 seconds to DescribeRegions
For ec2 client 1, it took 0.115956783295 seconds to DescribeRegions
For ec2 client 2, it took 0.106163024902 seconds to DescribeRegions
For ec2 client 3, it took 0.0971441268921 seconds to DescribeRegions
For ec2 client 4, it took 0.094202041626 seconds to DescribeRegions
```

For this pull request, here is the output:
```
For ec2 client 0, it took 0.110929012299 seconds to create
For ec2 client 1, it took 0.00621104240417 seconds to create
For ec2 client 2, it took 0.00612497329712 seconds to create
For ec2 client 3, it took 0.00794386863708 seconds to create
For ec2 client 4, it took 0.00695705413818 seconds to create

For ec2 client 0, it took 0.333263874054 seconds to DescribeRegions
For ec2 client 1, it took 0.112825870514 seconds to DescribeRegions
For ec2 client 2, it took 0.113487005234 seconds to DescribeRegions
For ec2 client 3, it took 0.0971641540527 seconds to DescribeRegions
For ec2 client 4, it took 0.0956180095673 seconds to DescribeRegions
```

cc @jamesls @mtdowling @rayluo 


Patch:
diff --git a/botocore/client.py b/botocore/client.py
index b650c5940..2268b4255 100644
--- a/botocore/client.py
+++ b/botocore/client.py
@@ -26,6 +26,7 @@ from botocore.model import ServiceModel
 from botocore.paginate import Paginator
 from botocore.signers import RequestSigner
 from botocore.utils import CachedProperty
+from botocore.docs.method import LazyLoadedDocstring
 
 
 logger = logging.getLogger(__name__)
@@ -269,7 +270,18 @@ class ClientCreator(object):
             return self._make_api_call(operation_name, kwargs)
 
         _api_call.__name__ = str(py_operation_name)
-        # TODO: docstrings.
+
+        # Add the docstring to the client method
+        operation_model = service_model.operation_model(operation_name)
+        docstring = LazyLoadedDocstring(
+            operation_model=operation_model,
+            method_name=operation_name,
+            event_emitter=self._event_emitter,
+            method_description=operation_model.documentation,
+            example_prefix='response = client.%s' % py_operation_name,
+            include_signature=False
+        )
+        _api_call.__doc__ = docstring
         return _api_call
 
 
diff --git a/botocore/docs/method.py b/botocore/docs/method.py
index 8abb2b3a9..1bb2d513a 100644
--- a/botocore/docs/method.py
+++ b/botocore/docs/method.py
@@ -16,6 +16,7 @@ from botocore.docs.params import RequestParamsDocumenter
 from botocore.docs.params import ResponseParamsDocumenter
 from botocore.docs.example import ResponseExampleDocumenter
 from botocore.docs.example import RequestExampleDocumenter
+from botocore.docs.bcdoc.restdoc import DocumentStructure
 
 
 def get_instance_public_methods(instance):
@@ -124,8 +125,8 @@ def document_model_driven_method(section, method_name, operation_model,
                                  event_emitter, method_description=None,
                                  example_prefix=None, include_input=None,
                                  include_output=None, exclude_input=None,
-                                 exclude_output=None,
-                                 document_output=True):
+                                 exclude_output=None, document_output=True,
+                                 include_signature=True):
     """Documents an individual method
 
     :param section: The section to write to
@@ -158,11 +159,15 @@ def document_model_driven_method(section, method_name, operation_model,
 
     :param document_output: A boolean flag to indicate whether to
         document the output.
+
+    :param include_signature: Whether or not to include the signature.
+        It is useful for generating docstrings.
     """
-    # Add the signature.
-    document_model_driven_signature(
-        section, method_name, operation_model, include=include_input,
-        exclude=exclude_input)
+    # Add the signature if specified.
+    if include_signature:
+        document_model_driven_signature(
+            section, method_name, operation_model, include=include_input,
+            exclude=exclude_input)
 
     # Add the description for the method.
     method_intro_section = section.add_new_section('method-intro')
@@ -231,3 +236,61 @@ def document_model_driven_method(section, method_name, operation_model,
                 include=include_output, exclude=exclude_output)
     else:
         return_section.write(':returns: None')
+
+
+class LazyLoadedDocstring(str):
+    """Used for lazily loading docstrings
+
+    You can instantiate this class and assign it to a __doc__ value.
+    The docstring will not be generated till accessed via __doc__ or
+    help().
+    """
+    def __init__(self, *args, **kwargs):
+        """
+        The args and kwargs are the same as the underlying document
+        generation function. These just get proxied to the underlying
+        function.
+        """
+        super(LazyLoadedDocstring, self).__init__()
+        self._gen_args = args
+        self._gen_kwargs = kwargs
+        self._docstring = None
+
+    def __new__(cls, *args, **kwargs):
+        # Needed in order to sub class from str with args and kwargs
+        return super(LazyLoadedDocstring, cls).__new__(cls)
+
+    def expandtabs(self, tabsize=8):
+        """Expands tabs to spaces
+
+        So this is a big hack in order to get lazy loaded docstring work
+        for the ``help()``. In the ``help()`` function, ``pydoc`` and
+        ``inspect`` are used. At some point the ``inspect.cleandoc``
+        method is called. To clean the docs ``expandtabs`` is called
+        and that is where we override the method to generate and return the
+        docstrings.
+        """
+        if self._docstring is None:
+            self._generate().expandtabs(tabsize)
+        return self._docstring.expandtabs(tabsize)
+
+    def __str__(self):
+        return self._generate()
+
+    # __doc__ of target will use either __repr__ or __str__ of this class.
+    __repr__ = __str__
+
+    def _generate(self):
+        # Generate the docstring if it is not already cached.
+        if self._docstring is None:
+            self._docstring = self._create_docstring()
+        return self._docstring
+
+    def _create_docstring(self):
+        docstring_structure = DocumentStructure('docstring')
+        # Call the document method function with the args and kwargs
+        # passed to the class.
+        document_model_driven_method(
+            docstring_structure, *self._gen_args,
+            **self._gen_kwargs)
+        return docstring_structure.flush_structure().decode('utf-8')


Test patch:
diff --git a/tests/unit/docs/test_method.py b/tests/unit/docs/test_method.py
index db9b8df66..aa7221b5e 100644
--- a/tests/unit/docs/test_method.py
+++ b/tests/unit/docs/test_method.py
@@ -10,7 +10,7 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-from tests import unittest
+from tests import unittest, mock
 from tests.unit.docs import BaseDocsTest
 from botocore.hooks import HierarchicalEmitter
 from botocore.docs.method import document_model_driven_signature
@@ -18,6 +18,7 @@ from botocore.docs.method import document_custom_signature
 from botocore.docs.method import document_custom_method
 from botocore.docs.method import document_model_driven_method
 from botocore.docs.method import get_instance_public_methods
+from botocore.docs.method import LazyLoadedDocstring
 from botocore.docs.utils import DocumentedShape
 
 
@@ -301,3 +302,44 @@ class TestDocumentModelDrivenMethod(BaseDocsTest):
             '\'Bar\': \'string\'',
             '- **Bar** *(string) --*',
         ])
+
+
+class TestLazyLoadedDocstring(unittest.TestCase):
+    def setUp(self):
+        self.document_method_patch = mock.patch(
+            'botocore.docs.method.document_model_driven_method')
+        self.document_method = self.document_method_patch.start()
+
+    def tearDown(self):
+        self.document_method_patch.stop()
+
+    def test_expandtabs(self):
+        self.document_method.side_effect = lambda section: section.write(
+            'foo\t')
+        docstring = LazyLoadedDocstring()
+        self.assertEqual('foo ', docstring.expandtabs(1))
+
+    def test_str(self):
+        self.document_method.side_effect = lambda section: section.write('foo')
+        docstring = LazyLoadedDocstring()
+        self.assertEqual('foo', str(docstring))
+
+    def test_repr(self):
+        self.document_method.side_effect = lambda section: section.write('foo')
+        docstring = LazyLoadedDocstring()
+        self.assertEqual('foo', repr(docstring))
+
+    def test_is_lazy_loaded(self):
+        docstring = LazyLoadedDocstring()
+        str(docstring)
+        str(docstring)
+        # The mock.ANY represents the DocumentStructure that is filled out.
+        self.document_method.assert_called_once_with(mock.ANY)
+
+    def test_args_kwargs_passed(self):
+        args = ['foo', 'bar']
+        kwargs = {'biz': 'baz'}
+        docstring = LazyLoadedDocstring(*args, **kwargs)
+        str(docstring)
+        # The mock.ANY represents the DocumentStructure that is filled out.
+        self.document_method.assert_called_with(mock.ANY, *args, **kwargs)
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
index a4a8c575d..56b7e0916 100644
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -20,6 +20,7 @@ from botocore import hooks
 from botocore.credentials import Credentials
 from botocore.exceptions import ParamValidationError
 from botocore import exceptions
+from botocore.compat import six
 
 
 class TestAutoGeneratedClient(unittest.TestCase):
@@ -40,6 +41,7 @@ class TestAutoGeneratedClient(unittest.TestCase):
                         'requestUri': '/',
                     },
                     'input': {'shape': 'TestOperationRequest'},
+                    'documentation': 'Documents TestOperation'
                 }
             },
             'shapes': {
@@ -47,8 +49,10 @@ class TestAutoGeneratedClient(unittest.TestCase):
                     'type': 'structure',
                     'required': ['Foo'],
                     'members': {
-                        'Foo': {'shape': 'StringType'},
-                        'Bar': {'shape': 'StringType'},
+                        'Foo': {'shape': 'StringType',
+                                'documentation': 'Documents Foo'},
+                        'Bar': {'shape': 'StringType',
+                                'documentation': 'Documents Bar'},
                     }
                 },
                 'StringType': {'type': 'string'}
@@ -155,6 +159,48 @@ class TestAutoGeneratedClient(unittest.TestCase):
             'myservice', 'us-west-2', credentials=self.credentials)
         self.assertTrue(hasattr(service_client, 'test_operation'))
 
+    def test_client_method_docstring(self):
+        creator = self.create_client_creator()
+        service_client = creator.create_client(
+            'myservice', 'us-west-2', credentials=self.credentials)
+        method_docstring = str(service_client.test_operation.__doc__)
+        ref_docstring_lines = [
+            'Documents TestOperation',
+            '**Request Syntax**',
+            '  response = client.test_operation(',
+            '      Bar=\'string\'',
+            '      Foo=\'string\'',
+            '  )',
+            ':type Bar: string',
+            ':param Bar: Documents Bar',
+            ':type Foo: string',
+            ':param Foo: **[REQUIRED]** Documents Foo'
+        ]
+        for line in ref_docstring_lines:
+            self.assertIn(line, method_docstring)
+
+    def test_client_method_help(self):
+        creator = self.create_client_creator()
+        service_client = creator.create_client(
+            'myservice', 'us-west-2', credentials=self.credentials)
+        with mock.patch('sys.stdout', six.StringIO()) as mock_stdout:
+            help(service_client.test_operation)
+        method_docstring = mock_stdout.getvalue()
+        ref_docstring_lines = [
+            'Documents TestOperation',
+            '**Request Syntax**',
+            '  response = client.test_operation(',
+            '      Bar=\'string\'',
+            '      Foo=\'string\'',
+            '  )',
+            ':type Bar: string',
+            ':param Bar: Documents Bar',
+            ':type Foo: string',
+            ':param Foo: **[REQUIRED]** Documents Foo'
+        ]
+        for line in ref_docstring_lines:
+            self.assertIn(line, method_docstring)
+
     def test_client_create_unicode(self):
         creator = self.create_client_creator()
         service_client = creator.create_client(