Repository: googleapis/google-cloud-python

Issue:
Python documentation missing examples, etc
It seems to have disappeared:

https://googlecloudplatform.github.io/gcloud-python/#/docs/master/gcloud/datastore/client/client



Discussion:
@danielecook How do you mean disappeared? They are different from the previous Sphinx docs, but the Sphinx docs are still up at http://gcloud-python.readthedocs.io/en/latest/

Maybe @daspecster can comment about the differences?

Look at the content on that link I provided. There are no examples. There are no descriptions. This was not the case previously. Compare with the ruby language page. When I google 'gcloud-python' the first result is the link above - not the sphinx docs you provided. Why does documentation exist in multiple places? Thanks!

@daspecster : I thought that we fixed this so the examples were still there... If not, can we... fix it?

Hey! So we're not pulling in the RST files and we have some examples in those.

I will have to check it out, but either we need to parse those usage RST files and find a place to put them in the doc site or we should move those over into the appropriate module doc strings. I think the latter would be the quickest.

The tricky part is going to be figuring out where to put things in the new doc site since I don't think there is really a good spot for those?

Actually I take that back, 
The corresponding example mentioned above would be here http://gcloud-python.readthedocs.io/en/latest/datastore-client.html which has some example code in it.  So this could really be a bug, I'll check it out. If the examples are literalincludes then yeah I don't have that built into the generator.

Thanks!

OK -- so long as we get snippets back. That is an absolute requirement here....


Resolving patch:
diff --git a/gcloud/datastore/client.py b/gcloud/datastore/client.py
--- a/gcloud/datastore/client.py
+++ b/gcloud/datastore/client.py
@@ -443,6 +443,37 @@ def query(self, **kwargs):
         """Proxy to :class:`gcloud.datastore.query.Query`.
 
         Passes our ``project``.
+
+        Using query to search a datastore::
+
+        >>> from gcloud import datastore
+        >>> client = datastore.Client()
+        >>> query = client.query(kind='MyKind')
+        >>> query.add_filter('property', '=', 'val')
+
+        Using the query iterator's
+        :meth:`next_page() <gcloud.datastore.query.Iterator.next_page>` method:
+
+        >>> query_iter = query.fetch()
+        >>> entities, more_results, cursor = query_iter.next_page()
+        >>> entities
+        [<list of Entity unmarshalled from protobuf>]
+        >>> more_results
+        <boolean of more results>
+        >>> cursor
+        <string containing cursor where fetch stopped>
+
+        Under the hood this is doing:
+
+        >>> connection.run_query('project', query.to_protobuf())
+        [<list of Entity Protobufs>], cursor, more_results, skipped_results
+
+        :type **kwargs: dict
+        :param **kwargs: Parameters for initializing and instance of
+                         :class:`gcloud.datastore.query.Query`.
+
+        :rtype: :class:`gcloud.datastore.query.Query`
+        :returns: An instance of :class:`gcloud.datastore.query.Query`
         """
         if 'client' in kwargs:
             raise TypeError('Cannot pass client')
diff --git a/gcloud/datastore/connection.py b/gcloud/datastore/connection.py
--- a/gcloud/datastore/connection.py
+++ b/gcloud/datastore/connection.py
@@ -227,29 +227,7 @@ def run_query(self, project, query_pb, namespace=None,
         :meth:`gcloud.datastore.query.Query.fetch` method.
 
         Under the hood, the :class:`gcloud.datastore.query.Query` class
-        uses this method to fetch data:
-
-        >>> from gcloud import datastore
-        >>> client = datastore.Client()
-        >>> query = client.query(kind='MyKind')
-        >>> query.add_filter('property', '=', 'val')
-
-        Using the query iterator's
-        :meth:`next_page() <.datastore.query.Iterator.next_page>` method:
-
-        >>> query_iter = query.fetch()
-        >>> entities, more_results, cursor = query_iter.next_page()
-        >>> entities
-        [<list of Entity unmarshalled from protobuf>]
-        >>> more_results
-        <boolean of more results>
-        >>> cursor
-        <string containing cursor where fetch stopped>
-
-        Under the hood this is doing:
-
-        >>> connection.run_query('project', query.to_protobuf())
-        [<list of Entity Protobufs>], cursor, more_results, skipped_results
+        uses this method to fetch data.
 
         :type project: string
         :param project: The project over which to run the query.
diff --git a/scripts/generate_json_docs.py b/scripts/generate_json_docs.py
--- a/scripts/generate_json_docs.py
+++ b/scripts/generate_json_docs.py
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import argparse
+import cgi
+import doctest
 import inspect
 import json
 import os
@@ -27,6 +29,9 @@
 from verify_included_modules import get_public_modules
 
 
+_DOCSTRING_TEST_PARSER = doctest.DocTestParser()
+
+
 class Module(object):
 
     def __init__(self, module_id, name, description=None,
@@ -148,26 +153,31 @@ def from_pdoc(cls, element):
         components = element.refname.split('.')
 
         mod = __import__(components[0])
-
         for comp in components[1:]:
             mod = getattr(mod, comp)
 
-        build_source(mod, method)
+        # Get method line number.
+        method.add_source_line(get_source_line_number(mod))
+
+        # Get method Examples.
+        examples = get_examples_from_docstring(element.docstring)
+        if examples:
+            method.add_example(examples)
 
         if element.docstring:
             if not isinstance(element, pdoc.Class) and element.cls:
-                cls = element.cls.cls
+                klass = element.cls.cls
             elif element.cls:
-                cls = element.cls
+                klass = element.cls
             else:
-                cls = None
+                klass = None
 
             # Hack for old-style classes
-            if str(cls)[0] != '<':
-                cls = '<class \'' + str(cls) + '\'>'
+            if not str(klass).startswith('<'):
+                klass = '<class \'%s\'>' % (klass,)
 
             try:
-                method_info = parse_docstring(element.docstring, cls)
+                method_info = parse_docstring(element.docstring, klass)
             except (MethodParsingException, IndexError):
                 return method
 
@@ -175,9 +185,6 @@ def from_pdoc(cls, element):
                 param = Param.from_docstring_section(name, data)
                 method.add_param(param)
 
-            if method_info.get('example'):
-                method.add_example(method_info['example'])
-
             if method_info.get('return'):
                 if len(method_info['return']['type_name']) > 0:
                     type_name = method_info.get('return').get('type_name')
@@ -296,9 +303,35 @@ def clean_source_path(module):
     return '%s.py' % (source_id.replace('.', '/'),)
 
 
+def get_examples_from_docstring(doc_str):
+    """Parse doctest style code examples from a docstring."""
+    examples = _DOCSTRING_TEST_PARSER.get_examples(doc_str)
+    example_str = ''
+    for example in examples:
+        example_str += '%s' % (example.source,)
+        example_str += '%s' % (example.want,)
+
+    return cgi.escape(example_str)
+
+
+def get_source_line_number(module):
+    if isinstance(module, (types.ModuleType, types.ClassType,
+                           types.MethodType, types.FunctionType,
+                           types.TracebackType, types.FrameType,
+                           types.CodeType, types.TypeType)):
+
+        _, line = inspect.getsourcelines(module)
+        source_path = clean_source_path(module)
+
+        if line:
+            source_path = source_path + '#L' + str(line)
+        return source_path
+
+
 def process_code_blocks(doc):
     blocks = []
     index = 0
+
     for line in doc.splitlines(True):
         if len(blocks) - 1 < index:
             blocks.append('')


Test patch: