Repository: sphinx-gallery/sphinx-gallery

Issue:
ENH: Allow setting example order
I recently migrated PySurfer over to SG, and want to be able to set the example file order. There currently is no way to do that, correct? The workarounds would be to either build subdirectories, or re-title the examples e.g. "01. Basic visualization" etc. to get them in the correct order, which is not so satisfactory.

I propose adding an `example_order` dict to `conf.py` to complement the existing `subsection_order`. The design would be nested dicts:
```
'example_order': {'examples': ['plot_basic.py', 'plot_advanced.py']}
```
So the first-level dict would be key-value pair for `subsection: order`. If a subdirectory isn't listed, use default (title-alphabetical) sorting. If one is listed, it must list all SG-relevant files in the directory or an error is thrown. This seems the most conservative API-wise and should be fully backward compatible.

Comments:

Patch:
diff --git a/doc/advanced_configuration.rst b/doc/advanced_configuration.rst
index 963d95f..d8a6c5d 100644
--- a/doc/advanced_configuration.rst
+++ b/doc/advanced_configuration.rst
@@ -18,6 +18,7 @@ file:
 - ``examples_dirs`` and ``gallery_dirs`` (:ref:`multiple_galleries_config`)
 - ``filename_pattern`` (:ref:`build_pattern`)
 - ``subsection_order`` (:ref:`sub_gallery_order`)
+- ``within_subsection_order`` (:ref:`within_gallery_order`)
 - ``reference_url`` (:ref:`link_to_documentation`)
 - ``backreferences_dir`` and ``doc_module`` (:ref:`references_to_examples`)
 - ``default_thumb_file`` (:ref:`custom_default_thumb`)
@@ -149,6 +150,33 @@ If you so desire you can implement your own sorting key. It will be
 provided the relative paths to `conf.py` of each sub gallery folder.
 
 
+.. _within_gallery_order:
+
+Sorting gallery examples
+========================
+
+Within a given gallery (sub)section, the example files are ordered by
+using the standard :func:`sorted` function with the ``key`` argument by default
+set to
+:class:`NumberOfCodeLinesSortKey(src_dir) <sphinx_gallery.sorting.NumberOfCodeLinesSortKey>`,
+which sorts the files based on the number of code lines::
+
+    from sphinx_gallery.sorting import NumberOfCodeLinesSortKey
+    sphinx_gallery_conf = {
+        ...
+        'within_subsection_order': NumberOfCodeLinesSortKey,
+    }
+
+In addition, multiple convenience classes are provided for use with
+``within_subsection_order``:
+
+- :class:`sphinx_gallery.sorting.NumberOfCodeLinesSortKey` (default) to sort by
+  the number of code lines.
+- :class:`sphinx_gallery.sorting.FileSizeSortKey` to sort by file size.
+- :class:`sphinx_gallery.sorting.FileNameSortKey` to sort by file name.
+- :class:`sphinx_gallery.sorting.ExampleTitleSortKey` to sort by example title.
+
+
 .. _link_to_documentation:
 
 Linking to documentation
diff --git a/doc/conf.py b/doc/conf.py
index eaa5dff..63cae10 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -306,7 +306,7 @@ intersphinx_mapping = {
     'mayavi': ('http://docs.enthought.com/mayavi/mayavi', None),
 }
 
-from sphinx_gallery.sorting import ExplicitOrder
+from sphinx_gallery.sorting import ExplicitOrder, NumberOfCodeLinesSortKey
 examples_dirs = ['../examples', '../tutorials']
 gallery_dirs = ['auto_examples', 'tutorials']
 
@@ -336,6 +336,7 @@ sphinx_gallery_conf = {
     'subsection_order': ExplicitOrder(['../examples/sin_func',
                                        '../examples/no_output',
                                        '../tutorials/seaborn']),
+    'within_subsection_order': NumberOfCodeLinesSortKey,
     'find_mayavi_figures': find_mayavi_figures,
     'expected_failing_examples': ['../examples/no_output/plot_raise.py',
                                   '../examples/no_output/plot_syntaxerror.py']
diff --git a/sphinx_gallery/gen_gallery.py b/sphinx_gallery/gen_gallery.py
index dff8170..38fb667 100644
--- a/sphinx_gallery/gen_gallery.py
+++ b/sphinx_gallery/gen_gallery.py
@@ -21,6 +21,7 @@ from . import sphinx_compatibility
 from .gen_rst import generate_dir_rst, SPHX_GLR_SIG
 from .docs_resolv import embed_code_links
 from .downloads import generate_zipfiles
+from .sorting import NumberOfCodeLinesSortKey
 
 try:
     FileNotFoundError
@@ -32,6 +33,7 @@ DEFAULT_GALLERY_CONF = {
     'filename_pattern': re.escape(os.sep) + 'plot',
     'examples_dirs': os.path.join('..', 'examples'),
     'subsection_order': None,
+    'within_subsection_order': NumberOfCodeLinesSortKey,
     'gallery_dirs': 'auto_examples',
     'backreferences_dir': None,
     'doc_module': (),
diff --git a/sphinx_gallery/gen_rst.py b/sphinx_gallery/gen_rst.py
index 73c86a0..41fd465 100644
--- a/sphinx_gallery/gen_rst.py
+++ b/sphinx_gallery/gen_rst.py
@@ -153,22 +153,30 @@ def codestr2rst(codestr, lang='python', lineno=None):
     return code_directive + indented_block
 
 
-def extract_intro(filename, docstring):
+def extract_intro_and_title(filename, docstring):
     """ Extract the first paragraph of module-level docstring. max:95 char"""
 
     # lstrip is just in case docstring has a '\n\n' at the beginning
     paragraphs = docstring.lstrip().split('\n\n')
-    if len(paragraphs) > 1:
-        first_paragraph = re.sub('\n', ' ', paragraphs[1])
-        first_paragraph = (first_paragraph[:95] + '...'
-                           if len(first_paragraph) > 95 else first_paragraph)
-    else:
+    # remove comments and other syntax like `.. _link:`
+    paragraphs = [p for p in paragraphs if not p.startswith('.. ')]
+    if len(paragraphs) <= 1:
         raise ValueError(
             "Example docstring should have a header for the example title "
             "and at least a paragraph explaining what the example is about. "
             "Please check the example file:\n {}\n".format(filename))
+    # Title is the first paragraph with any ReSTructuredText title chars
+    # removed, i.e. lines that consist of (all the same) 7-bit non-ASCII chars.
+    # This conditional is not perfect but should hopefully be good enough.
+    title = paragraphs[0].strip().split('\n')
+    title = ' '.join(t for t in title if len(t) > 0 and
+                     (ord(t[0]) >= 128 or t[0].isalnum()))
+    # Concatenate all lines of the first paragraph and truncate at 95 chars
+    first_paragraph = re.sub('\n', ' ', paragraphs[1])
+    first_paragraph = (first_paragraph[:95] + '...'
+                       if len(first_paragraph) > 95 else first_paragraph)
 
-    return first_paragraph
+    return first_paragraph, title
 
 
 def get_md5sum(src_file):
@@ -377,8 +385,10 @@ def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs):
 
     if not os.path.exists(target_dir):
         os.makedirs(target_dir)
-    sorted_listdir = [fname for fname in sorted(os.listdir(src_dir))
-                      if fname.endswith('.py')]
+    listdir = [fname for fname in os.listdir(src_dir)
+               if fname.endswith('.py')]
+    sorted_listdir = sorted(
+        listdir, key=gallery_conf['within_subsection_order'](src_dir))
     entries_text = []
     computation_times = []
     build_target_dir = os.path.relpath(target_dir, gallery_conf['src_dir'])
@@ -387,29 +397,25 @@ def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs):
         'Generating gallery for %s ' % build_target_dir,
         length=len(sorted_listdir))
     for fname in iterator:
-        intro, amount_of_code, time_elapsed = generate_file_rst(
+        intro, time_elapsed = generate_file_rst(
             fname,
             target_dir,
             src_dir,
             gallery_conf)
         computation_times.append((time_elapsed, fname))
-        new_fname = os.path.join(src_dir, fname)
         this_entry = _thumbnail_div(build_target_dir, fname, intro) + """
 
 .. toctree::
    :hidden:
 
    /%s\n""" % os.path.join(build_target_dir, fname[:-3]).replace(os.sep, '/')
-        entries_text.append((amount_of_code, this_entry))
+        entries_text.append(this_entry)
 
         if gallery_conf['backreferences_dir']:
             write_backreferences(seen_backrefs, gallery_conf,
                                  target_dir, fname, intro)
 
-    # sort to have the smallest entries in the beginning
-    entries_text.sort()
-
-    for _, entry_text in entries_text:
+    for entry_text in entries_text:
         fhindex += entry_text
 
     # clear at the end of the section
@@ -533,8 +539,6 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
     -------
     intro: str
         The introduction of the example
-    amount_of_code : int
-        character count of the corresponding python script in file
     time_elapsed : float
         seconds required to run the script
     """
@@ -543,13 +547,10 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
     example_file = os.path.join(target_dir, fname)
     shutil.copyfile(src_file, example_file)
     file_conf, script_blocks = split_code_and_text_blocks(src_file)
-    amount_of_code = sum([len(bcontent)
-                          for blabel, bcontent, lineno in script_blocks
-                          if blabel == 'code'])
-    intro = extract_intro(fname, script_blocks[0][1])
+    intro, title = extract_intro_and_title(fname, script_blocks[0][1])
 
     if md5sum_is_current(example_file):
-        return intro, amount_of_code, 0
+        return intro, 0
 
     image_dir = os.path.join(target_dir, 'images')
     if not os.path.exists(image_dir):
@@ -648,4 +649,4 @@ def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
     if block_vars['execute_script']:
         logger.debug("%s ran in : %.2g seconds", src_file, time_elapsed)
 
-    return intro, amount_of_code, time_elapsed
+    return intro, time_elapsed
diff --git a/sphinx_gallery/sorting.py b/sphinx_gallery/sorting.py
index 98b7dbe..d889e1c 100644
--- a/sphinx_gallery/sorting.py
+++ b/sphinx_gallery/sorting.py
@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 r"""
-Sorters for Sphinx-Gallery subsections
-======================================
+Sorters for Sphinx-Gallery (sub)sections
+========================================
 
-Sorting key functions for gallery subsection folders
+Sorting key functions for gallery subsection folders and section files.
 """
 # Created: Sun May 21 20:38:59 2017
 # Author: Óscar Nájera
@@ -13,21 +13,24 @@ from __future__ import division, absolute_import, print_function
 import os
 import types
 
+from .gen_rst import extract_intro_and_title
+from .py_source_parser import split_code_and_text_blocks
+
 
 class ExplicitOrder(object):
-    """Sorting key for all galleries subsections
+    """Sorting key for all gallery subsections.
 
-    This requires all folders to be listed otherwise an exception is raised
+    This requires all folders to be listed otherwise an exception is raised.
 
     Parameters
     ----------
     ordered_list : list, tuple, types.GeneratorType
-        Hold the paths of each galleries' subsections
+        Hold the paths of each galleries' subsections.
 
     Raises
     ------
     ValueError
-        Wrong input type or Subgallery path missing
+        Wrong input type or Subgallery path missing.
     """
 
     def __init__(self, ordered_list):
@@ -46,3 +49,71 @@ class ExplicitOrder(object):
             raise ValueError('If you use an explicit folder ordering, you '
                              'must specify all folders. Explicit order not '
                              'found for {}'.format(item))
+
+
+class _SortKey(object):
+    """Base class for section order key classes."""
+
+    def __init__(self, src_dir):
+        self.src_dir = src_dir
+
+
+class NumberOfCodeLinesSortKey(_SortKey):
+    """Sort examples in src_dir by the number of code lines.
+
+    Parameters
+    ----------
+    src_dir : str
+        The source directory.
+    """
+
+    def __call__(self, filename):
+        src_file = os.path.normpath(os.path.join(self.src_dir, filename))
+        file_conf, script_blocks = split_code_and_text_blocks(src_file)
+        amount_of_code = sum([len(bcontent)
+                              for blabel, bcontent, lineno in script_blocks
+                              if blabel == 'code'])
+        return amount_of_code
+
+
+class FileSizeSortKey(_SortKey):
+    """Sort examples in src_dir by file size.
+
+    Parameters
+    ----------
+    src_dir : str
+        The source directory.
+    """
+
+    def __call__(self, filename):
+        src_file = os.path.normpath(os.path.join(self.src_dir, filename))
+        return os.stat(src_file).st_size
+
+
+class FileNameSortKey(_SortKey):
+    """Sort examples in src_dir by file size.
+
+    Parameters
+    ----------
+    src_dir : str
+        The source directory.
+    """
+
+    def __call__(self, filename):
+        return filename
+
+
+class ExampleTitleSortKey(_SortKey):
+    """Sort examples in src_dir by example title.
+
+    Parameters
+    ----------
+    src_dir : str
+        The source directory.
+    """
+
+    def __call__(self, filename):
+        src_file = os.path.normpath(os.path.join(self.src_dir, filename))
+        _, script_blocks = split_code_and_text_blocks(src_file)
+        _, title = extract_intro_and_title(src_file, script_blocks[0][1])
+        return title


Test patch:
diff --git a/sphinx_gallery/tests/test_gen_gallery.py b/sphinx_gallery/tests/test_gen_gallery.py
index 79e0b49..09c11e8 100644
--- a/sphinx_gallery/tests/test_gen_gallery.py
+++ b/sphinx_gallery/tests/test_gen_gallery.py
@@ -5,15 +5,17 @@ r"""
 Test Sphinx-Gallery
 """
 
-from __future__ import division, absolute_import, print_function, unicode_literals
+from __future__ import (division, absolute_import, print_function,
+                        unicode_literals)
+import codecs
 import os
-import tempfile
+import re
 import shutil
+import tempfile
 import pytest
 from sphinx.application import Sphinx
 from sphinx.errors import ExtensionError
 from sphinx_gallery.gen_rst import MixedEncodingStringIO
-from sphinx_gallery.gen_gallery import DEFAULT_GALLERY_CONF
 from sphinx_gallery import sphinx_compatibility
 
 
@@ -162,3 +164,69 @@ def test_config_backreferences(config_app):
         'gen_modules', 'backreferences')
     build_warn = config_app._warning.getvalue()
     assert build_warn == ""
+
+
+def _check_order(config_app, key):
+    index_fname = os.path.join(config_app.outdir, '..', 'ex', 'index.rst')
+    order = list()
+    regex = '.*:%s=(.):.*' % key
+    with codecs.open(index_fname, 'r', 'utf-8') as fid:
+        for line in fid:
+            if 'sphx-glr-thumbcontainer' in line:
+                order.append(int(re.match(regex, line).group(1)))
+    assert len(order) == 3
+    assert order == [1, 2, 3]
+
+
+@pytest.mark.conf_file(content="""
+import sphinx_gallery
+extensions = ['sphinx_gallery.gen_gallery']
+sphinx_gallery_conf = {
+    'examples_dirs': 'src',
+    'gallery_dirs': 'ex',
+}""")
+def test_example_sorting_default(config_app):
+    """Test sorting of examples by default key (number of code lines)."""
+    _check_order(config_app, 'lines')
+
+
+@pytest.mark.conf_file(content="""
+import sphinx_gallery
+from sphinx_gallery.sorting import FileSizeSortKey
+extensions = ['sphinx_gallery.gen_gallery']
+sphinx_gallery_conf = {
+    'examples_dirs': 'src',
+    'gallery_dirs': 'ex',
+    'within_subsection_order': FileSizeSortKey,
+}""")
+def test_example_sorting_filesize(config_app):
+    """Test sorting of examples by filesize."""
+    _check_order(config_app, 'filesize')
+
+
+@pytest.mark.conf_file(content="""
+import sphinx_gallery
+from sphinx_gallery.sorting import FileNameSortKey
+extensions = ['sphinx_gallery.gen_gallery']
+sphinx_gallery_conf = {
+    'examples_dirs': 'src',
+    'gallery_dirs': 'ex',
+    'within_subsection_order': FileNameSortKey,
+}""")
+def test_example_sorting_filename(config_app):
+    """Test sorting of examples by filename."""
+    _check_order(config_app, 'filename')
+
+
+@pytest.mark.conf_file(content="""
+import sphinx_gallery
+from sphinx_gallery.sorting import ExampleTitleSortKey
+extensions = ['sphinx_gallery.gen_gallery']
+sphinx_gallery_conf = {
+    'examples_dirs': 'src',
+    'gallery_dirs': 'ex',
+    'within_subsection_order': ExampleTitleSortKey,
+}""")
+def test_example_sorting_title(config_app):
+    """Test sorting of examples by title."""
+    _check_order(config_app, 'title')
diff --git a/sphinx_gallery/tests/test_gen_rst.py b/sphinx_gallery/tests/test_gen_rst.py
index 1047f37..75d967d 100644
--- a/sphinx_gallery/tests/test_gen_rst.py
+++ b/sphinx_gallery/tests/test_gen_rst.py
@@ -25,7 +25,8 @@ from sphinx_gallery import sphinx_compatibility
 import matplotlib.pyplot as plt
 
 CONTENT = [
-    '"""'
+    '"""',
+    '================',
     'Docstring header',
     '================',
     '',
@@ -109,11 +110,13 @@ def test_codestr2rst():
     assert reference == output
 
 
-def test_extract_intro():
-    result = sg.extract_intro('<string>', '\n'.join(CONTENT[1:9]))
-    assert 'Docstring' not in result
-    assert result == 'This is the description of the example which goes on and on, Óscar'  # noqa
-    assert 'second paragraph' not in result
+def test_extract_intro_and_title():
+    intro, title = sg.extract_intro_and_title('<string>',
+                                              '\n'.join(CONTENT[1:10]))
+    assert title == 'Docstring header'
+    assert 'Docstring' not in intro
+    assert intro == 'This is the description of the example which goes on and on, Óscar'  # noqa
+    assert 'second paragraph' not in intro
 
 
 def test_md5sums():
diff --git a/sphinx_gallery/tests/test_sorting.py b/sphinx_gallery/tests/test_sorting.py
index 036e464..d6a866d 100644
--- a/sphinx_gallery/tests/test_sorting.py
+++ b/sphinx_gallery/tests/test_sorting.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 r"""
-Tests for sorting keys on gallery sections
-==========================================
+Tests for sorting keys on gallery (sub)sections
+===============================================
 
 """
 # Author: Óscar Nájera
diff --git a/sphinx_gallery/tests/testconfs/src/plot_1.py b/sphinx_gallery/tests/testconfs/src/plot_1.py
new file mode 100644
index 0000000..721bfc1
--- /dev/null
+++ b/sphinx_gallery/tests/testconfs/src/plot_1.py
@@ -0,0 +1,11 @@
+"""
+======
+B test
+======
+
+:filename=1:title=2:lines=3:filesize=2:
+"""
+
+print('foo')
+print('bar')
+print('again')
diff --git a/sphinx_gallery/tests/testconfs/src/plot_2.py b/sphinx_gallery/tests/testconfs/src/plot_2.py
new file mode 100644
index 0000000..d468d79
--- /dev/null
+++ b/sphinx_gallery/tests/testconfs/src/plot_2.py
@@ -0,0 +1,9 @@
+"""
+======
+C test
+======
+
+:filename=2:title=3:lines=1:filesize=1:
+"""
+
+print('foo')
diff --git a/sphinx_gallery/tests/testconfs/src/plot_3.py b/sphinx_gallery/tests/testconfs/src/plot_3.py
new file mode 100644
index 0000000..ceea4c0
--- /dev/null
+++ b/sphinx_gallery/tests/testconfs/src/plot_3.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""
+.. _extra_ref:
+
+===========================================================
+A test with a really long title to make the filesize larger
+===========================================================
+
+:filename=3:title=1:lines=2:filesize=3:
+"""
+
+print('foo')
+print('bar')