Repository: takluyver/flit (python)

Issue:
Support publishing markdown descriptions to Warehouse
PEP 566 is now implemented in Warehouse:
https://dustingram.com/articles/2018/03/16/markdown-descriptions-on-pypi

Pull request description:
Official support for markdown and plain text descriptions
Fixes #166.

Until recently, PyPI only supported rst descriptions, falling back to plain text if the rst was invalid. Now a new metadata field `Description-Content-Type` lets us specify whether the description is rst, markdown or plain text. This new code uses that, picking the content type based on the file extension (`.rst`, `.md` or `.txt`). If the extension is something else, Flit will warn the user and leave the field unset, so it's up to PyPI what to do with it.

Markdown support has been a secret feature of Flit for some time, by using pandoc to convert markdown to rst (thanks @Carreau!). This makes it an official feature, and gets rid of the code to call pandoc, since PyPI can now handle markdown natively.

Patch:
diff --git a/.gitignore b/.gitignore
index c2acbe3..d64b220 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ __pycache__/
 /tests/samples/dist/
 /htmlcov/
 /.coverage
+/.pytest_cache
diff --git a/.travis.yml b/.travis.yml
index 8afc588..b43c127 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,7 @@ python:
   - "3.5"
   - "3.4"
 install:
+  - pip install --upgrade pytest
   - pip install -r requirements-test.txt
   - pip install codecov
 script: py.test --cov=flit
diff --git a/doc/history.rst b/doc/history.rst
index 4effcd6..a9375e7 100644
--- a/doc/history.rst
+++ b/doc/history.rst
@@ -4,6 +4,10 @@ Release history
 Version 0.14
 ------------
 
+- The description file may now be written in reStructuredText, Markdown or
+  plain text. The file extension should indicate which of these formats it is
+  (``.rst``, ``.md`` or ``.txt``). Previously, only reStructuredText was
+  officially supported.
 - Multiple links (e.g. documentation, bug tracker) can now be specified in a
   new :ref:`[tool.flit.metadata.urls] section <pyproject_toml_urls>` of
   ``pyproject.toml``.
diff --git a/doc/pyproject_toml.rst b/doc/pyproject_toml.rst
index 09dfcf4..8e7a826 100644
--- a/doc/pyproject_toml.rst
+++ b/doc/pyproject_toml.rst
@@ -71,11 +71,11 @@ dev-requires
   These are not (yet) encoded in the wheel, but are used when doing
   ``flit install``.
 description-file
-  A path (relative to the .ini file) to a file containing a longer description
+  A path (relative to the .toml file) to a file containing a longer description
   of your package to show on PyPI. This should be written in `reStructuredText
-  <http://docutils.sourceforge.net/docs/user/rst/quickref.html>`_, if your long
-  description is not valid reStructuredText, a warning will be printed,
-  and it will be interpreted as plain text on PyPI.
+  <http://docutils.sourceforge.net/docs/user/rst/quickref.html>`_, Markdown or
+  plain text, and the filename should have the appropriate extension
+  (``.rst``, ``.md`` or ``.txt``).
 classifiers
   A list of `Trove classifiers <https://pypi.python.org/pypi?%3Aaction=list_classifiers>`_.
 requires-python
diff --git a/flit/common.py b/flit/common.py
index ad79813..b73ebc9 100644
--- a/flit/common.py
+++ b/flit/common.py
@@ -212,6 +212,7 @@ class Metadata:
     keywords = None
     download_url = None
     requires_python = None
+    description_content_type = None
 
     platform = ()
     supported_platform = ()
@@ -224,8 +225,9 @@ class Metadata:
     requires_dist = ()
     obsoletes_dist = ()
     requires_external = ()
+    provides_extra = ()
 
-    metadata_version="1.2"
+    metadata_version="2.1"
 
     # this is part of metadata spec 2, we are using it for installation but it
     # doesn't actually get written to the metadata file
@@ -244,7 +246,7 @@ class Metadata:
         return n.lower().replace('-', '_')
 
     def write_metadata_file(self, fp):
-        """Write out metadata in the 1.x format (email like)"""
+        """Write out metadata in the email headers format"""
         fields = [
             'Metadata-Version',
             'Name',
@@ -260,6 +262,7 @@ class Metadata:
             'Maintainer',
             'Maintainer-email',
             'Requires-Python',
+            'Description-Content-Type',
         ]
 
         for field in fields:
@@ -280,6 +283,9 @@ class Metadata:
         for url in self.project_urls:
             fp.write('Project-URL: {}\n'.format(url))
 
+        for extra in self.provides_extra:
+            fp.write('Provides-Extra: {}\n'.format(extra))
+
         if self.description is not None:
             fp.write('\n' + self.description + '\n')
 
diff --git a/flit/inifile.py b/flit/inifile.py
index 234683f..2af1ae2 100644
--- a/flit/inifile.py
+++ b/flit/inifile.py
@@ -157,6 +157,12 @@ def _read_pkg_ini(path):
 
     return cp
 
+readme_ext_to_content_type = {
+    '.rst': 'text/x-rst',
+    '.md': 'text/markdown',
+    '.txt': 'text/plain',
+}
+
 def _prep_metadata(md_sect, path):
     """Process & verify the metadata from a config file
     
@@ -180,23 +186,26 @@ def _prep_metadata(md_sect, path):
         description_file = path.parent / md_sect.get('description-file')
         with description_file.open(encoding='utf-8') as f:
             raw_desc =  f.read()
-        if description_file.suffix == '.md':
-            try:
-                import pypandoc
-                log.debug('will convert %s to rst', description_file)
-                raw_desc = pypandoc.convert(raw_desc, 'rst', format='markdown')
-            except Exception:
-                log.warning('Unable to convert markdown to rst. Please install '
-                    '`pypandoc` and `pandoc` to use markdown long description.')
-
-        # rst check
-        stream = io.StringIO()
-        res = render(raw_desc, stream)
-        if not res:
-            log.warning("The file description seems not to be valid rst for PyPI;"
-                    " it will be interpreted as plain text")
-            log.warning(stream.getvalue())
+        ext = description_file.suffix
+        try:
+            mimetype = readme_ext_to_content_type[ext]
+        except KeyError:
+            log.warning("Unknown extension %r for description file.", ext)
+            log.warning("  Recognised extensions: %s",
+                        " ".join(readme_ext_to_content_type))
+            mimetype = None
+
+        if mimetype == 'text/x-rst':
+            # rst check
+            stream = io.StringIO()
+            res = render(raw_desc, stream)
+            if not res:
+                log.warning("The file description seems not to be valid rst for PyPI;"
+                        " it will be interpreted as plain text")
+                log.warning(stream.getvalue())
+
         md_dict['description'] =  raw_desc
+        md_dict['description_content_type'] = mimetype
 
     if 'urls' in md_sect:
         project_urls = md_dict['project_urls'] = []


Test patch:
diff --git a/tests/samples/bad-description-ext.toml b/tests/samples/bad-description-ext.toml
new file mode 100644
index 0000000..1062829
--- /dev/null
+++ b/tests/samples/bad-description-ext.toml
@@ -0,0 +1,9 @@
+[build-system]
+requires = ["flit"]
+
+[tool.flit.metadata]
+module = "module1"
+author = "Sir Robin"
+author-email = "robin@camelot.uk"
+home-page = "http://github.com/sirrobin/module1"
+description-file = "module1.py"  # WRONG
diff --git a/tests/test_inifile.py b/tests/test_inifile.py
index d506651..c7d3e5f 100644
--- a/tests/test_inifile.py
+++ b/tests/test_inifile.py
@@ -1,3 +1,4 @@
+import logging
 import pathlib
 
 import pytest
@@ -48,4 +49,10 @@ def test_description_file():
     info = read_pkg_ini(samples_dir / 'package1-pkg.ini')
     assert info['metadata']['description'] == \
         "Sample description for test.\n"
+    assert info['metadata']['description_content_type'] == 'text/x-rst'
 
+def test_bad_description_extension(caplog):
+    info = read_pkg_ini(samples_dir / 'bad-description-ext.toml')
+    assert info['metadata']['description_content_type'] is None
+    assert any((r.levelno == logging.WARN and "Unknown extension" in r.msg)
+                for r in caplog.records)