Repository: encode/django-rest-framework

Issue:
format_suffix_pattern not working with path() in django 2.0
## Checklist

- [x] I have verified that that issue exists against the `master` branch of Django REST framework.
- [x] I have searched for similar issues in both open and closed tickets and cannot find a duplicate.
- [x] This is not a usage question. (Those should be directed to the [discussion group](https://groups.google.com/forum/#!forum/django-rest-framework) instead.)
- [x] This cannot be dealt with as a third party library. (We prefer new functionality to be [in the form of third party libraries](http://www.django-rest-framework.org/topics/third-party-resources/#about-third-party-packages) where possible.)
- [x] I have reduced the issue to the simplest possible case.
- [ ] I have included a failing test as a pull request. (If you are unable to do so we can still accept the issue.)

## Steps to reproduce

## Expected behavior
when using format_suffix_patterns() from the rest_framwork.urlpatterns class, it should format the url pattern for url() from django.conf.urls as well as the newly introduced path() from django.urls

## Actual behavior
getting an error when using path() in urlpatterns[], and then formatting it using the format_suffix_patterns() method


Discussion:
Next step on that issue would be to write a failing test case.
I am having the same issue with the following urls.py file:

```
from django.urls import path, re_path
from rest_framework.urlpatterns import format_suffix_patterns
from snippets import views

urlpatterns = [
    path('snippets/', views.snippet_list),
    re_path('snippets/(?P<pk>[0-9]+)/', views.snippet_detail)
]

urlpatterns = format_suffix_patterns(urlpatterns)
```

Resolving patch:
diff --git a/rest_framework/compat.py b/rest_framework/compat.py
--- a/rest_framework/compat.py
+++ b/rest_framework/compat.py
@@ -29,7 +29,11 @@
     )
 
 
-def get_regex_pattern(urlpattern):
+def get_original_route(urlpattern):
+    """
+    Get the original route/regex that was typed in by the user into the path(), re_path() or url() directive. This
+    is in contrast with get_regex_pattern below, which for RoutePattern returns the raw regex generated from the path().
+    """
     if hasattr(urlpattern, 'pattern'):
         # Django 2.0
         return str(urlpattern.pattern)
@@ -38,6 +42,29 @@ def get_regex_pattern(urlpattern):
         return urlpattern.regex.pattern
 
 
+def get_regex_pattern(urlpattern):
+    """
+    Get the raw regex out of the urlpattern's RegexPattern or RoutePattern. This is always a regular expression,
+    unlike get_original_route above.
+    """
+    if hasattr(urlpattern, 'pattern'):
+        # Django 2.0
+        return urlpattern.pattern.regex.pattern
+    else:
+        # Django < 2.0
+        return urlpattern.regex.pattern
+
+
+def is_route_pattern(urlpattern):
+    if hasattr(urlpattern, 'pattern'):
+        # Django 2.0
+        from django.urls.resolvers import RoutePattern
+        return isinstance(urlpattern.pattern, RoutePattern)
+    else:
+        # Django < 2.0
+        return False
+
+
 def make_url_resolver(regex, urlpatterns):
     try:
         # Django 2.0
@@ -257,10 +284,11 @@ def md_filter_add_syntax_highlight(md):
 
 # Django 1.x url routing syntax. Remove when dropping Django 1.11 support.
 try:
-    from django.urls import include, path, re_path # noqa
+    from django.urls import include, path, re_path, register_converter  # noqa
 except ImportError:
     from django.conf.urls import include, url # noqa
     path = None
+    register_converter = None
     re_path = url
 
 
diff --git a/rest_framework/schemas/generators.py b/rest_framework/schemas/generators.py
--- a/rest_framework/schemas/generators.py
+++ b/rest_framework/schemas/generators.py
@@ -16,7 +16,7 @@
 
 from rest_framework import exceptions
 from rest_framework.compat import (
-    URLPattern, URLResolver, coreapi, coreschema, get_regex_pattern
+    URLPattern, URLResolver, coreapi, coreschema, get_original_route
 )
 from rest_framework.request import clone_request
 from rest_framework.settings import api_settings
@@ -170,7 +170,7 @@ def get_api_endpoints(self, patterns=None, prefix=''):
         api_endpoints = []
 
         for pattern in patterns:
-            path_regex = prefix + get_regex_pattern(pattern)
+            path_regex = prefix + get_original_route(pattern)
             if isinstance(pattern, URLPattern):
                 path = self.get_path_from_regex(path_regex)
                 callback = pattern.callback
diff --git a/rest_framework/urlpatterns.py b/rest_framework/urlpatterns.py
--- a/rest_framework/urlpatterns.py
+++ b/rest_framework/urlpatterns.py
@@ -2,11 +2,39 @@
 
 from django.conf.urls import include, url
 
-from rest_framework.compat import URLResolver, get_regex_pattern
+from rest_framework.compat import (
+    URLResolver, get_regex_pattern, is_route_pattern, path, register_converter
+)
 from rest_framework.settings import api_settings
 
 
-def apply_suffix_patterns(urlpatterns, suffix_pattern, suffix_required):
+def _get_format_path_converter(suffix_kwarg, allowed):
+    if allowed:
+        if len(allowed) == 1:
+            allowed_pattern = allowed[0]
+        else:
+            allowed_pattern = '(?:%s)' % '|'.join(allowed)
+        suffix_pattern = r"\.%s/?" % allowed_pattern
+    else:
+        suffix_pattern = r"\.[a-z0-9]+/?"
+
+    class FormatSuffixConverter:
+        regex = suffix_pattern
+
+        def to_python(self, value):
+            return value.strip('./')
+
+        def to_url(self, value):
+            return '.' + value + '/'
+
+    converter_name = 'drf_format_suffix'
+    if allowed:
+        converter_name += '_' + '_'.join(allowed)
+
+    return converter_name, FormatSuffixConverter
+
+
+def apply_suffix_patterns(urlpatterns, suffix_pattern, suffix_required, suffix_route=None):
     ret = []
     for urlpattern in urlpatterns:
         if isinstance(urlpattern, URLResolver):
@@ -18,8 +46,18 @@ def apply_suffix_patterns(urlpatterns, suffix_pattern, suffix_required):
             # Add in the included patterns, after applying the suffixes
             patterns = apply_suffix_patterns(urlpattern.url_patterns,
                                              suffix_pattern,
-                                             suffix_required)
-            ret.append(url(regex, include((patterns, app_name), namespace), kwargs))
+                                             suffix_required,
+                                             suffix_route)
+
+            # if the original pattern was a RoutePattern we need to preserve it
+            if is_route_pattern(urlpattern):
+                assert path is not None
+                route = str(urlpattern.pattern)
+                new_pattern = path(route, include((patterns, app_name), namespace), kwargs)
+            else:
+                new_pattern = url(regex, include((patterns, app_name), namespace), kwargs)
+
+            ret.append(new_pattern)
         else:
             # Regular URL pattern
             regex = get_regex_pattern(urlpattern).rstrip('$').rstrip('/') + suffix_pattern
@@ -29,7 +67,17 @@ def apply_suffix_patterns(urlpatterns, suffix_pattern, suffix_required):
             # Add in both the existing and the new urlpattern
             if not suffix_required:
                 ret.append(urlpattern)
-            ret.append(url(regex, view, kwargs, name))
+
+            # if the original pattern was a RoutePattern we need to preserve it
+            if is_route_pattern(urlpattern):
+                assert path is not None
+                assert suffix_route is not None
+                route = str(urlpattern.pattern).rstrip('$').rstrip('/') + suffix_route
+                new_pattern = path(route, view, kwargs, name)
+            else:
+                new_pattern = url(regex, view, kwargs, name)
+
+            ret.append(new_pattern)
 
     return ret
 
@@ -60,4 +108,12 @@ def format_suffix_patterns(urlpatterns, suffix_required=False, allowed=None):
     else:
         suffix_pattern = r'\.(?P<%s>[a-z0-9]+)/?$' % suffix_kwarg
 
-    return apply_suffix_patterns(urlpatterns, suffix_pattern, suffix_required)
+    if path and register_converter:
+        converter_name, suffix_converter = _get_format_path_converter(suffix_kwarg, allowed)
+        register_converter(suffix_converter, converter_name)
+
+        suffix_route = '<%s:%s>' % (converter_name, suffix_kwarg)
+    else:
+        suffix_route = None
+
+    return apply_suffix_patterns(urlpatterns, suffix_pattern, suffix_required, suffix_route)


Test patch:
diff --git a/tests/test_urlpatterns.py b/tests/test_urlpatterns.py
--- a/tests/test_urlpatterns.py
+++ b/tests/test_urlpatterns.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
+import unittest
 from collections import namedtuple
 
 from django.conf.urls import include, url
 from django.test import TestCase
 from django.urls import Resolver404
 
-from rest_framework.compat import make_url_resolver
+from rest_framework.compat import make_url_resolver, path, re_path
 from rest_framework.test import APIRequestFactory
 from rest_framework.urlpatterns import format_suffix_patterns
 
@@ -23,52 +24,58 @@ class FormatSuffixTests(TestCase):
     Tests `format_suffix_patterns` against different URLPatterns to ensure the
     URLs still resolve properly, including any captured parameters.
     """
-    def _resolve_urlpatterns(self, urlpatterns, test_paths):
+    def _resolve_urlpatterns(self, urlpatterns, test_paths, allowed=None):
         factory = APIRequestFactory()
         try:
-            urlpatterns = format_suffix_patterns(urlpatterns)
+            urlpatterns = format_suffix_patterns(urlpatterns, allowed=allowed)
         except Exception:
             self.fail("Failed to apply `format_suffix_patterns` on  the supplied urlpatterns")
         resolver = make_url_resolver(r'^/', urlpatterns)
         for test_path in test_paths:
+            try:
+                test_path, expected_resolved = test_path
+            except (TypeError, ValueError):
+                expected_resolved = True
+
             request = factory.get(test_path.path)
             try:
                 callback, callback_args, callback_kwargs = resolver.resolve(request.path_info)
+            except Resolver404:
+                callback, callback_args, callback_kwargs = (None, None, None)
+                if expected_resolved:
+                    raise
             except Exception:
                 self.fail("Failed to resolve URL: %s" % request.path_info)
+
+            if not expected_resolved:
+                assert callback is None
+                continue
+
             assert callback_args == test_path.args
             assert callback_kwargs == test_path.kwargs
 
-    def test_trailing_slash(self):
-        factory = APIRequestFactory()
-        urlpatterns = format_suffix_patterns([
-            url(r'^test/$', dummy_view),
-        ])
-        resolver = make_url_resolver(r'^/', urlpatterns)
-
+    def _test_trailing_slash(self, urlpatterns):
         test_paths = [
             (URLTestPath('/test.api', (), {'format': 'api'}), True),
             (URLTestPath('/test/.api', (), {'format': 'api'}), False),
             (URLTestPath('/test.api/', (), {'format': 'api'}), True),
         ]
+        self._resolve_urlpatterns(urlpatterns, test_paths)
 
-        for test_path, expected_resolved in test_paths:
-            request = factory.get(test_path.path)
-            try:
-                callback, callback_args, callback_kwargs = resolver.resolve(request.path_info)
-            except Resolver404:
-                callback, callback_args, callback_kwargs = (None, None, None)
-            if not expected_resolved:
-                assert callback is None
-                continue
-
-            assert callback_args == test_path.args
-            assert callback_kwargs == test_path.kwargs
+    def test_trailing_slash(self):
+        urlpatterns = [
+            url(r'^test/$', dummy_view),
+        ]
+        self._test_trailing_slash(urlpatterns)
 
-    def test_format_suffix(self):
+    @unittest.skipUnless(path, 'needs Django 2')
+    def test_trailing_slash_django2(self):
         urlpatterns = [
-            url(r'^test$', dummy_view),
+            path('test/', dummy_view),
         ]
+        self._test_trailing_slash(urlpatterns)
+
+    def _test_format_suffix(self, urlpatterns):
         test_paths = [
             URLTestPath('/test', (), {}),
             URLTestPath('/test.api', (), {'format': 'api'}),
@@ -76,10 +83,36 @@ def test_format_suffix(self):
         ]
         self._resolve_urlpatterns(urlpatterns, test_paths)
 
-    def test_default_args(self):
+    def test_format_suffix(self):
         urlpatterns = [
-            url(r'^test$', dummy_view, {'foo': 'bar'}),
+            url(r'^test$', dummy_view),
+        ]
+        self._test_format_suffix(urlpatterns)
+
+    @unittest.skipUnless(path, 'needs Django 2')
+    def test_format_suffix_django2(self):
+        urlpatterns = [
+            path('test', dummy_view),
         ]
+        self._test_format_suffix(urlpatterns)
+
+    @unittest.skipUnless(path, 'needs Django 2')
+    def test_format_suffix_django2_args(self):
+        urlpatterns = [
+            path('convtest/<int:pk>', dummy_view),
+            re_path(r'^retest/(?P<pk>[0-9]+)$', dummy_view),
+        ]
+        test_paths = [
+            URLTestPath('/convtest/42', (), {'pk': 42}),
+            URLTestPath('/convtest/42.api', (), {'pk': 42, 'format': 'api'}),
+            URLTestPath('/convtest/42.asdf', (), {'pk': 42, 'format': 'asdf'}),
+            URLTestPath('/retest/42', (), {'pk': '42'}),
+            URLTestPath('/retest/42.api', (), {'pk': '42', 'format': 'api'}),
+            URLTestPath('/retest/42.asdf', (), {'pk': '42', 'format': 'asdf'}),
+        ]
+        self._resolve_urlpatterns(urlpatterns, test_paths)
+
+    def _test_default_args(self, urlpatterns):
         test_paths = [
             URLTestPath('/test', (), {'foo': 'bar', }),
             URLTestPath('/test.api', (), {'foo': 'bar', 'format': 'api'}),
@@ -87,6 +120,27 @@ def test_default_args(self):
         ]
         self._resolve_urlpatterns(urlpatterns, test_paths)
 
+    def test_default_args(self):
+        urlpatterns = [
+            url(r'^test$', dummy_view, {'foo': 'bar'}),
+        ]
+        self._test_default_args(urlpatterns)
+
+    @unittest.skipUnless(path, 'needs Django 2')
+    def test_default_args_django2(self):
+        urlpatterns = [
+            path('test', dummy_view, {'foo': 'bar'}),
+        ]
+        self._test_default_args(urlpatterns)
+
+    def _test_included_urls(self, urlpatterns):
+        test_paths = [
+            URLTestPath('/test/path', (), {'foo': 'bar', }),
+            URLTestPath('/test/path.api', (), {'foo': 'bar', 'format': 'api'}),
+            URLTestPath('/test/path.asdf', (), {'foo': 'bar', 'format': 'asdf'}),
+        ]
+        self._resolve_urlpatterns(urlpatterns, test_paths)
+
     def test_included_urls(self):
         nested_patterns = [
             url(r'^path$', dummy_view)
@@ -94,9 +148,79 @@ def test_included_urls(self):
         urlpatterns = [
             url(r'^test/', include(nested_patterns), {'foo': 'bar'}),
         ]
+        self._test_included_urls(urlpatterns)
+
+    @unittest.skipUnless(path, 'needs Django 2')
+    def test_included_urls_django2(self):
+        nested_patterns = [
+            path('path', dummy_view)
+        ]
+        urlpatterns = [
+            path('test/', include(nested_patterns), {'foo': 'bar'}),
+        ]
+        self._test_included_urls(urlpatterns)
+
+    @unittest.skipUnless(path, 'needs Django 2')
+    def test_included_urls_django2_mixed(self):
+        nested_patterns = [
+            path('path', dummy_view)
+        ]
+        urlpatterns = [
+            url('^test/', include(nested_patterns), {'foo': 'bar'}),
+        ]
+        self._test_included_urls(urlpatterns)
+
+    @unittest.skipUnless(path, 'needs Django 2')
+    def test_included_urls_django2_mixed_args(self):
+        nested_patterns = [
+            path('path/<int:child>', dummy_view),
+            url('^url/(?P<child>[0-9]+)$', dummy_view)
+        ]
+        urlpatterns = [
+            url('^purl/(?P<parent>[0-9]+)/', include(nested_patterns), {'foo': 'bar'}),
+            path('ppath/<int:parent>/', include(nested_patterns), {'foo': 'bar'}),
+        ]
         test_paths = [
-            URLTestPath('/test/path', (), {'foo': 'bar', }),
-            URLTestPath('/test/path.api', (), {'foo': 'bar', 'format': 'api'}),
-            URLTestPath('/test/path.asdf', (), {'foo': 'bar', 'format': 'asdf'}),
+            # parent url() nesting child path()
+            URLTestPath('/purl/87/path/42', (), {'parent': '87', 'child': 42, 'foo': 'bar', }),
+            URLTestPath('/purl/87/path/42.api', (), {'parent': '87', 'child': 42, 'foo': 'bar', 'format': 'api'}),
+            URLTestPath('/purl/87/path/42.asdf', (), {'parent': '87', 'child': 42, 'foo': 'bar', 'format': 'asdf'}),
+
+            # parent path() nesting child url()
+            URLTestPath('/ppath/87/url/42', (), {'parent': 87, 'child': '42', 'foo': 'bar', }),
+            URLTestPath('/ppath/87/url/42.api', (), {'parent': 87, 'child': '42', 'foo': 'bar', 'format': 'api'}),
+            URLTestPath('/ppath/87/url/42.asdf', (), {'parent': 87, 'child': '42', 'foo': 'bar', 'format': 'asdf'}),
+
+            # parent path() nesting child path()
+            URLTestPath('/ppath/87/path/42', (), {'parent': 87, 'child': 42, 'foo': 'bar', }),
+            URLTestPath('/ppath/87/path/42.api', (), {'parent': 87, 'child': 42, 'foo': 'bar', 'format': 'api'}),
+            URLTestPath('/ppath/87/path/42.asdf', (), {'parent': 87, 'child': 42, 'foo': 'bar', 'format': 'asdf'}),
+
+            # parent url() nesting child url()
+            URLTestPath('/purl/87/url/42', (), {'parent': '87', 'child': '42', 'foo': 'bar', }),
+            URLTestPath('/purl/87/url/42.api', (), {'parent': '87', 'child': '42', 'foo': 'bar', 'format': 'api'}),
+            URLTestPath('/purl/87/url/42.asdf', (), {'parent': '87', 'child': '42', 'foo': 'bar', 'format': 'asdf'}),
         ]
         self._resolve_urlpatterns(urlpatterns, test_paths)
+
+    def _test_allowed_formats(self, urlpatterns):
+        allowed_formats = ['good', 'ugly']
+        test_paths = [
+            (URLTestPath('/test.good/', (), {'format': 'good'}), True),
+            (URLTestPath('/test.bad', (), {}), False),
+            (URLTestPath('/test.ugly', (), {'format': 'ugly'}), True),
+        ]
+        self._resolve_urlpatterns(urlpatterns, test_paths, allowed=allowed_formats)
+
+    def test_allowed_formats(self):
+        urlpatterns = [
+            url('^test$', dummy_view),
+        ]
+        self._test_allowed_formats(urlpatterns)
+
+    @unittest.skipUnless(path, 'needs Django 2')
+    def test_allowed_formats_django2(self):
+        urlpatterns = [
+            path('test', dummy_view),
+        ]
+        self._test_allowed_formats(urlpatterns)