Repository: getnikola/nikola

Issue:
utils.slugify sometimes returns a str, sometimes a unicode
Debian Sid, Python 2.7.10.

```
>>> import nikola.utils
>>> a = nikola.utils.slugify(u'aaaaa')
>>> type(a)
<type 'str'>
>>> a = nikola.utils.slugify(u'aaa-aa')
>>> type(a)
<type 'unicode'>
```

It seems, that if `slugify` does not replace anything, it will return a `str` instead of an `unicode`. This breaks the wordpress import because it calls `slugify` twice:

```
        for tag in self.all_tags:
            try:
                tag_str = tag.decode('utf8')
            except AttributeError:
                tag_str = tag
            tag = utils.slugify(tag_str)
            src_url = '{}tag/{}'.format(self.context['SITE_URL'], tag)
            dst_url = self.site.link('tag', tag)
            if src_url != dst_url:
                self.url_map[src_url] = dst_url
```

(`site.link` calls `slugify` too)



Discussion:

Resolving patch:
diff --git a/nikola/utils.py b/nikola/utils.py
--- a/nikola/utils.py
+++ b/nikola/utils.py
@@ -728,9 +728,9 @@ def slugify(value, force=False):
     if USE_SLUGIFY or force:
         # This is the standard state of slugify, which actually does some work.
         # It is the preferred style, especially for Western languages.
-        value = unidecode(value)
-        value = str(_slugify_strip_re.sub('', value).strip().lower())
-        return _slugify_hyphenate_re.sub('-', value)
+        value = unicode_str(unidecode(value))
+        value = _slugify_strip_re.sub('', value, re.UNICODE).strip().lower()
+        return _slugify_hyphenate_re.sub('-', value, re.UNICODE)
     else:
         # This is the “disarmed” state of slugify, which lets the user
         # have any character they please (be it regular ASCII with spaces,
@@ -740,7 +740,7 @@ def slugify(value, force=False):
         # We still replace some characters, though.  In particular, we need
         # to replace ? and #, which should not appear in URLs, and some
         # Windows-unsafe characters.  This list might be even longer.
-        rc = '/\\?#"\'\r\n\t*:<>|"'
+        rc = '/\\?#"\'\r\n\t*:<>|'
 
         for c in rc:
             value = value.replace(c, '-')


Test patch:
diff --git a/tests/test_slugify.py b/tests/test_slugify.py
new file mode 100644
--- /dev/null
+++ b/tests/test_slugify.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+
+u"""Test slugify."""
+
+from __future__ import unicode_literals
+import nikola.utils
+
+
+def test_ascii():
+    """Test an ASCII-only string."""
+    o = nikola.utils.slugify(u'hello')
+    assert o == u'hello'
+    assert isinstance(o, nikola.utils.unicode_str)
+
+
+def test_ascii_dash():
+    """Test an ASCII string, with dashes."""
+    o = nikola.utils.slugify(u'hello-world')
+    assert o == u'hello-world'
+    assert isinstance(o, nikola.utils.unicode_str)
+
+
+def test_ascii_fancy():
+    """Test an ASCII string, with fancy characters."""
+    o = nikola.utils.slugify(u'The quick brown fox jumps over the lazy dog!-123.456')
+    assert o == u'the-quick-brown-fox-jumps-over-the-lazy-dog-123456'
+    assert isinstance(o, nikola.utils.unicode_str)
+
+
+def test_pl():
+    """Test a string with Polish diacritical characters."""
+    o = nikola.utils.slugify(u'zażółćgęśląjaźń')
+    assert o == u'zazolcgeslajazn'
+    assert isinstance(o, nikola.utils.unicode_str)
+
+
+def test_pl_dash():
+    """Test a string with Polish diacritical characters and dashes."""
+    o = nikola.utils.slugify(u'zażółć-gęślą-jaźń')
+    assert o == u'zazolc-gesla-jazn'
+
+
+def test_pl_fancy():
+    """Test a string with Polish diacritical characters and fancy characters."""
+    o = nikola.utils.slugify(u'Zażółć gęślą jaźń!-123.456')
+    assert o == u'zazolc-gesla-jazn-123456'
+    assert isinstance(o, nikola.utils.unicode_str)
+
+
+def test_disarmed():
+    """Test disarmed slugify."""
+    nikola.utils.USE_SLUGIFY = False
+    o = nikola.utils.slugify(u'Zażółć gęślą jaźń!-123.456')
+    assert o == u'Zażółć gęślą jaźń!-123.456'
+    assert isinstance(o, nikola.utils.unicode_str)
+    nikola.utils.USE_SLUGIFY = True
+
+
+def test_disarmed_weird():
+    """Test disarmed slugify with banned characters."""
+    nikola.utils.USE_SLUGIFY = False
+    o = nikola.utils.slugify(u'Zażółć gęślą jaźń!-123.456 "Hello World"?#H<e>l/l\\o:W\'o\rr*l\td|!\n')
+    assert o == u'Zażółć gęślą jaźń!-123.456 -Hello World---H-e-l-l-o-W-o-r-l-d-!-'
+    assert isinstance(o, nikola.utils.unicode_str)
+    nikola.utils.USE_SLUGIFY = True