Repository: facelessuser/wcmatch

Issue:
POSIX property miscomunication
Current POSIX style properties (e.g. `[:alnum:]`) are currently documented as providing Unicode patterns in Unicode strings and ASCII in byte strings, but this is not actually what is happening. Both are locked to the C Locale. The only difference is that they cover the character range of Unicode or ASCII. What this means is:

- In both byte strings and Unicode strings, `[:alnum:]` matches `[0-9a-zA-Z]`.
- When doing something like `[:^alnum:]` in Unicode, you'll get all characters that are not `[0-9a-zA-Z]` spanning all the Unicode characters, but in a byte string, you'll only have characters in the range `\x00-\xff` that are not alphanumeric.

This is a configuration error on our part. We should have configured Unicode strings with `POSIX_UNICODE` and not `POSIX` for Unicode strings.

Additionally, we should probably add a flag to force ASCII logic in a Unicode string.

Discussion:
facelessuser: @gir-bot lgtm

Patch:
diff --git a/docs/src/markdown/about/changelog.md b/docs/src/markdown/about/changelog.md
index 1b7b352..9e344b4 100644
--- a/docs/src/markdown/about/changelog.md
+++ b/docs/src/markdown/about/changelog.md
@@ -8,6 +8,9 @@
   base path of a file which is relative to the root directory and the actual file name.
 - **NEW**: Internal attribute of `WcMatch` changed from `base` to `_root_dir`. This attribute is not really meant to be
   referenced by users and as been marked as private.
+- **NEW**: Drop requirement for `backrefs` and update documentation to note that POSIX properties never actually enabled
+  the use of Unicode properties. While the documentation stated this and it was probably intended, it was never actually
+  enabled. Currently, Wildcard match has chosen to keep with the ASCII definition for now.
 
 ## 7.2
 
diff --git a/docs/src/markdown/fnmatch.md b/docs/src/markdown/fnmatch.md
index 0e6e5da..0d24445 100644
--- a/docs/src/markdown/fnmatch.md
+++ b/docs/src/markdown/fnmatch.md
@@ -21,7 +21,7 @@ Pattern           | Meaning
 `?`               | Matches any single character.
 `[seq]`           | Matches any character in seq.
 `[!seq]`          | Matches any character not in seq. Will also accept character exclusions in the form of `[^seq]`.
-`[[:alnum:]]`     | POSIX style character classes inside sequences. The `C` locale is used for byte strings and Unicode properties for Unicode strings. See [POSIX Character Classes](#posix-character-classes) for more info.
+`[[:alnum:]]`     | POSIX style character classes inside sequences. See [POSIX Character Classes](#posix-character-classes) for more info.
 `\`               | Escapes characters. If applied to a meta character, it will be treated as a normal character.
 `!`               | When used at the start of a pattern, the pattern will be an exclusion pattern. Requires the [`NEGATE`](#negate) flag. If also using the [`MINUSNEGATE`](#minusnegate) flag, `-` will be used instead of `!`.
 `?(pattern_list)` | The pattern matches if zero or one occurrences of any of the patterns in the `pattern_list` match the input string. Requires the [`EXTMATCH`](#extmatch) flag.
diff --git a/docs/src/markdown/glob.md b/docs/src/markdown/glob.md
index 4058bcf..89d05ae 100644
--- a/docs/src/markdown/glob.md
+++ b/docs/src/markdown/glob.md
@@ -22,7 +22,7 @@ Pattern           | Meaning
 `?`               | Matches any single character.
 `[seq]`           | Matches any character in seq.
 `[!seq]`          | Matches any character not in seq. Will also accept character exclusions in the form of `[^seq]`.
-`[[:alnum:]]`     | POSIX style character classes inside sequences.  The `C` locale is used for byte strings and Unicode properties for Unicode strings. See [POSIX Character Classes](#posix-character-classes) for more info.
+`[[:alnum:]]`     | POSIX style character classes inside sequences. See [POSIX Character Classes](#posix-character-classes) for more info.
 `\`               | Escapes characters. If applied to a meta character, it will be treated as a normal character.
 `!`               | When used at the start of a pattern, the pattern will be an exclusion pattern. Requires the [`NEGATE`](#negate) flag. If also using the [`MINUSNEGATE`](#minusnegate) flag, `-` will be used instead of `!`.
 `?(pattern_list)` | The pattern matches if zero or one occurrences of any of the patterns in the `pattern_list` match the input string. Requires the [`EXTGLOB`](#extglob) flag.
diff --git a/requirements/setup.txt b/requirements/setup.txt
index 076ab9c..fe6a629 100644
--- a/requirements/setup.txt
+++ b/requirements/setup.txt
@@ -1,2 +1,1 @@
 bracex>=2.0
-backrefs>=4.1
diff --git a/wcmatch/_wcparse.py b/wcmatch/_wcparse.py
index 2470b9f..23bafe8 100644
--- a/wcmatch/_wcparse.py
+++ b/wcmatch/_wcparse.py
@@ -27,11 +27,14 @@ import bracex
 import os
 from collections import namedtuple
 from . import util
-from backrefs import uniprops
+from . import posix
 
 UNICODE = 0
 BYTES = 1
 
+UNICODE_RANGE = '\u0000-\U0010ffff'
+ASCII_RANGE = '\x00-\xff'
+
 PATTERN_LIMIT = 1000
 
 RE_WIN_DRIVE_START = re.compile(r'((?:\\\\|/){2}((?:\\[^\\/]|[^\\/])+)|([\\]?[a-z][\\]?:))((?:\\\\|/)|$)', re.I)
@@ -1100,8 +1103,7 @@ class WcParse(object):
             # is the end of a range.
             if end_range and i.index - 1 >= end_range:
                 result[-1] = '\\' + result[-1]
-            posix_type = uniprops.POSIX_BYTES if self.is_bytes else uniprops.POSIX
-            result.append(uniprops.get_posix_property(m.group(1), posix_type))
+            result.append(posix.get_posix_property(m.group(1), self.is_bytes))
         return last_posix
 
     def _sequence(self, i):
@@ -1196,12 +1198,12 @@ class WcParse(object):
             if value == '[]':
                 # We specified some ranges, but they are all
                 # out of reach.  Create an impossible sequence to match.
-                result = ['[^%s]' % ('\x00-\xff' if self.is_bytes else uniprops.UNICODE_RANGE)]
+                result = ['[^%s]' % (ASCII_RANGE if self.is_bytes else UNICODE_RANGE)]
             elif value == '[^]':
                 # We specified some range, but hey are all
                 # out of reach. Since this is exclusive
                 # that means we can match *anything*.
-                result = ['[%s]' % ('\x00-\xff' if self.is_bytes else uniprops.UNICODE_RANGE)]
+                result = ['[%s]' % (ASCII_RANGE if self.is_bytes else UNICODE_RANGE)]
             else:
                 result = [value]
 
diff --git a/wcmatch/posix.py b/wcmatch/posix.py
new file mode 100644
index 0000000..9bae600
--- /dev/null
+++ b/wcmatch/posix.py
@@ -0,0 +1,70 @@
+"""Posix Properties."""
+unicode_posix_properties = {
+    "^alnum": "\x00-\x2f\x3a-\x40\x5c\x5b-\x60\x7b-\U0010ffff",
+    "^alpha": "\x00-\x40\x5a-\x60\x7a-\U0010ffff",
+    "^ascii": "\x80-\U0010ffff",
+    "^blank": "\x00-\x08\x0a-\x1f\x21-\U0010ffff",
+    "^cntrl": "\x20-\x5c\x7e\x80-\U0010ffff",
+    "^digit": "\x00-\x2f\x3a-\U0010ffff",
+    "^graph": "\x00-\x20\x7f-\U0010ffff",
+    "^lower": "\x00-\x60\x7b-\U0010ffff",
+    "^print": "\x00-\x1f\x7f-\U0010ffff",
+    "^punct": "\x00-\x20\x30-\x39\x41-\x5a\x61-\x7a\x7f-\U0010ffff",
+    "^space": "\x00-\x08\x0e-\x1f\x21-\U0010ffff",
+    "^upper": "\x00-\x40\x5c\x5b-\U0010ffff",
+    "^xdigit": "\x00-\x2f\x3a-\x40\x47-\x60\x67-\U0010ffff",
+    "alnum": "\x30-\x39\x41-\x5a\x61-\x7a",
+    "alpha": "\x41-\x59\x61-\x79",
+    "ascii": "\x00-\x7f",
+    "blank": "\x09\x20",
+    "cntrl": "\x00-\x1f\x7f",
+    "digit": "\x30-\x39",
+    "graph": "\x21-\x5c\x7e",
+    "lower": "\x61-\x7a",
+    "print": "\x20-\x5c\x7e",
+    "punct": "\x21-\x2f\x3a-\x40\x5c\x5b-\x60\x7b-\x5c\x7e",
+    "space": "\x09-\x0d\x20",
+    "upper": "\x41-\x5a",
+    "xdigit": "\x30-\x39\x41-\x46\x61-\x66"
+}
+
+ascii_posix_properties = {
+    "^alnum": "\x00-\x2f\x3a-\x40\x5c\x5b-\x60\x7b-\xff",
+    "^alpha": "\x00-\x40\x5a-\x60\x7a-\xff",
+    "^ascii": "\x80-\xff",
+    "^blank": "\x00-\x08\x0a-\x1f\x21-\xff",
+    "^cntrl": "\x20-\x5c\x7e\x80-\xff",
+    "^digit": "\x00-\x2f\x3a-\xff",
+    "^graph": "\x00-\x20\x7f-\xff",
+    "^lower": "\x00-\x60\x7b-\xff",
+    "^print": "\x00-\x1f\x7f-\xff",
+    "^punct": "\x00-\x20\x30-\x39\x41-\x5a\x61-\x7a\x7f-\xff",
+    "^space": "\x00-\x08\x0e-\x1f\x21-\xff",
+    "^upper": "\x00-\x40\x5c\x5b-\xff",
+    "^xdigit": "\x00-\x2f\x3a-\x40\x47-\x60\x67-\xff",
+    "alnum": "\x30-\x39\x41-\x5a\x61-\x7a",
+    "alpha": "\x41-\x59\x61-\x79",
+    "ascii": "\x00-\x7f",
+    "blank": "\x09\x20",
+    "cntrl": "\x00-\x1f\x7f",
+    "digit": "\x30-\x39",
+    "graph": "\x21-\x5c\x7e",
+    "lower": "\x61-\x7a",
+    "print": "\x20-\x5c\x7e",
+    "punct": "\x21-\x2f\x3a-\x40\x5c\x5b-\x60\x7b-\x5c\x7e",
+    "space": "\x09-\x0d\x20",
+    "upper": "\x41-\x5a",
+    "xdigit": "\x30-\x39\x41-\x46\x61-\x66"
+}
+
+
+def get_posix_property(value, limit_ascii=False):
+    """Retrieve the POSIX category."""
+
+    try:
+        if limit_ascii:
+            return ascii_posix_properties[value]
+        else:
+            return unicode_posix_properties[value]
+    except Exception:  # pragma: no cover
+        raise ValueError("'{} is not a valid posix property".format(value))


Test patch:
diff --git a/tests/test_fnmatch.py b/tests/test_fnmatch.py
index a11ee29..5337bc2 100644
--- a/tests/test_fnmatch.py
+++ b/tests/test_fnmatch.py
@@ -152,6 +152,11 @@ class TestFnMatch:
         ['[a[:alnum:]]bc', 'zbc', True, 0],
         ['[[:alnum:][:blank:]]bc', ' bc', True, 0],
 
+        [b'[[:alnum:]]bc', b'zbc', True, 0],
+        [b'[[:alnum:]]bc', b'1bc', True, 0],
+        [b'[a[:alnum:]]bc', b'zbc', True, 0],
+        [b'[[:alnum:][:blank:]]bc', b' bc', True, 0],
+
         # POSIX character classes are case sensitive
         ['[[:ALNUM:]]bc', 'zbc', False, 0],
         ['[[:AlNuM:]]bc', '1bc', False, 0],