Repository: byllyfish/precis_i18n

Issue:
Codepoint sets in unicode.py must be updated
These are hard-wired and must be cross-checked against the current unicode data files in https://www.unicode.org/Public/UNIDATA/extracted/DerivedJoiningType.txt among others.

Discussion:

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bf0853d..3b7a867 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,12 +29,14 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        python -m pip install flake8
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Install unicodedata2 (Python 3.7 only)
-      if: matrix.python-version == '3.7'
-      run: python -m pip install unicodedata2==12.0.0
+        python -m pip install --upgrade pip flake8
+        if [ "${{ matrix.python-version }}" = "3.7" ]; then
+          python -m pip install unicodedata2==12.0.0
+        elif [ "${{ matrix.python-version }}" = "3.8" ]; then
+          python -m pip install unicodedata2==14.0.0
+        elif [ "${{ matrix.python-version }}" = "3.9" ]; then
+          python -m pip install unicodedata2==10.0.0
+        fi
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
diff --git a/precis_i18n/unicode.py b/precis_i18n/unicode.py
index 4710ff7..c8c9b03 100644
--- a/precis_i18n/unicode.py
+++ b/precis_i18n/unicode.py
@@ -140,7 +140,7 @@ class UnicodeData:
         return None
 
 
-# http://www.unicode.org/Public/10.0.0/ucd/DerivedCoreProperties.txt
+# https://www.unicode.org/Public/UNIDATA/DerivedCoreProperties.txt
 # Derived Property: Default_Ignorable_Code_Point
 _DEFAULT_IGNORABLE = CodepointSet('''
 00AD
@@ -150,6 +150,7 @@ _DEFAULT_IGNORABLE = CodepointSet('''
 17B4..17B5
 180B..180D
 180E
+180F
 200B..200F
 202A..202E
 2060..2064
@@ -170,9 +171,9 @@ E0080..E00FF
 E0100..E01EF
 E01F0..E0FFF
 ''')
-assert len(_DEFAULT_IGNORABLE) == 4173
+assert len(_DEFAULT_IGNORABLE) == 4174
 
-# http://www.unicode.org/Public/10.0.0/ucd/extracted/DerivedJoiningType.txt
+# https://www.unicode.org/Public/UNIDATA/extracted/DerivedJoiningType.txt
 # Joining_Type=Dual_Joining
 _JOINTYPE_DUAL_JOINING = CodepointSet('''
 0620
@@ -211,15 +212,16 @@ _JOINTYPE_DUAL_JOINING = CodepointSet('''
 0860
 0862..0865
 0868
+0886
+0889..088D
 08A0..08A9
 08AF..08B0
-08B3..08B4
-08B6..08B8
-08BA..08BD
+08B3..08B8
+08BA..08C8
 1807
 1820..1842
 1843
-1844..1877
+1844..1878
 1887..18A8
 18AA
 A840..A871
@@ -235,11 +237,26 @@ A840..A871
 10B8D
 10B90
 10BAD..10BAE
+10D01..10D21
+10D23
+10F30..10F32
+10F34..10F44
+10F51..10F53
+10F70..10F73
+10F76..10F81
+10FB0
+10FB2..10FB3
+10FB8
+10FBB..10FBC
+10FBE..10FBF
+10FC1
+10FC4
+10FCA
 1E900..1E943
 ''')
-assert len(_JOINTYPE_DUAL_JOINING) == 507
+assert len(_JOINTYPE_DUAL_JOINING) == 610
 
-# http://www.unicode.org/Public/10.0.0/ucd/extracted/DerivedJoiningType.txt
+# https://www.unicode.org/Public/UNIDATA/extracted/DerivedJoiningType.txt
 # Joining_Type=Right_Joining
 _JOINTYPE_RIGHT_JOINING = CodepointSet('''
 0622..0625
@@ -274,8 +291,11 @@ _JOINTYPE_RIGHT_JOINING = CodepointSet('''
 0846..0847
 0849
 0854
+0856..0858
 0867
 0869..086A
+0870..0882
+088E
 08AA..08AC
 08AE
 08B1..08B2
@@ -295,19 +315,30 @@ _JOINTYPE_RIGHT_JOINING = CodepointSet('''
 10B8E..10B8F
 10B91
 10BA9..10BAC
+10D22
+10F33
+10F54
+10F74..10F75
+10FB4..10FB6
+10FB9..10FBA
+10FBD
+10FC2..10FC3
+10FC9
 ''')
-assert len(_JOINTYPE_RIGHT_JOINING) == 115
+assert len(_JOINTYPE_RIGHT_JOINING) == 152
 
-# http://www.unicode.org/Public/10.0.0/ucd/extracted/DerivedJoiningType.txt
+# https://www.unicode.org/Public/UNIDATA/extracted/DerivedJoiningType.txt
 # Joining_Type=Left_Joining
 _JOINTYPE_LEFT_JOINING = CodepointSet('''
 A872
 10ACD
 10AD7
+10D00
+10FCB
 ''')
-assert len(_JOINTYPE_LEFT_JOINING) == 3
+assert len(_JOINTYPE_LEFT_JOINING) == 5
 
-# http://www.unicode.org/Public/10.0.0/ucd/extracted/DerivedJoiningType.txt
+# https://www.unicode.org/Public/UNIDATA/extracted/DerivedJoiningType.txt
 # Joining_Type=Transparent
 _JOINTYPE_TRANSPARENT = CodepointSet('''
 00AD
@@ -332,12 +363,14 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 0730..074A
 07A6..07B0
 07EB..07F3
+07FD
 0816..0819
 081B..0823
 0825..0827
 0829..082D
 0859..085B
-08D4..08E1
+0898..089F
+08CA..08E1
 08E3..0902
 093A
 093C
@@ -350,6 +383,7 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 09C1..09C4
 09CD
 09E2..09E3
+09FE
 0A01..0A02
 0A3C
 0A41..0A42
@@ -370,12 +404,14 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 0B3F
 0B41..0B44
 0B4D
-0B56
+0B55..0B56
 0B62..0B63
 0B82
 0BC0
 0BCD
 0C00
+0C04
+0C3C
 0C3E..0C40
 0C46..0C48
 0C4A..0C4D
@@ -392,6 +428,7 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 0D41..0D44
 0D4D
 0D62..0D63
+0D81
 0DCA
 0DD2..0DD4
 0DD6
@@ -399,8 +436,7 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 0E34..0E3A
 0E47..0E4E
 0EB1
-0EB4..0EB9
-0EBB..0EBC
+0EB4..0EBC
 0EC8..0ECD
 0F18..0F19
 0F35
@@ -425,7 +461,7 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 109D
 135D..135F
 1712..1714
-1732..1734
+1732..1733
 1752..1753
 1772..1773
 17B4..17B5
@@ -434,6 +470,7 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 17C9..17D3
 17DD
 180B..180D
+180F
 1885..1886
 18A9
 1920..1922
@@ -451,6 +488,7 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 1A7F
 1AB0..1ABD
 1ABE
+1ABF..1ACE
 1B00..1B03
 1B34
 1B36..1B3A
@@ -473,8 +511,7 @@ _JOINTYPE_TRANSPARENT = CodepointSet('''
 1CED
 1CF4
 1CF8..1CF9
-1DC0..1DF9
-1DFB..1DFF
+1DC0..1DFF
 200B
 200E..200F
 202A..202E
@@ -499,14 +536,16 @@ A802
 A806
 A80B
 A825..A826
+A82C
 A8C4..A8C5
 A8E0..A8F1
+A8FF
 A926..A92D
 A947..A951
 A980..A982
 A9B3
 A9B6..A9B9
-A9BC
+A9BC..A9BD
 A9E5
 AA29..AA2E
 AA31..AA32
@@ -538,19 +577,26 @@ FFF9..FFFB
 10A38..10A3A
 10A3F
 10AE5..10AE6
+10D24..10D27
+10EAB..10EAC
+10F46..10F50
+10F82..10F85
 11001
 11038..11046
+11070
+11073..11074
 1107F..11081
 110B3..110B6
 110B9..110BA
-110BD
+110C2
 11100..11102
 11127..1112B
 1112D..11134
 11173
 11180..11181
 111B6..111BE
-111CA..111CC
+111C9..111CC
+111CF
 1122F..11231
 11234
 11236..11237
@@ -558,13 +604,14 @@ FFF9..FFFB
 112DF
 112E3..112EA
 11300..11301
-1133C
+1133B..1133C
 11340
 11366..1136C
 11370..11374
 11438..1143F
 11442..11444
 11446
+1145E
 114B3..114B8
 114BA
 114BF..114C0
@@ -583,8 +630,15 @@ FFF9..FFFB
 1171D..1171F
 11722..11725
 11727..1172B
-11A01..11A06
-11A09..11A0A
+1182F..11837
+11839..1183A
+1193B..1193C
+1193E
+11943
+119D4..119D7
+119DA..119DB
+119E0
+11A01..11A0A
 11A33..11A38
 11A3B..11A3E
 11A47
@@ -604,11 +658,20 @@ FFF9..FFFB
 11D3C..11D3D
 11D3F..11D45
 11D47
+11D90..11D91
+11D95
+11D97
+11EF3..11EF4
+13430..13438
 16AF0..16AF4
 16B30..16B36
+16F4F
 16F8F..16F92
+16FE4
 1BC9D..1BC9E
 1BCA0..1BCA3
+1CF00..1CF2D
+1CF30..1CF46
 1D167..1D169
 1D173..1D17A
 1D17B..1D182
@@ -626,15 +689,19 @@ FFF9..FFFB
 1E01B..1E021
 1E023..1E024
 1E026..1E02A
+1E130..1E136
+1E2AE
+1E2EC..1E2EF
 1E8D0..1E8D6
 1E944..1E94A
+1E94B
 E0001
 E0020..E007F
 E0100..E01EF
 ''')
-assert len(_JOINTYPE_TRANSPARENT) == 1912
+assert len(_JOINTYPE_TRANSPARENT) == 2108
 
-# http://www.unicode.org/Public/10.0.0/ucd/Scripts.txt
+# https://www.unicode.org/Public/UNIDATA/Scripts.txt
 # Greek
 _GREEK_SCRIPT = CodepointSet('''
 0370..0373
@@ -695,7 +762,7 @@ AB65
 ''')
 assert len(_GREEK_SCRIPT) == 518
 
-# http://www.unicode.org/Public/10.0.0/ucd/Scripts.txt
+# https://www.unicode.org/Public/UNIDATA/Scripts.txt
 # Hebrew
 _HEBREW_SCRIPT = CodepointSet('''
 0591..05BD
@@ -708,7 +775,7 @@ _HEBREW_SCRIPT = CodepointSet('''
 05C6
 05C7
 05D0..05EA
-05F0..05F2
+05EF..05F2
 05F3..05F4
 FB1D
 FB1E
@@ -721,18 +788,19 @@ FB40..FB41
 FB43..FB44
 FB46..FB4F
 ''')
-assert len(_HEBREW_SCRIPT) == 133
+assert len(_HEBREW_SCRIPT) == 134
 
-# http://www.unicode.org/Public/10.0.0/ucd/Scripts.txt
+# https://www.unicode.org/Public/UNIDATA/Scripts.txt
 # Hiragana, Katakana, Han
 _HIRAGANA_KATAKANA_HAN = CodepointSet('''
-# Hiragana (376)
+# Hiragana (380)
 3041..3096
 309D..309E
 309F
-1B001..1B11E
+1B001..1B11F
+1B150..1B152
 1F200
-# Katakana (300)
+# Katakana (320)
 30A1..30FA
 30FD..30FE
 30FF
@@ -741,8 +809,13 @@ _HIRAGANA_KATAKANA_HAN = CodepointSet('''
 3300..3357
 FF66..FF6F
 FF71..FF9D
+1AFF0..1AFF3
+1AFF5..1AFFB
+1AFFD..1AFFE
 1B000
-# Han (89228)
+1B120..1B122
+1B164..1B167
+# Han (94215)
 2E80..2E99
 2E9B..2EF3
 2F00..2FD5
@@ -751,20 +824,24 @@ FF71..FF9D
 3021..3029
 3038..303A
 303B
-3400..4DB5
-4E00..9FEA
+3400..4DBF
+4E00..9FFF
 F900..FA6D
 FA70..FAD9
-20000..2A6D6
-2A700..2B734
+16FE2
+16FE3
+16FF0..16FF1
+20000..2A6DF
+2A700..2B738
 2B740..2B81D
 2B820..2CEA1
 2CEB0..2EBE0
 2F800..2FA1D
+30000..3134A
 ''')
-assert len(_HIRAGANA_KATAKANA_HAN) == (376 + 300 + 89228)
+assert len(_HIRAGANA_KATAKANA_HAN) == (380 + 320 + 94215)
 
-# http://www.unicode.org/Public/10.0.0/ucd/HangulSyllableType.txt
+# https://www.unicode.org/Public/UNIDATA/HangulSyllableType.txt
 # Leading_Jamo, Vowel_Jamo, Trailing_Jamo
 _OLD_HANGUL_JAMO = CodepointSet('''
 # Leading_Jamo (125)
diff --git a/tools/check_codepoints.py b/tools/check_codepoints.py
new file mode 100644
index 0000000..a1cead7
--- /dev/null
+++ b/tools/check_codepoints.py
@@ -0,0 +1,68 @@
+"""
+Program to check tables included in precis_i18n.unicode module.
+"""
+
+import re
+from urllib.request import urlopen
+
+import precis_i18n.unicode as ucd
+from precis_i18n.codepointset import CodepointSet
+
+PROP_REGEX = re.compile(rb"^([0-9A-Za-z.]+)\s+;\s+(\w+)\s+#")
+DATAFILE_URLS = [
+    "https://www.unicode.org/Public/UNIDATA/DerivedCoreProperties.txt",
+    "https://www.unicode.org/Public/UNIDATA/extracted/DerivedJoiningType.txt#Join_Type",
+    "https://www.unicode.org/Public/UNIDATA/Scripts.txt",
+    'https://www.unicode.org/Public/UNIDATA/HangulSyllableType.txt#Hangul_Type',
+]
+
+def parse_unicode_datafile(url, props):
+    if '#' in url:
+        url, prop = url.split("#")
+        prop += '='
+    else:
+        prop = ''
+
+    for line in urlopen(url):
+        m = PROP_REGEX.match(line)
+        if m:
+            codepoints, name = m.groups()
+            props.setdefault('%s%s' % (prop, name.decode()), []).append(codepoints.decode())
+
+
+def compare_codepoints(varname, codepoint_str):
+    codepoints = CodepointSet(codepoint_str)
+    if codepoints != getattr(ucd, varname):
+        print(varname)
+        print(codepoint_str)
+
+
+def combine(props, *names):
+    result = ''
+    for name in names:
+        cnt = len(CodepointSet(props[name]))
+        result += '# %s (%d)\n%s\n' % (name, cnt, props[name])
+    return result
+
+
+def main():
+    props = {}
+    for url in DATAFILE_URLS:
+        parse_unicode_datafile(url, props)
+
+    for prop in props:
+        props[prop] = '\n'.join(props[prop])
+
+    compare_codepoints('_DEFAULT_IGNORABLE', props['Default_Ignorable_Code_Point'])
+    compare_codepoints('_JOINTYPE_DUAL_JOINING', props['Join_Type=D'])
+    compare_codepoints('_JOINTYPE_RIGHT_JOINING', props['Join_Type=R'])
+    compare_codepoints('_JOINTYPE_LEFT_JOINING', props['Join_Type=L'])
+    compare_codepoints('_JOINTYPE_TRANSPARENT', props['Join_Type=T'])
+    compare_codepoints('_GREEK_SCRIPT', props['Greek'])
+    compare_codepoints('_HEBREW_SCRIPT', props['Hebrew'])
+    compare_codepoints('_HIRAGANA_KATAKANA_HAN', combine(props, 'Hiragana', 'Katakana', 'Han'))
+    compare_codepoints('_OLD_HANGUL_JAMO', combine(props, 'Hangul_Type=L', 'Hangul_Type=V', 'Hangul_Type=T'))
+
+
+if __name__ == '__main__':
+    main()


Test patch:
diff --git a/test/derived-props-14.0.txt b/test/derived-props-14.0.txt
index abb9605..ef84c51 100644
--- a/test/derived-props-14.0.txt
+++ b/test/derived-props-14.0.txt
@@ -658,8 +658,8 @@
 17F0-17F9 FREE_PVAL/other_letter_digits
 17FA-17FF UNASSIGNED/unassigned
 1800-180A FREE_PVAL/punctuation
-180B-180E DISALLOWED/precis_ignorable_properties
-180F-1819 PVALID/letter_digits
+180B-180F DISALLOWED/precis_ignorable_properties
+1810-1819 PVALID/letter_digits
 181A-181F UNASSIGNED/unassigned
 1820-1878 PVALID/letter_digits
 1879-187F UNASSIGNED/unassigned
diff --git a/test/golden.json b/test/golden.json
index 6b81618..de6f312 100644
--- a/test/golden.json
+++ b/test/golden.json
@@ -1795,6 +1795,40 @@
     "error": null,
     "unicode_version": 11.0
   },
+  {
+    "profile": "UsernameCasePreserved",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "UsernameCasePreserved",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "UsernameCasePreserved",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "UsernameCasePreserved",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "UsernameCasePreserved",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "UsernameCaseMapped",
     "input": "",
@@ -3592,6 +3626,40 @@
     "error": null,
     "unicode_version": 11.0
   },
+  {
+    "profile": "UsernameCaseMapped",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "UsernameCaseMapped",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "UsernameCaseMapped",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "UsernameCaseMapped",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "UsernameCaseMapped",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "OpaqueString",
     "input": "",
@@ -5388,6 +5456,40 @@
     "error": null,
     "unicode_version": 8.0
   },
+  {
+    "profile": "OpaqueString",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "OpaqueString",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "OpaqueString",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "OpaqueString",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "OpaqueString",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "NicknameCaseMapped",
     "input": "",
@@ -7185,6 +7287,40 @@
     "error": null,
     "unicode_version": 8.0
   },
+  {
+    "profile": "NicknameCaseMapped",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "NicknameCaseMapped",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "NicknameCaseMapped",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "NicknameCaseMapped",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "NicknameCaseMapped",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "UsernameCaseMapped:ToLower",
     "input": "",
@@ -8982,6 +9118,40 @@
     "error": null,
     "unicode_version": 11.0
   },
+  {
+    "profile": "UsernameCaseMapped:ToLower",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "UsernameCaseMapped:ToLower",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "UsernameCaseMapped:ToLower",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "UsernameCaseMapped:ToLower",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "UsernameCaseMapped:ToLower",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "NicknameCasePreserved",
     "input": "",
@@ -10778,6 +10948,40 @@
     "error": null,
     "unicode_version": 8.0
   },
+  {
+    "profile": "NicknameCasePreserved",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "NicknameCasePreserved",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "NicknameCasePreserved",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "NicknameCasePreserved",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "NicknameCasePreserved",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "NicknameCaseMapped:ToLower",
     "input": "",
@@ -12575,6 +12779,40 @@
     "error": null,
     "unicode_version": 8.0
   },
+  {
+    "profile": "NicknameCaseMapped:ToLower",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "NicknameCaseMapped:ToLower",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "NicknameCaseMapped:ToLower",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "NicknameCaseMapped:ToLower",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "NicknameCaseMapped:ToLower",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "FreeFormClass",
     "input": "",
@@ -14371,6 +14609,40 @@
     "error": null,
     "unicode_version": 8.0
   },
+  {
+    "profile": "FreeFormClass",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "FreeFormClass",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "FreeFormClass",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "FreeFormClass",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "FreeFormClass",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "IdentifierClass",
     "input": "",
@@ -16167,6 +16439,40 @@
     "error": null,
     "unicode_version": 11.0
   },
+  {
+    "profile": "IdentifierClass",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "IdentifierClass",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "IdentifierClass",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "IdentifierClass",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "IdentifierClass",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "UsernameCaseMapped:CaseFold",
     "input": "",
@@ -17963,6 +18269,40 @@
     "error": null,
     "unicode_version": 11.0
   },
+  {
+    "profile": "UsernameCaseMapped:CaseFold",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "UsernameCaseMapped:CaseFold",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "UsernameCaseMapped:CaseFold",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "UsernameCaseMapped:CaseFold",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "UsernameCaseMapped:CaseFold",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
+  },
   {
     "profile": "NicknameCaseMapped:CaseFold",
     "input": "",
@@ -19758,5 +20098,39 @@
     "output": "\ud804\uddc9",
     "error": null,
     "unicode_version": 8.0
+  },
+  {
+    "profile": "NicknameCaseMapped:CaseFold",
+    "input": "\u05ef\u05f3",
+    "output": "\u05ef\u05f3",
+    "error": null,
+    "unicode_version": 11.0
+  },
+  {
+    "profile": "NicknameCaseMapped:CaseFold",
+    "input": "\u30fb\u3400",
+    "output": "\u30fb\u3400",
+    "error": null
+  },
+  {
+    "profile": "NicknameCaseMapped:CaseFold",
+    "input": "\ud803\udd00\u200c\u088e",
+    "output": "\ud803\udd00\u200c\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "NicknameCaseMapped:CaseFold",
+    "input": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "output": "\ud803\udd00\u07fd\u200c\u07fd\u088e",
+    "error": null,
+    "unicode_version": 14.0
+  },
+  {
+    "profile": "NicknameCaseMapped:CaseFold",
+    "input": "\u0886\u200c\u0886",
+    "output": "\u0886\u200c\u0886",
+    "error": null,
+    "unicode_version": 14.0
   }
 ]
diff --git a/test/golden_source.txt b/test/golden_source.txt
index a5591a0..46f9dd3 100644
--- a/test/golden_source.txt
+++ b/test/golden_source.txt
@@ -415,3 +415,15 @@ t\u00FCrk\u0131ye
 
 \U000111c9
 
+# Codepoint Table updates (Issue #19)
+
+# rule_hebrew_punctuation
+\u05EF\u05f3
+# katakana_middle_dot
+\u30fb\u3400
+# valid_join_type: L J R
+\U00010D00\u200c\u088E
+# valid_join_type: L T J T R
+\U00010D00\u07fd\u200c\u07fd\u088E
+# valid_join_type: D J D
+\u0886\u200c\u0886
diff --git a/test/make_golden.py b/test/make_golden.py
index 9d17824..579099a 100644
--- a/test/make_golden.py
+++ b/test/make_golden.py
@@ -20,16 +20,26 @@ PROFILES = [
     'NicknameCaseMapped:CaseFold'
 ]
 
+_ANY_PROFILE = re.compile(r'.')
+
 EXCEPTIONS = {
     # ToLower difference before Unicode 8.0. The lower case characters weren't
     # added until Unicode 8.0.
     '\u13da\u13a2\u13b5\u13ac\u13a2\u13ac\u13d2':
     [(re.compile(r'.+CaseMapped(:ToLower)?$'), 8.0)],
     # U+1AB6 was introduced in 7.0.
-    '\u05d0\u1ab6\u05d1': [(re.compile(r'.'), 7.0)],
+    '\u05d0\u1ab6\u05d1': [(_ANY_PROFILE, 7.0)],
     # U+0111C9 changed to PVALID in 11.0. It was introduced in 8.0.
     '\U000111c9': [(re.compile(r'^(?:Username.*|IdentifierClass)$'), 11.0),
-                   (re.compile(r'.'), 8.0)],
+                   (_ANY_PROFILE, 8.0)],
+
+    # U+05EF was introduced in 11.0.
+    '\u05EF\u05f3': [(_ANY_PROFILE, 11.0)],
+
+    # U+088E, U+0886 were introduced in 14.0.
+    '\U00010D00\u200c\u088E': [(_ANY_PROFILE, 14.0)],
+    '\U00010D00\u07fd\u200c\u07fd\u088E': [(_ANY_PROFILE, 14.0)],
+    '\u0886\u200c\u0886': [(_ANY_PROFILE, 14.0)],
 }
 
 
diff --git a/test/test_derived_props.py b/test/test_derived_props.py
index 476a799..091efe5 100644
--- a/test/test_derived_props.py
+++ b/test/test_derived_props.py
@@ -27,7 +27,7 @@ class TestDerivedProperties(unittest.TestCase):
         """
 
         ucd = UnicodeData(unicodedata2)
-        assert ucd.version == 12.0
+        assert ucd.version in  {10.0, 12.0, 14.0}
         self._test_derived_props(ucd)
 
     def _test_derived_props(self, ucd):
diff --git a/test/test_precis.py b/test/test_precis.py
index a276a78..6921533 100644
--- a/test/test_precis.py
+++ b/test/test_precis.py
@@ -310,6 +310,8 @@ class TestPrecisContextRule(unittest.TestCase):
         self.assertFalse(_context_rule('\u0031\u05f4', 1, UCD))
         # Invalid: undefined after
         self.assertFalse(_context_rule('\u05f3', 0, UCD))
+        # Valid: 05EF 05f3 (Unicode >= 11.0)
+        self.assertTrue(_context_rule('\u05EF\u05f3', 1, UCD))
 
         # 6. katakana_middle_dot
         # Valid: 0x30fb 0x2e99
@@ -322,6 +324,8 @@ class TestPrecisContextRule(unittest.TestCase):
         self.assertFalse(_context_rule('\u0021\u30fb', 1, UCD))
         # Invalid: 0x30fb
         self.assertFalse(_context_rule('\u30fb', 0, UCD))
+        # Valid: 0x30fb 0x3400 (Unicode >= X)
+        self.assertTrue(_context_rule('\u30fb\u3400', 0, UCD))
 
         # 7. arabic_indic
         # Valid: 0x0660 0x0661 0x0662 0x0669
@@ -398,6 +402,7 @@ class TestPrecisUnicodeData(unittest.TestCase):
             self.assertTrue(UCD.combining_virama(0x1715))
         else:
             self.assertFalse(UCD.combining_virama(0x1715))
+        self.assertFalse(UCD.combining_virama(0x1716))
 
     def test_arabic_indic(self):
         self.assertTrue(UCD.arabic_indic(0x669))
@@ -432,6 +437,14 @@ class TestPrecisUnicodeData(unittest.TestCase):
         self.assertFalse(
             UCD.valid_jointype('\u0031\u0300\u200c\u0301\u0032', 2))
 
+        # Valid: L J R   (Unicode >= 14.0)
+        self.assertTrue(UCD.valid_jointype('\U00010D00\u200c\u088E', 1))
+        # Valid: L T J T R  (Unicode >= 14.0)
+        self.assertTrue(UCD.valid_jointype('\U00010D00\u07fd\u200c\u07fd\u088E', 2))
+        # Valid: D J D  (Unicode >= 14.0)
+        self.assertTrue(UCD.valid_jointype('\u0886\u200c\u0886', 1))
+
+
     def test_version_to_float(self):
         self.assertEqual(_version_to_float('8.0.0'), 8.0)
         self.assertEqual(_version_to_float('6.3.1'), 6.3)