166 lines
6.2 KiB
Python
166 lines
6.2 KiB
Python
|
"""Case-folding and whitespace normalization"""
|
||
|
|
# Unicode Case Folding table has been derived from the following work:
|
||
|
|
#
|
||
|
|
# CaseFolding-12.0.0.txt
|
||
|
|
# Date: 2019-01-22, 08:18:22 GMT
|
||
|
|
# (c) 2019 Unicode(R) Inc.
|
||
|
|
# Unicode and the Unicode Logo are registered trademarks
|
||
|
|
# of Unicode, Inc. in the U.S. and other countries.
|
||
|
|
# For terms of use, see http://www.unicode.org/terms_of_use.html
|
||
|
|
#
|
||
|
|
# Unicode Character Database
|
||
|
|
# For documentation, see http://www.unicode.org/reports/tr44/
|
||
|
|
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
from builtins import str, chr
|
||
|
|
|
||
|
|
__all__ = ["normalize_reference"]
|
||
|
|
|
||
|
|
if sys.version_info < (3,) and sys.maxunicode <= 0xffff:
|
||
|
|
# shim for Python 2.x UCS2 build
|
||
|
|
_unichr = chr
|
||
|
|
|
||
|
|
def chr(cdp):
|
||
|
|
if 0x10000 <= cdp < 0x110000:
|
||
|
|
cdp -= 0x10000
|
||
|
|
return (_unichr(0xd800 | (cdp >> 10)) +
|
||
|
|
_unichr(0xdc00 | (cdp & 0x3ff)))
|
||
|
|
return _unichr(cdp)
|
||
|
|
|
||
|
|
|
||
|
|
def _parse_table(tbl):
|
||
|
|
xlat = {}
|
||
|
|
cur_i, cur_j = -1, 0
|
||
|
|
for entry in tbl.split(';'):
|
||
|
|
arr = entry.split(',')
|
||
|
|
info = [int(x, 36) if x else 0 for x in arr[0].split(':')]
|
||
|
|
arr = [int(x, 36) for x in arr[1:]]
|
||
|
|
assert not any(x in xlat for x in arr)
|
||
|
|
sfx = ''.join(map(chr, arr))
|
||
|
|
streak, stride = 0, 1
|
||
|
|
if len(info) == 2:
|
||
|
|
fdt, delta = info
|
||
|
|
elif len(info) == 3:
|
||
|
|
fdt, streak, delta = info
|
||
|
|
else:
|
||
|
|
fdt, streak, delta, stride = info
|
||
|
|
assert streak >= 0 and stride >= 1
|
||
|
|
cur_i += fdt + 1
|
||
|
|
cur_j -= delta
|
||
|
|
assert cur_j != 0
|
||
|
|
i = cur_i
|
||
|
|
last = cur_i + streak
|
||
|
|
while i <= last:
|
||
|
|
# uniqueness and idempotency
|
||
|
|
assert i not in xlat and i + cur_j not in xlat
|
||
|
|
assert i not in arr
|
||
|
|
xlat[i] = chr(i + cur_j) + sfx
|
||
|
|
i += stride
|
||
|
|
return xlat
|
||
|
|
|
||
|
|
|
||
|
|
XLAT = _parse_table(
|
||
|
|
# ===== Start of Unicode Case Folding table =====
|
||
|
|
'1t:p:-w;37:-kn;a:m:kn;n:6:;6:3w,37;w:1a:-31:2;1b:5k,lj;1:4:-5k:2;6:e::'
|
||
|
|
'2;f:-aa,32;:18:aa:2;19:3e;:4:-3e:2;5:7h;1:-da;:2:5t:2;3:-5p;:5p;1:1:-5'
|
||
|
|
'o;1:5o;2:-26;:-3f;:-1;:5m;1:-5o;:-2;1:-4;:2;:5s;3:-5u;:-2;1:-1;:4:5x:2'
|
||
|
|
';5:-61;:61;1:-61;2:61;1:-61;:61;1:1:-60;1:2:60:2;3:-62;:4:62:4;b:-1;:1'
|
||
|
|
';1:-1;:1;1:-1;:g:1:2;i:g::2;h:av,lo;:-aw;:2:1:2;3:2q;:-15;:12:-1l:2;13'
|
||
|
|
':3n;1:g:-3n:2;n:-8bu;:8bu;1:4k;:-8gb;2:8br;1:5g;:-7c;:-2;:8:1y:2;72:-3'
|
||
|
|
'7;16:2:37:2;5:;8:-37;6:26;1:2:1;3:-r;1:1:1;1:m,lk,ld;:g:9;h:8:;c:b,lk,'
|
||
|
|
'ld;h:k;c:-7;:12;:-5;3:-a;:7;1:m:-n:2;n:1j;:-6;2:c;:4;1:-1t;1:8;:-8;2:2'
|
||
|
|
':3n;2:f:-5u;f:v:1c;27:w:v:2;15:1g::2;1h:-e;:c:e:2;e:2m::2;2o:11:-1b;2d'
|
||
|
|
':2a,136;26w:11:-5mq;12:6::6;mo:5:5m0;1on:4sm;:-1;:-9;:1:-2;1:1;:-7;:-o'
|
||
|
|
';:-vzb;7:16:tj7;18:2:;8y:44:-2bl:2;45:5yn,mp;:-b,lk;:-2,lm;:-1,lm;:p,j'
|
||
|
|
'i;:-5xb;2:5wx,37;1:2m:-5yk:2;2v:7:9;f:5:;f:7:;f:7:;f:5:;7:5fn,lv;1:2,l'
|
||
|
|
'v,lc;1:2,lv,ld;1:2,lv,n6;2:6:-5ft:2;e:7:;n:7:3c,qh;7:7:8,qh;7:7:-o,qh;'
|
||
|
|
'7:7:8,qh;7:7:-1k,qh;7:7:8,qh;9:-6,qh;:5hc,qh;:6,qh;1:-3,n6;:1,n6,qh;:1'
|
||
|
|
':-5j2;1:1:1u;1:5hd,qh;1:-6;3:-5h3,qh;:5ha,qh;:a,qh;1:-7,n6;:1,n6,qh;:3'
|
||
|
|
':-5h6;3:5hb,qh;5:4,lk,lc;:1,lk,ld;2:3,n6;:1,lk,n6;:1:-5jq;1:1:2k;7:5h5'
|
||
|
|
',lk,lc;:1,lk,ld;:5,lv;1:-2,n6;:1,lk,n6;:1:-5ju;1:1:2w;1:-2x;5:33,qh;:5'
|
||
|
|
'h0,qh;:-4,qh;1:7,n6;:1,n6,qh;:1:-5gu;1:1:-2;1:5h1,qh;89:8a;3:o2;:-3d;6'
|
||
|
|
':-6ea;19:f:c;y:f;mq:p:-p;1ft:1a:-m;2n:1b;1:8ag;:-5ch;:5c1;2:4:-8a0:2;5'
|
||
|
|
':8bh;:-v;:y;:-1;1:3:-8bj:3;b:1:8cg;1:2q:-8cg:2;2y:2::2;6:nym::nym;nyn:'
|
||
|
|
'16::2;1p:q::2;4h:c::2;f:1o::2;1y:2::2;3:r9h;:8:-r9h:2;c:;1:wmh;2:2:-wm'
|
||
|
|
'h:2;5:i::2;j:wn9;:b;:-4;:-a;:3;1:-1e;:o;:-l;:-xbp;:a:pr:2;d:;1:1d;:wlv'
|
||
|
|
';:-5cb;q1:27:2oo;fpr:jii,2u;:1,2x;:1,30;:1,2u,2x;:1,2u,30;:-c,38;:1,38'
|
||
|
|
';c:-z8,12u;:1,12d;:1,12j;:-9,12u;:b,12l;sp:p:-1cjn;ym:13:-8;4v:z:;1jj:'
|
||
|
|
'1e:-o;2e7:v:w;gwv:v:;o8v:x:-2'
|
||
|
|
# ===== End of Unicode Case Folding table =====
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _check_native(tbl):
|
||
|
|
"""
|
||
|
|
Determine if Python's own native implementation
|
||
|
|
subsumes the supplied case folding table
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
for i in tbl:
|
||
|
|
stv = chr(i)
|
||
|
|
if stv.casefold() == stv:
|
||
|
|
return False
|
||
|
|
except AttributeError:
|
||
|
|
return False
|
||
|
|
return True
|
||
|
|
|
||
|
|
|
||
|
|
# Hoist version check out of function for performance
|
||
|
|
SPACE_RE = re.compile(r'[ \t\r\n]+')
|
||
|
|
if _check_native(XLAT):
|
||
|
|
def normalize_reference(string):
|
||
|
|
"""
|
||
|
|
Normalize reference label: collapse internal whitespace
|
||
|
|
to single space, remove leading/trailing whitespace, case fold.
|
||
|
|
"""
|
||
|
|
return SPACE_RE.sub(' ', string[1:-1].strip()).casefold()
|
||
|
|
elif sys.version_info >= (3,) or sys.maxunicode > 0xffff:
|
||
|
|
def normalize_reference(string):
|
||
|
|
"""
|
||
|
|
Normalize reference label: collapse internal whitespace
|
||
|
|
to single space, remove leading/trailing whitespace, case fold.
|
||
|
|
"""
|
||
|
|
return SPACE_RE.sub(' ', string[1:-1].strip()).translate(XLAT)
|
||
|
|
else:
|
||
|
|
def _get_smp_regex():
|
||
|
|
xls = sorted(x - 0x10000 for x in XLAT if x >= 0x10000)
|
||
|
|
xls.append(-1)
|
||
|
|
fmt, (dsh, opn, pip, cse) = str('\\u%04x'), str('-[|]')
|
||
|
|
rga, srk, erk = [str(r'[ \t\r\n]+')], 0, -2
|
||
|
|
for k in xls:
|
||
|
|
new_hir = (erk ^ k) >> 10 != 0
|
||
|
|
if new_hir or erk + 1 != k:
|
||
|
|
if erk >= 0 and srk != erk:
|
||
|
|
if srk + 1 != erk:
|
||
|
|
rga.append(dsh)
|
||
|
|
rga.append(fmt % (0xdc00 + (erk & 0x3ff)))
|
||
|
|
if new_hir:
|
||
|
|
if erk >= 0:
|
||
|
|
rga.append(cse)
|
||
|
|
if k < 0:
|
||
|
|
break
|
||
|
|
rga.append(pip)
|
||
|
|
rga.append(fmt % (0xd800 + (k >> 10)))
|
||
|
|
rga.append(opn)
|
||
|
|
srk = k
|
||
|
|
rga.append(fmt % (0xdc00 + (srk & 0x3ff)))
|
||
|
|
erk = k
|
||
|
|
return re.compile(str().join(rga))
|
||
|
|
|
||
|
|
def _subst_handler(matchobj):
|
||
|
|
src = matchobj.group(0)
|
||
|
|
hiv = ord(src[0])
|
||
|
|
if hiv < 0xd800:
|
||
|
|
return ' '
|
||
|
|
return XLAT[0x10000 + ((hiv & 0x3ff) << 10) | (ord(src[1]) & 0x3ff)]
|
||
|
|
|
||
|
|
SMP_RE = _get_smp_regex()
|
||
|
|
|
||
|
|
def normalize_reference(string):
|
||
|
|
"""
|
||
|
|
Normalize reference label: collapse internal whitespace
|
||
|
|
to single space, remove leading/trailing whitespace, case fold.
|
||
|
|
"""
|
||
|
|
return SMP_RE.sub(_subst_handler, string[1:-1].strip()).translate(XLAT)
|