"""Canonical accent-folding primitive for diacritic-insensitive text matching.
Nine call sites across ``core``, ``application``, ``domain``, and ``adapters``
independently hand-rolled the same NFKD-decompose-and-strip-combining-marks
routine (label matching, denylist matching, dedup fingerprinting, header
aliasing, terminology search, corpus-text normalisation). This module gives
the shared inner step one canonical home; each caller keeps composing its own
trailing transform (whitespace collapse, non-alphanumeric squeeze, HTML-tag
strip, casing) on top of it.
Where two callers compose the SAME trailing transform, that composition earns a
name here rather than being written twice: :func:`fold_for_matching` is the
fold-collapse-casefold pipeline the terminology search and the AEAT marker
matcher had each written out.
"""
from __future__ import annotations
import re
import unicodedata
__all__ = [
"COMBINING_MARK_RANGES",
"COMBINING_MARK_UNIDATA_VERSION",
"ascii_slug",
"combining_mark_ranges_from_unicodedata",
"fold_diacritics",
"fold_for_matching",
"fold_printed_phrase",
"unicode_compose",
]
#: A run of one or more whitespace characters, collapsed to a single space by
#: :func:`fold_for_matching`. Distinct from a single-whitespace pattern used to
#: DELETE whitespace, which is a different operation and is why the deleting
#: caller in the PDF label reader names its pattern for what it does.
_WHITESPACE_RUN_RE = re.compile(r"\s+")
#: Unicode database version :data:`COMBINING_MARK_RANGES` was derived from.
COMBINING_MARK_UNIDATA_VERSION = "15.1.0"
#: Inclusive codepoint ranges of Unicode category ``Mn`` (Mark, nonspacing) as
#: of :data:`COMBINING_MARK_UNIDATA_VERSION`. Checked in because deriving them
#: at import scans all 0x110000 codepoints through ``unicodedata.category``,
#: which dominated the import of every process that folds text. The text-fold
#: tests re-derive the ranges from the live database and print the literal to
#: paste when it moves.
COMBINING_MARK_RANGES: tuple[tuple[int, int], ...] = (
(0x0300, 0x036F),
(0x0483, 0x0487),
(0x0591, 0x05BD),
(0x05BF, 0x05BF),
(0x05C1, 0x05C2),
(0x05C4, 0x05C5),
(0x05C7, 0x05C7),
(0x0610, 0x061A),
(0x064B, 0x065F),
(0x0670, 0x0670),
(0x06D6, 0x06DC),
(0x06DF, 0x06E4),
(0x06E7, 0x06E8),
(0x06EA, 0x06ED),
(0x0711, 0x0711),
(0x0730, 0x074A),
(0x07A6, 0x07B0),
(0x07EB, 0x07F3),
(0x07FD, 0x07FD),
(0x0816, 0x0819),
(0x081B, 0x0823),
(0x0825, 0x0827),
(0x0829, 0x082D),
(0x0859, 0x085B),
(0x0898, 0x089F),
(0x08CA, 0x08E1),
(0x08E3, 0x0902),
(0x093A, 0x093A),
(0x093C, 0x093C),
(0x0941, 0x0948),
(0x094D, 0x094D),
(0x0951, 0x0957),
(0x0962, 0x0963),
(0x0981, 0x0981),
(0x09BC, 0x09BC),
(0x09C1, 0x09C4),
(0x09CD, 0x09CD),
(0x09E2, 0x09E3),
(0x09FE, 0x09FE),
(0x0A01, 0x0A02),
(0x0A3C, 0x0A3C),
(0x0A41, 0x0A42),
(0x0A47, 0x0A48),
(0x0A4B, 0x0A4D),
(0x0A51, 0x0A51),
(0x0A70, 0x0A71),
(0x0A75, 0x0A75),
(0x0A81, 0x0A82),
(0x0ABC, 0x0ABC),
(0x0AC1, 0x0AC5),
(0x0AC7, 0x0AC8),
(0x0ACD, 0x0ACD),
(0x0AE2, 0x0AE3),
(0x0AFA, 0x0AFF),
(0x0B01, 0x0B01),
(0x0B3C, 0x0B3C),
(0x0B3F, 0x0B3F),
(0x0B41, 0x0B44),
(0x0B4D, 0x0B4D),
(0x0B55, 0x0B56),
(0x0B62, 0x0B63),
(0x0B82, 0x0B82),
(0x0BC0, 0x0BC0),
(0x0BCD, 0x0BCD),
(0x0C00, 0x0C00),
(0x0C04, 0x0C04),
(0x0C3C, 0x0C3C),
(0x0C3E, 0x0C40),
(0x0C46, 0x0C48),
(0x0C4A, 0x0C4D),
(0x0C55, 0x0C56),
(0x0C62, 0x0C63),
(0x0C81, 0x0C81),
(0x0CBC, 0x0CBC),
(0x0CBF, 0x0CBF),
(0x0CC6, 0x0CC6),
(0x0CCC, 0x0CCD),
(0x0CE2, 0x0CE3),
(0x0D00, 0x0D01),
(0x0D3B, 0x0D3C),
(0x0D41, 0x0D44),
(0x0D4D, 0x0D4D),
(0x0D62, 0x0D63),
(0x0D81, 0x0D81),
(0x0DCA, 0x0DCA),
(0x0DD2, 0x0DD4),
(0x0DD6, 0x0DD6),
(0x0E31, 0x0E31),
(0x0E34, 0x0E3A),
(0x0E47, 0x0E4E),
(0x0EB1, 0x0EB1),
(0x0EB4, 0x0EBC),
(0x0EC8, 0x0ECE),
(0x0F18, 0x0F19),
(0x0F35, 0x0F35),
(0x0F37, 0x0F37),
(0x0F39, 0x0F39),
(0x0F71, 0x0F7E),
(0x0F80, 0x0F84),
(0x0F86, 0x0F87),
(0x0F8D, 0x0F97),
(0x0F99, 0x0FBC),
(0x0FC6, 0x0FC6),
(0x102D, 0x1030),
(0x1032, 0x1037),
(0x1039, 0x103A),
(0x103D, 0x103E),
(0x1058, 0x1059),
(0x105E, 0x1060),
(0x1071, 0x1074),
(0x1082, 0x1082),
(0x1085, 0x1086),
(0x108D, 0x108D),
(0x109D, 0x109D),
(0x135D, 0x135F),
(0x1712, 0x1714),
(0x1732, 0x1733),
(0x1752, 0x1753),
(0x1772, 0x1773),
(0x17B4, 0x17B5),
(0x17B7, 0x17BD),
(0x17C6, 0x17C6),
(0x17C9, 0x17D3),
(0x17DD, 0x17DD),
(0x180B, 0x180D),
(0x180F, 0x180F),
(0x1885, 0x1886),
(0x18A9, 0x18A9),
(0x1920, 0x1922),
(0x1927, 0x1928),
(0x1932, 0x1932),
(0x1939, 0x193B),
(0x1A17, 0x1A18),
(0x1A1B, 0x1A1B),
(0x1A56, 0x1A56),
(0x1A58, 0x1A5E),
(0x1A60, 0x1A60),
(0x1A62, 0x1A62),
(0x1A65, 0x1A6C),
(0x1A73, 0x1A7C),
(0x1A7F, 0x1A7F),
(0x1AB0, 0x1ABD),
(0x1ABF, 0x1ACE),
(0x1B00, 0x1B03),
(0x1B34, 0x1B34),
(0x1B36, 0x1B3A),
(0x1B3C, 0x1B3C),
(0x1B42, 0x1B42),
(0x1B6B, 0x1B73),
(0x1B80, 0x1B81),
(0x1BA2, 0x1BA5),
(0x1BA8, 0x1BA9),
(0x1BAB, 0x1BAD),
(0x1BE6, 0x1BE6),
(0x1BE8, 0x1BE9),
(0x1BED, 0x1BED),
(0x1BEF, 0x1BF1),
(0x1C2C, 0x1C33),
(0x1C36, 0x1C37),
(0x1CD0, 0x1CD2),
(0x1CD4, 0x1CE0),
(0x1CE2, 0x1CE8),
(0x1CED, 0x1CED),
(0x1CF4, 0x1CF4),
(0x1CF8, 0x1CF9),
(0x1DC0, 0x1DFF),
(0x20D0, 0x20DC),
(0x20E1, 0x20E1),
(0x20E5, 0x20F0),
(0x2CEF, 0x2CF1),
(0x2D7F, 0x2D7F),
(0x2DE0, 0x2DFF),
(0x302A, 0x302D),
(0x3099, 0x309A),
(0xA66F, 0xA66F),
(0xA674, 0xA67D),
(0xA69E, 0xA69F),
(0xA6F0, 0xA6F1),
(0xA802, 0xA802),
(0xA806, 0xA806),
(0xA80B, 0xA80B),
(0xA825, 0xA826),
(0xA82C, 0xA82C),
(0xA8C4, 0xA8C5),
(0xA8E0, 0xA8F1),
(0xA8FF, 0xA8FF),
(0xA926, 0xA92D),
(0xA947, 0xA951),
(0xA980, 0xA982),
(0xA9B3, 0xA9B3),
(0xA9B6, 0xA9B9),
(0xA9BC, 0xA9BD),
(0xA9E5, 0xA9E5),
(0xAA29, 0xAA2E),
(0xAA31, 0xAA32),
(0xAA35, 0xAA36),
(0xAA43, 0xAA43),
(0xAA4C, 0xAA4C),
(0xAA7C, 0xAA7C),
(0xAAB0, 0xAAB0),
(0xAAB2, 0xAAB4),
(0xAAB7, 0xAAB8),
(0xAABE, 0xAABF),
(0xAAC1, 0xAAC1),
(0xAAEC, 0xAAED),
(0xAAF6, 0xAAF6),
(0xABE5, 0xABE5),
(0xABE8, 0xABE8),
(0xABED, 0xABED),
(0xFB1E, 0xFB1E),
(0xFE00, 0xFE0F),
(0xFE20, 0xFE2F),
(0x101FD, 0x101FD),
(0x102E0, 0x102E0),
(0x10376, 0x1037A),
(0x10A01, 0x10A03),
(0x10A05, 0x10A06),
(0x10A0C, 0x10A0F),
(0x10A38, 0x10A3A),
(0x10A3F, 0x10A3F),
(0x10AE5, 0x10AE6),
(0x10D24, 0x10D27),
(0x10EAB, 0x10EAC),
(0x10EFD, 0x10EFF),
(0x10F46, 0x10F50),
(0x10F82, 0x10F85),
(0x11001, 0x11001),
(0x11038, 0x11046),
(0x11070, 0x11070),
(0x11073, 0x11074),
(0x1107F, 0x11081),
(0x110B3, 0x110B6),
(0x110B9, 0x110BA),
(0x110C2, 0x110C2),
(0x11100, 0x11102),
(0x11127, 0x1112B),
(0x1112D, 0x11134),
(0x11173, 0x11173),
(0x11180, 0x11181),
(0x111B6, 0x111BE),
(0x111C9, 0x111CC),
(0x111CF, 0x111CF),
(0x1122F, 0x11231),
(0x11234, 0x11234),
(0x11236, 0x11237),
(0x1123E, 0x1123E),
(0x11241, 0x11241),
(0x112DF, 0x112DF),
(0x112E3, 0x112EA),
(0x11300, 0x11301),
(0x1133B, 0x1133C),
(0x11340, 0x11340),
(0x11366, 0x1136C),
(0x11370, 0x11374),
(0x11438, 0x1143F),
(0x11442, 0x11444),
(0x11446, 0x11446),
(0x1145E, 0x1145E),
(0x114B3, 0x114B8),
(0x114BA, 0x114BA),
(0x114BF, 0x114C0),
(0x114C2, 0x114C3),
(0x115B2, 0x115B5),
(0x115BC, 0x115BD),
(0x115BF, 0x115C0),
(0x115DC, 0x115DD),
(0x11633, 0x1163A),
(0x1163D, 0x1163D),
(0x1163F, 0x11640),
(0x116AB, 0x116AB),
(0x116AD, 0x116AD),
(0x116B0, 0x116B5),
(0x116B7, 0x116B7),
(0x1171D, 0x1171F),
(0x11722, 0x11725),
(0x11727, 0x1172B),
(0x1182F, 0x11837),
(0x11839, 0x1183A),
(0x1193B, 0x1193C),
(0x1193E, 0x1193E),
(0x11943, 0x11943),
(0x119D4, 0x119D7),
(0x119DA, 0x119DB),
(0x119E0, 0x119E0),
(0x11A01, 0x11A0A),
(0x11A33, 0x11A38),
(0x11A3B, 0x11A3E),
(0x11A47, 0x11A47),
(0x11A51, 0x11A56),
(0x11A59, 0x11A5B),
(0x11A8A, 0x11A96),
(0x11A98, 0x11A99),
(0x11C30, 0x11C36),
(0x11C38, 0x11C3D),
(0x11C3F, 0x11C3F),
(0x11C92, 0x11CA7),
(0x11CAA, 0x11CB0),
(0x11CB2, 0x11CB3),
(0x11CB5, 0x11CB6),
(0x11D31, 0x11D36),
(0x11D3A, 0x11D3A),
(0x11D3C, 0x11D3D),
(0x11D3F, 0x11D45),
(0x11D47, 0x11D47),
(0x11D90, 0x11D91),
(0x11D95, 0x11D95),
(0x11D97, 0x11D97),
(0x11EF3, 0x11EF4),
(0x11F00, 0x11F01),
(0x11F36, 0x11F3A),
(0x11F40, 0x11F40),
(0x11F42, 0x11F42),
(0x13440, 0x13440),
(0x13447, 0x13455),
(0x16AF0, 0x16AF4),
(0x16B30, 0x16B36),
(0x16F4F, 0x16F4F),
(0x16F8F, 0x16F92),
(0x16FE4, 0x16FE4),
(0x1BC9D, 0x1BC9E),
(0x1CF00, 0x1CF2D),
(0x1CF30, 0x1CF46),
(0x1D167, 0x1D169),
(0x1D17B, 0x1D182),
(0x1D185, 0x1D18B),
(0x1D1AA, 0x1D1AD),
(0x1D242, 0x1D244),
(0x1DA00, 0x1DA36),
(0x1DA3B, 0x1DA6C),
(0x1DA75, 0x1DA75),
(0x1DA84, 0x1DA84),
(0x1DA9B, 0x1DA9F),
(0x1DAA1, 0x1DAAF),
(0x1E000, 0x1E006),
(0x1E008, 0x1E018),
(0x1E01B, 0x1E021),
(0x1E023, 0x1E024),
(0x1E026, 0x1E02A),
(0x1E08F, 0x1E08F),
(0x1E130, 0x1E136),
(0x1E2AE, 0x1E2AE),
(0x1E2EC, 0x1E2EF),
(0x1E4EC, 0x1E4EF),
(0x1E8D0, 0x1E8D6),
(0x1E944, 0x1E94A),
(0xE0100, 0xE01EF),
)
[docs]
def combining_mark_ranges_from_unicodedata() -> tuple[tuple[int, int], ...]:
"""Derive the inclusive ``Mn`` codepoint ranges from the running interpreter."""
ranges: list[tuple[int, int]] = []
for codepoint in range(0x110000):
if unicodedata.category(chr(codepoint)) != "Mn":
continue
if ranges and ranges[-1][1] == codepoint - 1:
ranges[-1] = (ranges[-1][0], codepoint)
else:
ranges.append((codepoint, codepoint))
return tuple(ranges)
#: Every ``Mn`` codepoint mapped to ``None`` for :meth:`str.translate`, so
#: folding costs one C-level lookup per character instead of one
#: ``unicodedata.category`` call. An interpreter shipping a different Unicode
#: database derives the table live rather than folding with a stale mark set.
_COMBINING_MARK_STRIP_TABLE: dict[int, None] = {
codepoint: None
for first, last in (
COMBINING_MARK_RANGES
if unicodedata.unidata_version == COMBINING_MARK_UNIDATA_VERSION
else combining_mark_ranges_from_unicodedata()
)
for codepoint in range(first, last + 1)
}
[docs]
def fold_diacritics(text: str) -> str:
"""Return *text* with combining diacritical marks removed.
NFKD-decomposes *text*, then drops every character in Unicode category
``Mn`` (Mark, nonspacing) — the combining accents NFKD decomposition
exposes (``"ó"`` -> ``"o"`` + U+0301 COMBINING ACUTE ACCENT, which this
function then drops). Every other codepoint passes through unchanged,
including one with no ASCII-compatible decomposition (an em dash, the
euro sign, ``"ø"``): this function folds ACCENTS, it does not
transliterate to ASCII. A caller that also needs to discard such
codepoints composes its own ``str.encode("ascii", "ignore")`` pass
afterward rather than relying on this function to do it.
Case is left untouched. Callers that need case-insensitive matching
compose this with their own :meth:`str.casefold` (preferred over
:meth:`str.lower` for caseless comparison) call, since the diacritic
fold and the case fold are orthogonal concerns.
Args:
text: The text to fold.
Returns:
*text* with every combining diacritical mark removed.
"""
return unicodedata.normalize("NFKD", text).translate(_COMBINING_MARK_STRIP_TABLE)
[docs]
def fold_printed_phrase(printed: str) -> str:
"""Return the form a printed multi-word phrase is matched under.
Three transforms and no more, in this order: fold case, fold accents,
collapse every run of whitespace to one space and trim the ends. Each earns
its place against how a phrase arrives from a document rather than from a
keyboard -- typography is the issuer's choice, a phrase set as a line wraps
with the break inside it, and a text layer or an OCR pass routinely returns
the ASCII spelling of an accented word.
**Punctuation is deliberately NOT stripped**, which is the transform a
fourth one would be. A vocabulary can carry a name whose stops are part of
it, and squeezing punctuation generally starts matching strings that are not
phrases at all.
Case folds BEFORE accents, and the order is load-bearing rather than
incidental: a caller that folds accents first and cases afterward is a
different function, because :meth:`str.casefold` can itself emit a combining
mark that the earlier accent fold is no longer there to remove.
Args:
printed: The phrase as it was printed.
Returns:
The normalised form to match on.
"""
return " ".join(fold_diacritics(printed.casefold()).split())
[docs]
def unicode_compose(text: str) -> str:
"""Return *text* NFKC-composed, every diacritic and printed character kept.
A DIFFERENT normalization form from :func:`fold_diacritics`, for a
different job: two documents printing the same figure can differ in
whether a composed character (``"ó"``) arrived pre-composed or as a base
letter plus a combining accent, with no difference in what was printed.
NFKC folds that representational gap closed without touching accents,
case, digits, or punctuation — unlike :func:`fold_diacritics`, this must
never be used where the accent itself is evidence to preserve.
Args:
text: The text to compose.
Returns:
*text* in NFKC normal form.
"""
return unicodedata.normalize("NFKC", text)
[docs]
def fold_for_matching(text: str) -> str:
"""Fold *text* to the form two matchers compare against.
Drops diacritics, collapses every run of whitespace to one space, trims the
ends, and casefolds. This is the exact pipeline the terminology search and
the AEAT response-marker matcher had each spelled out.
The casefold runs last, which is equivalent to casefolding first: neither
``str.casefold`` nor the whitespace collapse affects what the other does.
That equivalence is what let the two callers, which ordered the steps
differently, converge on one function.
Args:
text: Raw text to fold.
Returns:
The folded, whitespace-collapsed, casefolded form.
"""
return _WHITESPACE_RUN_RE.sub(" ", fold_diacritics(text)).strip().casefold()
#: A run of characters that cannot appear in an ASCII slug, collapsed to a
#: single hyphen by :func:`ascii_slug`.
_SLUG_RUN_RE = re.compile(r"[^a-z0-9]+")
[docs]
def ascii_slug(text: str) -> str:
"""Return the lowercase ASCII slug of *text*, or ``""`` when it has none.
Folds accents, discards every remaining non-ASCII codepoint, casefolds,
then collapses each run of non-slug characters to a single hyphen and trims
the hyphens from both ends.
Returning the empty string rather than raising is deliberate: the callers
that need a slug disagree about what an absent one MEANS -- one is parsing
an HTML source and one is compiling a registry declaration, and each owes
its own typed error -- so the shared code computes and the caller judges.
Args:
text: Raw text to reduce to a slug.
Returns:
The slug, or the empty string when *text* carries no slug characters.
"""
decomposed = fold_diacritics(text).encode("ascii", "ignore").decode("ascii").casefold()
return _SLUG_RUN_RE.sub("-", decomposed).strip("-")