import re import unicodedata from better_profanity import profanity as _bp _bp.load_censor_words() _EXTRA_WORDS = [ 'kys', 'kms', 'jewishdoc', 'fck', 'fk', ] _bp.add_censor_words(_EXTRA_WORDS) _HATE_WORDS = {'kys', 'kms'} _HATE_PATTERNS = [ r"n[i1!|][gq9][gq9][e3]r", r"n[i1!|][gq9][gq9][ae3]", r"n[i1!|][gq9][gq9][ae3][sz]", r"f[a4][gq9][gq9][o0]t", r"f[a4][gq9][gq9][o0]t[sz]", r"f[a4][gq9]", r"ch[i1][nk][gq][sz]?", r"g[o0][o0]k[sz]?", r"sp[i1][cks][sz]?", r"w[e3]tb[a4][cks][gq][sz]?", r"k[i1][ck][e3][sz]?", r"j[e3]w[sz]?\b", r"h[e3][e3]b[sz]?", r"cr[a4][cks][gq][e3]r[sz]?", r"j[i1][gq][a4]b[o0][o0]", r"t[o0]w[e3]lh[e3][a4]d", r"r[e3]t[a4]rd", r"tr[a4]nn[yi1][e3]?[sz]?", r"[sz]h[e3]m[a4][l1][e3]", ] _HATE_RE = re.compile( "|".join(r"(? str: normalized = unicodedata.normalize('NFKD', text) ascii_only = normalized.encode('ascii', 'ignore').decode('ascii') return ascii_only def _squish(s: str) -> str: s = re.sub(r'[^a-zA-Z0-9]', '', s) s = re.sub(r'(.)\1{2,}', r'\1', s) return s def is_clean(text: str) -> bool: for variant in (text, _normalize(text)): if _bp.contains_profanity(variant): return False squished = _squish(variant) if squished and _bp.contains_profanity(squished): return False return True def is_hate_speech(text: str) -> bool: lower = text.lower() for word in _HATE_WORDS: if re.search(r'(?