Upload files to "core"

This commit is contained in:
2026-07-24 15:38:19 +00:00
parent ff0783480a
commit 4abb065fac
5 changed files with 647 additions and 0 deletions

66
core/profanity.py Normal file
View File

@@ -0,0 +1,66 @@
import re
import unicodedata
from better_profanity import profanity as _bp
_bp.load_censor_words()
_EXTRA_WORDS = [
'kys', 'kms', 'jewishdoc', 'fck', 'fk',
]
_bp.add_censor_words(_EXTRA_WORDS)
_HATE_WORDS = {'kys', 'kms'}
_HATE_PATTERNS = [
r"n[i1!|][gq9][gq9][e3]r",
r"n[i1!|][gq9][gq9][ae3]",
r"n[i1!|][gq9][gq9][ae3][sz]",
r"f[a4][gq9][gq9][o0]t",
r"f[a4][gq9][gq9][o0]t[sz]",
r"f[a4][gq9]",
r"ch[i1][nk][gq][sz]?",
r"g[o0][o0]k[sz]?",
r"sp[i1][cks][sz]?",
r"w[e3]tb[a4][cks][gq][sz]?",
r"k[i1][ck][e3][sz]?",
r"j[e3]w[sz]?\b",
r"h[e3][e3]b[sz]?",
r"cr[a4][cks][gq][e3]r[sz]?",
r"j[i1][gq][a4]b[o0][o0]",
r"t[o0]w[e3]lh[e3][a4]d",
r"r[e3]t[a4]rd",
r"tr[a4]nn[yi1][e3]?[sz]?",
r"[sz]h[e3]m[a4][l1][e3]",
]
_HATE_RE = re.compile(
"|".join(r"(?<!\w)(?:" + p + r")(?!\w)" for p in _HATE_PATTERNS),
re.IGNORECASE,
)
def _normalize(text: str) -> str:
normalized = unicodedata.normalize('NFKD', text)
ascii_only = normalized.encode('ascii', 'ignore').decode('ascii')
return ascii_only
def _squish(s: str) -> str:
s = re.sub(r'[^a-zA-Z0-9]', '', s)
s = re.sub(r'(.)\1{2,}', r'\1', s)
return s
def is_clean(text: str) -> bool:
for variant in (text, _normalize(text)):
if _bp.contains_profanity(variant):
return False
squished = _squish(variant)
if squished and _bp.contains_profanity(squished):
return False
return True
def is_hate_speech(text: str) -> bool:
lower = text.lower()
for word in _HATE_WORDS:
if re.search(r'(?<!\w)' + re.escape(word) + r'(?!\w)', lower):
return True
if _HATE_RE.search(text):
return True
return False