Upload files to "core"
This commit is contained in:
66
core/profanity.py
Normal file
66
core/profanity.py
Normal file
@@ -0,0 +1,66 @@
|
||||
import re
|
||||
import unicodedata
|
||||
from better_profanity import profanity as _bp
|
||||
|
||||
_bp.load_censor_words()
|
||||
|
||||
_EXTRA_WORDS = [
|
||||
'kys', 'kms', 'jewishdoc', 'fck', 'fk',
|
||||
]
|
||||
_bp.add_censor_words(_EXTRA_WORDS)
|
||||
|
||||
_HATE_WORDS = {'kys', 'kms'}
|
||||
|
||||
_HATE_PATTERNS = [
|
||||
r"n[i1!|][gq9][gq9][e3]r",
|
||||
r"n[i1!|][gq9][gq9][ae3]",
|
||||
r"n[i1!|][gq9][gq9][ae3][sz]",
|
||||
r"f[a4][gq9][gq9][o0]t",
|
||||
r"f[a4][gq9][gq9][o0]t[sz]",
|
||||
r"f[a4][gq9]",
|
||||
r"ch[i1][nk][gq][sz]?",
|
||||
r"g[o0][o0]k[sz]?",
|
||||
r"sp[i1][cks][sz]?",
|
||||
r"w[e3]tb[a4][cks][gq][sz]?",
|
||||
r"k[i1][ck][e3][sz]?",
|
||||
r"j[e3]w[sz]?\b",
|
||||
r"h[e3][e3]b[sz]?",
|
||||
r"cr[a4][cks][gq][e3]r[sz]?",
|
||||
r"j[i1][gq][a4]b[o0][o0]",
|
||||
r"t[o0]w[e3]lh[e3][a4]d",
|
||||
r"r[e3]t[a4]rd",
|
||||
r"tr[a4]nn[yi1][e3]?[sz]?",
|
||||
r"[sz]h[e3]m[a4][l1][e3]",
|
||||
]
|
||||
_HATE_RE = re.compile(
|
||||
"|".join(r"(?<!\w)(?:" + p + r")(?!\w)" for p in _HATE_PATTERNS),
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
def _normalize(text: str) -> str:
|
||||
normalized = unicodedata.normalize('NFKD', text)
|
||||
ascii_only = normalized.encode('ascii', 'ignore').decode('ascii')
|
||||
return ascii_only
|
||||
|
||||
def _squish(s: str) -> str:
|
||||
s = re.sub(r'[^a-zA-Z0-9]', '', s)
|
||||
s = re.sub(r'(.)\1{2,}', r'\1', s)
|
||||
return s
|
||||
|
||||
def is_clean(text: str) -> bool:
|
||||
for variant in (text, _normalize(text)):
|
||||
if _bp.contains_profanity(variant):
|
||||
return False
|
||||
squished = _squish(variant)
|
||||
if squished and _bp.contains_profanity(squished):
|
||||
return False
|
||||
return True
|
||||
|
||||
def is_hate_speech(text: str) -> bool:
|
||||
lower = text.lower()
|
||||
for word in _HATE_WORDS:
|
||||
if re.search(r'(?<!\w)' + re.escape(word) + r'(?!\w)', lower):
|
||||
return True
|
||||
if _HATE_RE.search(text):
|
||||
return True
|
||||
return False
|
||||
Reference in New Issue
Block a user