183 lines
7.0 KiB
Python
183 lines
7.0 KiB
Python
|
# Fixed tools.py - all string inputs normalized, no Kodi imports
|
|||
|
|
import hashlib, re, os, pyaes, base64
|
|||
|
|
from urllib.parse import quote, unquote, quote_plus, unquote_plus, urlparse
|
|||
|
|
from html.entities import name2codepoint
|
|||
|
|
from difflib import SequenceMatcher
|
|||
|
|
from functools import lru_cache
|
|||
|
|
|
|||
|
|
def _e(s):
|
|||
|
|
if s is None: return ''
|
|||
|
|
if isinstance(s, bytes): return s.decode('utf-8')
|
|||
|
|
if not isinstance(s, str): return str(s)
|
|||
|
|
return s
|
|||
|
|
|
|||
|
|
class cParser:
|
|||
|
|
@staticmethod
|
|||
|
|
def _get_compiled_pattern(pattern, flags=0):
|
|||
|
|
return re.compile(pattern, flags)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _replaceSpecialCharacters(s):
|
|||
|
|
try:
|
|||
|
|
for t in (('\\/', '/'), ('&', '&'), ('\\u00c4', 'Ä'), ('\\u00e4', 'ä'),
|
|||
|
|
('\\u00d6', 'Ö'), ('\\u00f6', 'ö'), ('\\u00dc', 'Ü'), ('\\u00fc', 'ü'),
|
|||
|
|
('\\u00df', 'ß'), ('\\u2013', '-'), ('\\u00b2', '²'), ('\\u00b3', '³'),
|
|||
|
|
('\\u00e9', 'é'), ('\\u2018', "'"), ('\\u201e', '"'), ('\\u201c', '"'),
|
|||
|
|
('\\u00c9', 'É'), ('\\u2026', '...'), ('\\u202f', 'h'), ('\\u2019', "'"),
|
|||
|
|
('\\u0308', '̈'), ('\\u00e8', 'è'), ('#038;', ''), ('\\u00f8', 'ø'),
|
|||
|
|
('／', '/'), ('\\u00e1', 'á'), ('–', '-'), ('“', '"'), ('„', '"'),
|
|||
|
|
('’', "'"), ('…', '…'), ('\\u00bc', '¼'), ('\\u00bd', '½'), ('\\u00be', '¾'),
|
|||
|
|
('\\u2153', '⅓'), ('\\u002A', '*')):
|
|||
|
|
s = s.replace(*t)
|
|||
|
|
for h in (('\\/', '/'), ('&', '&'), (''', "'"), ("'", "'"),
|
|||
|
|
('Ä', 'Ä'), ('ä', 'ä'), ('Ö', 'Ö'), ('ö', 'ö'),
|
|||
|
|
('Ü', 'Ü'), ('ü', 'ü'), ('ß', 'ß'), ('²', '²'),
|
|||
|
|
('Ü', '³'), ('¼', '¼'), ('½', '½'), ('¾', '¾'),
|
|||
|
|
('⅓', '⅓'), ('∗', '*')):
|
|||
|
|
s = s.replace(*h)
|
|||
|
|
except: pass
|
|||
|
|
return s
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def parseSingleResult(sHtmlContent, pattern, ignoreCase=False):
|
|||
|
|
if sHtmlContent:
|
|||
|
|
flags = re.S | re.M
|
|||
|
|
if ignoreCase: flags |= re.I
|
|||
|
|
matches = cParser._get_compiled_pattern(pattern, flags).search(sHtmlContent)
|
|||
|
|
if matches:
|
|||
|
|
if matches.lastindex is not None and matches.lastindex >= 1:
|
|||
|
|
return True, cParser._replaceSpecialCharacters(matches.group(1))
|
|||
|
|
else:
|
|||
|
|
return True, cParser._replaceSpecialCharacters(matches.group(0))
|
|||
|
|
return False, None
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def parse(sHtmlContent, pattern, iMinFoundValue=1, ignoreCase=False):
|
|||
|
|
if sHtmlContent:
|
|||
|
|
flags = re.DOTALL
|
|||
|
|
if ignoreCase: flags |= re.I
|
|||
|
|
aMatches = cParser._get_compiled_pattern(pattern, flags).findall(sHtmlContent)
|
|||
|
|
if len(aMatches) >= iMinFoundValue:
|
|||
|
|
if isinstance(aMatches[0], tuple):
|
|||
|
|
aMatches = [tuple(cParser._replaceSpecialCharacters(x) if isinstance(x, str) and x is not None else '' for x in match) for match in aMatches]
|
|||
|
|
else:
|
|||
|
|
aMatches = [cParser._replaceSpecialCharacters(x) if isinstance(x, str) and x is not None else '' for x in aMatches]
|
|||
|
|
return True, aMatches
|
|||
|
|
return False, None
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def replace(pattern, sReplaceString, sValue):
|
|||
|
|
return cParser._get_compiled_pattern(pattern).sub(sReplaceString, sValue)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def search(pattern, sValue, ignoreCase=True):
|
|||
|
|
flags = re.IGNORECASE if ignoreCase else 0
|
|||
|
|
return cParser._get_compiled_pattern(pattern, flags).search(sValue)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def escape(sValue):
|
|||
|
|
return re.escape(sValue)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def getNumberFromString(sValue):
|
|||
|
|
aMatches = re.compile(r'\d+').findall(_e(sValue))
|
|||
|
|
return int(aMatches[0]) if aMatches else 0
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def urlparse(sUrl):
|
|||
|
|
return urlparse(_e(sUrl).replace('www.', '')).netloc.title()
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def urlDecode(sUrl):
|
|||
|
|
return unquote(_e(sUrl))
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def urlEncode(sUrl, safe=''):
|
|||
|
|
return quote(_e(sUrl), safe)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def quote(sUrl):
|
|||
|
|
return quote(_e(sUrl))
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def unquotePlus(sUrl):
|
|||
|
|
return unquote_plus(_e(sUrl))
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def quotePlus(sUrl):
|
|||
|
|
return quote_plus(_e(sUrl))
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def B64decode(text):
|
|||
|
|
return base64.b64decode(_e(text)).decode('utf-8')
|
|||
|
|
|
|||
|
|
|
|||
|
|
class cUtil:
|
|||
|
|
@staticmethod
|
|||
|
|
def removeHtmlTags(sValue, sReplace=''):
|
|||
|
|
return re.compile(r'<.*?>').sub(sReplace, sValue)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def unescape(text):
|
|||
|
|
def fixup(m):
|
|||
|
|
text = m.group(0)
|
|||
|
|
if not text.endswith(';'): text += ';'
|
|||
|
|
if text[:2] == '&#':
|
|||
|
|
try:
|
|||
|
|
if text[:3] == '&#x': return chr(int(text[3:-1], 16))
|
|||
|
|
else: return chr(int(text[2:-1]))
|
|||
|
|
except ValueError: pass
|
|||
|
|
else:
|
|||
|
|
try: return chr(name2codepoint[text[1:-1]])
|
|||
|
|
except KeyError: pass
|
|||
|
|
return text
|
|||
|
|
t = _e(text)
|
|||
|
|
try: t = t.decode('utf-8')
|
|||
|
|
except: pass
|
|||
|
|
return re.compile(r'&(\w+;|#x?\d+;?)').sub(fixup, t.strip())
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def cleanse_text(text):
|
|||
|
|
if text is None: text = ''
|
|||
|
|
return cUtil.removeHtmlTags(text)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def evp_decode(cipher_text, passphrase, salt=None):
|
|||
|
|
ct = _e(cipher_text).encode('latin-1') if isinstance(cipher_text, str) else cipher_text
|
|||
|
|
pwd = _e(passphrase).encode('latin-1') if isinstance(passphrase, str) else passphrase
|
|||
|
|
if not salt: salt = ct[8:16]; ct = ct[16:]
|
|||
|
|
else: salt = _e(salt).encode('latin-1') if isinstance(salt, str) else salt
|
|||
|
|
key, iv = cUtil.evpKDF(pwd, salt)
|
|||
|
|
decrypter = pyaes.Decrypter(pyaes.AESModeOfOperationCBC(key, iv))
|
|||
|
|
plain_text = decrypter.feed(ct)
|
|||
|
|
plain_text += decrypter.feed()
|
|||
|
|
return plain_text.decode("utf-8")
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def evpKDF(pwd, salt, key_size=32, iv_size=16):
|
|||
|
|
temp = b''
|
|||
|
|
fd = b''
|
|||
|
|
while len(fd) < key_size + iv_size:
|
|||
|
|
h = hashlib.md5()
|
|||
|
|
h.update(temp + pwd + salt)
|
|||
|
|
temp = h.digest()
|
|||
|
|
fd += temp
|
|||
|
|
return fd[0:key_size], fd[key_size:key_size + iv_size]
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def isSimilar(sSearch, sText, threshold=0.9):
|
|||
|
|
return SequenceMatcher(None, _e(sSearch), _e(sText)).ratio() >= threshold
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
@lru_cache(maxsize=200000)
|
|||
|
|
def get_seq_match_ratio(token1, token2):
|
|||
|
|
return SequenceMatcher(None, _e(token1), _e(token2)).ratio()
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def isSimilarByToken(sSearch, sText, threshold=0.9):
|
|||
|
|
tokens_sSearch = _e(sSearch).split()
|
|||
|
|
tokens_sText = _e(sText).split()
|
|||
|
|
if not tokens_sSearch: return False
|
|||
|
|
best_ratios = [max(cUtil.get_seq_match_ratio(token, token2) for token2 in tokens_sText) for token in tokens_sSearch]
|
|||
|
|
return (sum(best_ratios) / len(best_ratios)) >= threshold
|