mirror of
https://github.com/JimmXinu/FanFicFare.git
synced 2025-12-24 01:33:39 +01:00
448 lines
13 KiB
Python
448 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
||
|
||
import re
|
||
|
||
def _unirepl(match):
|
||
"Return the unicode string for a decimal number"
|
||
if match.group(1)=='x':
|
||
radix=16
|
||
else:
|
||
radix=10
|
||
value = int(match.group(2), radix )
|
||
return unichr(value)
|
||
|
||
def _replaceNumberEntities(data):
|
||
p = re.compile(r'&#(x?)(\d+);')
|
||
return p.sub(_unirepl, data)
|
||
|
||
def _replaceNotEntities(data):
|
||
# not just \w or \S. regexp from c:\Python25\lib\sgmllib.py
|
||
# (or equiv), SGMLParser, entityref
|
||
p = re.compile(r'&([a-zA-Z][-.a-zA-Z0-9]*);')
|
||
return p.sub(r'&\1', data)
|
||
|
||
def stripHTML(soup):
|
||
return removeAllEntities(re.sub(r'<[^>]+>','',"%s" % soup)).strip()
|
||
|
||
def conditionalRemoveEntities(value):
|
||
if isinstance(value,str) or isinstance(value,unicode) :
|
||
return removeEntities(value).strip()
|
||
else:
|
||
return value
|
||
|
||
def removeAllEntities(text):
|
||
# Remove < < and &
|
||
return removeEntities(text).replace('<', '<').replace('>', '>').replace('&', '&')
|
||
|
||
def removeEntities(text):
|
||
|
||
# replace numeric versions of [&<>] with named versions,
|
||
# then replace named versions with actual characters,
|
||
|
||
if text is None:
|
||
return ""
|
||
if not (isinstance(text,str) or isinstance(text,unicode)):
|
||
return str(text)
|
||
|
||
try:
|
||
t = text.decode('utf-8')
|
||
except UnicodeEncodeError, e:
|
||
try:
|
||
t = text.encode ('ascii', 'xmlcharrefreplace')
|
||
except UnicodeEncodeError, e:
|
||
t = text
|
||
text = t
|
||
text = re.sub(r'�*38;','&',text)
|
||
text = re.sub(r'�*60;','<',text)
|
||
text = re.sub(r'�*62;','>',text)
|
||
|
||
# replace remaining � entities with unicode value, such as ' -> '
|
||
text = _replaceNumberEntities(text)
|
||
|
||
# replace several named entities with character, such as — -> -
|
||
# see constants.py for the list.
|
||
# reverse sort will put entities with ; before the same one without, when valid.
|
||
for e in reversed(sorted(entities.keys())):
|
||
v = entities[e]
|
||
try:
|
||
text = text.replace(e, v)
|
||
except UnicodeDecodeError, ex:
|
||
# for the pound symbol in constants.py
|
||
text = text.replace(e, v.decode('utf-8'))
|
||
|
||
# SGMLParser, and in turn, BeautifulStoneSoup doesn't parse
|
||
# entities terribly well and inserts (;) after something that
|
||
# it thinks might be an entity. AT&T becomes AT&T; All of my
|
||
# attempts to fix this by changing the input to
|
||
# BeautifulStoneSoup break something else instead. But at
|
||
# this point, there should be *no* real entities left, so find
|
||
# these not-entities and removing them here should be safe.
|
||
text = _replaceNotEntities(text)
|
||
|
||
# < < and & are the only html entities allowed in xhtml, put those back.
|
||
return text.replace('&', '&').replace('&lt', '<').replace('&gt', '>')
|
||
|
||
# entity list from http://code.google.com/p/doctype/wiki/CharacterEntitiesConsistent
|
||
entities = { 'á' : 'á',
|
||
'Á' : 'Á',
|
||
'Á' : 'Á',
|
||
'á' : 'á',
|
||
'â' : 'â',
|
||
'Â' : 'Â',
|
||
'Â' : 'Â',
|
||
'â' : 'â',
|
||
'´' : '´',
|
||
'´' : '´',
|
||
'Æ' : 'Æ',
|
||
'æ' : 'æ',
|
||
'Æ' : 'Æ',
|
||
'æ' : 'æ',
|
||
'à' : 'à',
|
||
'À' : 'À',
|
||
'À' : 'À',
|
||
'à' : 'à',
|
||
'ℵ' : 'ℵ',
|
||
'α' : 'α',
|
||
'Α' : 'Α',
|
||
'&' : '&',
|
||
'&' : '&',
|
||
'&' : '&',
|
||
'&' : '&',
|
||
'∧' : '∧',
|
||
'∠' : '∠',
|
||
'å' : 'å',
|
||
'Å' : 'Å',
|
||
'Å' : 'Å',
|
||
'å' : 'å',
|
||
'≈' : '≈',
|
||
'ã' : 'ã',
|
||
'Ã' : 'Ã',
|
||
'Ã' : 'Ã',
|
||
'ã' : 'ã',
|
||
'ä' : 'ä',
|
||
'Ä' : 'Ä',
|
||
'Ä' : 'Ä',
|
||
'ä' : 'ä',
|
||
'„' : '„',
|
||
'β' : 'β',
|
||
'Β' : 'Β',
|
||
'¦' : '¦',
|
||
'¦' : '¦',
|
||
'•' : '•',
|
||
'∩' : '∩',
|
||
'ç' : 'ç',
|
||
'Ç' : 'Ç',
|
||
'Ç' : 'Ç',
|
||
'ç' : 'ç',
|
||
'¸' : '¸',
|
||
'¸' : '¸',
|
||
'¢' : '¢',
|
||
'¢' : '¢',
|
||
'χ' : 'χ',
|
||
'Χ' : 'Χ',
|
||
'ˆ' : 'ˆ',
|
||
'♣' : '♣',
|
||
'≅' : '≅',
|
||
'©' : '©',
|
||
'©' : '©',
|
||
'©' : '©',
|
||
'©' : '©',
|
||
'↵' : '↵',
|
||
'∪' : '∪',
|
||
'¤' : '¤',
|
||
'¤' : '¤',
|
||
'†' : '†',
|
||
'‡' : '‡',
|
||
'↓' : '↓',
|
||
'⇓' : '⇓',
|
||
'°' : '°',
|
||
'°' : '°',
|
||
'δ' : 'δ',
|
||
'Δ' : 'Δ',
|
||
'♦' : '♦',
|
||
'÷' : '÷',
|
||
'÷' : '÷',
|
||
'é' : 'é',
|
||
'É' : 'É',
|
||
'É' : 'É',
|
||
'é' : 'é',
|
||
'ê' : 'ê',
|
||
'Ê' : 'Ê',
|
||
'Ê' : 'Ê',
|
||
'ê' : 'ê',
|
||
'è' : 'è',
|
||
'È' : 'È',
|
||
'È' : 'È',
|
||
'è' : 'è',
|
||
'∅' : '∅',
|
||
' ' : ' ',
|
||
' ' : ' ',
|
||
'ε' : 'ε',
|
||
'Ε' : 'Ε',
|
||
'≡' : '≡',
|
||
'η' : 'η',
|
||
'Η' : 'Η',
|
||
'ð' : 'ð',
|
||
'Ð' : 'Ð',
|
||
'Ð' : 'Ð',
|
||
'ð' : 'ð',
|
||
'ë' : 'ë',
|
||
'Ë' : 'Ë',
|
||
'Ë' : 'Ë',
|
||
'ë' : 'ë',
|
||
'€' : '€',
|
||
'∃' : '∃',
|
||
'ƒ' : 'ƒ',
|
||
'∀' : '∀',
|
||
'½' : '½',
|
||
'½' : '½',
|
||
'¼' : '¼',
|
||
'¼' : '¼',
|
||
'¾' : '¾',
|
||
'¾' : '¾',
|
||
'⁄' : '⁄',
|
||
'γ' : 'γ',
|
||
'Γ' : 'Γ',
|
||
'≥' : '≥',
|
||
#'>' : '>',
|
||
#'>' : '>',
|
||
#'>' : '>',
|
||
#'>' : '>',
|
||
'↔' : '↔',
|
||
'⇔' : '⇔',
|
||
'♥' : '♥',
|
||
'…' : '…',
|
||
'í' : 'í',
|
||
'Í' : 'Í',
|
||
'Í' : 'Í',
|
||
'í' : 'í',
|
||
'î' : 'î',
|
||
'Î' : 'Î',
|
||
'Î' : 'Î',
|
||
'î' : 'î',
|
||
'¡' : '¡',
|
||
'¡' : '¡',
|
||
'ì' : 'ì',
|
||
'Ì' : 'Ì',
|
||
'Ì' : 'Ì',
|
||
'ì' : 'ì',
|
||
'ℑ' : 'ℑ',
|
||
'∞' : '∞',
|
||
'∫' : '∫',
|
||
'ι' : 'ι',
|
||
'Ι' : 'Ι',
|
||
'¿' : '¿',
|
||
'¿' : '¿',
|
||
'∈' : '∈',
|
||
'ï' : 'ï',
|
||
'Ï' : 'Ï',
|
||
'Ï' : 'Ï',
|
||
'ï' : 'ï',
|
||
'κ' : 'κ',
|
||
'Κ' : 'Κ',
|
||
'λ' : 'λ',
|
||
'Λ' : 'Λ',
|
||
'«' : '«',
|
||
'«' : '«',
|
||
'←' : '←',
|
||
'⇐' : '⇐',
|
||
'⌈' : '⌈',
|
||
'“' : '“',
|
||
'≤' : '≤',
|
||
'⌊' : '⌊',
|
||
'∗' : '∗',
|
||
'◊' : '◊',
|
||
'‎' : '',
|
||
'‹' : '‹',
|
||
'‘' : '‘',
|
||
#'<' : '<',
|
||
#'<' : '<',
|
||
#'<' : '<',
|
||
#'<' : '<',
|
||
'¯' : '¯',
|
||
'¯' : '¯',
|
||
'—' : '—',
|
||
'µ' : 'µ',
|
||
'µ' : 'µ',
|
||
'·' : '·',
|
||
'·' : '·',
|
||
'−' : '−',
|
||
'μ' : 'μ',
|
||
'Μ' : 'Μ',
|
||
'∇' : '∇',
|
||
' ' : ' ',
|
||
' ' : ' ',
|
||
'–' : '–',
|
||
'≠' : '≠',
|
||
'∋' : '∋',
|
||
'¬' : '¬',
|
||
'¬' : '¬',
|
||
'∉' : '∉',
|
||
'⊄' : '⊄',
|
||
'ñ' : 'ñ',
|
||
'Ñ' : 'Ñ',
|
||
'Ñ' : 'Ñ',
|
||
'ñ' : 'ñ',
|
||
'ν' : 'ν',
|
||
'Ν' : 'Ν',
|
||
'ó' : 'ó',
|
||
'Ó' : 'Ó',
|
||
'Ó' : 'Ó',
|
||
'ó' : 'ó',
|
||
'ô' : 'ô',
|
||
'Ô' : 'Ô',
|
||
'Ô' : 'Ô',
|
||
'ô' : 'ô',
|
||
'Œ' : 'Œ',
|
||
'œ' : 'œ',
|
||
'ò' : 'ò',
|
||
'Ò' : 'Ò',
|
||
'Ò' : 'Ò',
|
||
'ò' : 'ò',
|
||
'‾' : '‾',
|
||
'ω' : 'ω',
|
||
'Ω' : 'Ω',
|
||
'ο' : 'ο',
|
||
'Ο' : 'Ο',
|
||
'⊕' : '⊕',
|
||
'∨' : '∨',
|
||
'ª' : 'ª',
|
||
'ª' : 'ª',
|
||
'º' : 'º',
|
||
'º' : 'º',
|
||
'ø' : 'ø',
|
||
'Ø' : 'Ø',
|
||
'Ø' : 'Ø',
|
||
'ø' : 'ø',
|
||
'õ' : 'õ',
|
||
'Õ' : 'Õ',
|
||
'Õ' : 'Õ',
|
||
'õ' : 'õ',
|
||
'⊗' : '⊗',
|
||
'ö' : 'ö',
|
||
'Ö' : 'Ö',
|
||
'Ö' : 'Ö',
|
||
'ö' : 'ö',
|
||
'¶' : '¶',
|
||
'¶' : '¶',
|
||
'∂' : '∂',
|
||
'‰' : '‰',
|
||
'⊥' : '⊥',
|
||
'φ' : 'φ',
|
||
'Φ' : 'Φ',
|
||
'π' : 'π',
|
||
'Π' : 'Π',
|
||
'ϖ' : 'ϖ',
|
||
'±' : '±',
|
||
'±' : '±',
|
||
'£' : '£',
|
||
'£' : '£',
|
||
'′' : '′',
|
||
'″' : '″',
|
||
'∏' : '∏',
|
||
'∝' : '∝',
|
||
'ψ' : 'ψ',
|
||
'Ψ' : 'Ψ',
|
||
'"' : '"',
|
||
'"' : '"',
|
||
'"' : '"',
|
||
'"' : '"',
|
||
'√' : '√',
|
||
'»' : '»',
|
||
'»' : '»',
|
||
'→' : '→',
|
||
'⇒' : '⇒',
|
||
'⌉' : '⌉',
|
||
'”' : '”',
|
||
'ℜ' : 'ℜ',
|
||
'®' : '®',
|
||
'®' : '®',
|
||
'®' : '®',
|
||
'®' : '®',
|
||
'⌋' : '⌋',
|
||
'ρ' : 'ρ',
|
||
'Ρ' : 'Ρ',
|
||
'‏' : '',
|
||
'›' : '›',
|
||
'’' : '’',
|
||
'‚' : '‚',
|
||
'š' : 'š',
|
||
'Š' : 'Š',
|
||
'⋅' : '⋅',
|
||
'§' : '§',
|
||
'§' : '§',
|
||
'­' : '', # strange optional hyphenation control character, not just a dash
|
||
'­' : '',
|
||
'σ' : 'σ',
|
||
'Σ' : 'Σ',
|
||
'ς' : 'ς',
|
||
'∼' : '∼',
|
||
'♠' : '♠',
|
||
'⊂' : '⊂',
|
||
'⊆' : '⊆',
|
||
'∑' : '∑',
|
||
'¹' : '¹',
|
||
'¹' : '¹',
|
||
'²' : '²',
|
||
'²' : '²',
|
||
'³' : '³',
|
||
'³' : '³',
|
||
'⊃' : '⊃',
|
||
'⊇' : '⊇',
|
||
'ß' : 'ß',
|
||
'ß' : 'ß',
|
||
'τ' : 'τ',
|
||
'Τ' : 'Τ',
|
||
'∴' : '∴',
|
||
'θ' : 'θ',
|
||
'Θ' : 'Θ',
|
||
'ϑ' : 'ϑ',
|
||
' ' : ' ',
|
||
'þ' : 'þ',
|
||
'Þ' : 'Þ',
|
||
'Þ' : 'Þ',
|
||
'þ' : 'þ',
|
||
'˜' : '˜',
|
||
'×' : '×',
|
||
'×' : '×',
|
||
'™' : '™',
|
||
'ú' : 'ú',
|
||
'Ú' : 'Ú',
|
||
'Ú' : 'Ú',
|
||
'ú' : 'ú',
|
||
'↑' : '↑',
|
||
'⇑' : '⇑',
|
||
'û' : 'û',
|
||
'Û' : 'Û',
|
||
'Û' : 'Û',
|
||
'û' : 'û',
|
||
'ù' : 'ù',
|
||
'Ù' : 'Ù',
|
||
'Ù' : 'Ù',
|
||
'ù' : 'ù',
|
||
'¨' : '¨',
|
||
'¨' : '¨',
|
||
'ϒ' : 'ϒ',
|
||
'υ' : 'υ',
|
||
'Υ' : 'Υ',
|
||
'ü' : 'ü',
|
||
'Ü' : 'Ü',
|
||
'Ü' : 'Ü',
|
||
'ü' : 'ü',
|
||
'℘' : '℘',
|
||
'ξ' : 'ξ',
|
||
'Ξ' : 'Ξ',
|
||
'ý' : 'ý',
|
||
'Ý' : 'Ý',
|
||
'Ý' : 'Ý',
|
||
'ý' : 'ý',
|
||
'¥' : '¥',
|
||
'¥' : '¥',
|
||
'ÿ' : 'ÿ',
|
||
'Ÿ' : 'Ÿ',
|
||
'ÿ' : 'ÿ',
|
||
'ζ' : 'ζ',
|
||
'Ζ' : 'Ζ',
|
||
'‍' : '', # strange spacing control character, not just a space
|
||
'‌' : '', # strange spacing control character, not just a space
|
||
}
|