import itertools
import logging
import re
import sys
import urlparse
import html5lib
from html5lib.sanitizer import HTMLSanitizer
from html5lib.serializer.htmlserializer import HTMLSerializer
from encoding import force_unicode
from sanitizer import BleachSanitizer
VERSION = (1, 1, 5)
__version__ = '.'.join(map(str, VERSION))
__all__ = ['clean', 'linkify']
log = logging.getLogger('bleach')
ALLOWED_TAGS = [
'a',
'abbr',
'acronym',
'b',
'blockquote',
'code',
'em',
'i',
'li',
'ol',
'strong',
'ul',
]
ALLOWED_ATTRIBUTES = {
'a': ['href', 'title'],
'abbr': ['title'],
'acronym': ['title'],
}
ALLOWED_STYLES = []
TLDS = """ac ad ae aero af ag ai al am an ao aq ar arpa as asia at au aw ax az
ba bb bd be bf bg bh bi biz bj bm bn bo br bs bt bv bw by bz ca cat
cc cd cf cg ch ci ck cl cm cn co com coop cr cu cv cx cy cz de dj dk
dm do dz ec edu ee eg er es et eu fi fj fk fm fo fr ga gb gd ge gf gg
gh gi gl gm gn gov gp gq gr gs gt gu gw gy hk hm hn hr ht hu id ie il
im in info int io iq ir is it je jm jo jobs jp ke kg kh ki km kn kp
kr kw ky kz la lb lc li lk lr ls lt lu lv ly ma mc md me mg mh mil mk
ml mm mn mo mobi mp mq mr ms mt mu museum mv mw mx my mz na name nc ne
net nf ng ni nl no np nr nu nz om org pa pe pf pg ph pk pl pm pn pr pro
ps pt pw py qa re ro rs ru rw sa sb sc sd se sg sh si sj sk sl sm sn so
sr st su sv sy sz tc td tel tf tg th tj tk tl tm tn to tp tr travel tt
tv tw tz ua ug uk us uy uz va vc ve vg vi vn vu wf ws xn ye yt yu za zm
zw""".split()
TLDS.reverse()
url_re = re.compile(
r"""\(* # Match any opening parentheses.
\b(?"]*)?
# /path/zz (excluding "unsafe" chars from RFC 1738,
# except for # and ~, which happen in practice)
""" % u'|'.join(TLDS), re.VERBOSE | re.UNICODE)
proto_re = re.compile(r'^[\w-]+:/{0,3}')
punct_re = re.compile(r'([\.,]+)$')
email_re = re.compile(
r"""(?%(mail)s'
return repl % {'mail': match.group(0).replace('"', '"')}
def link_repl(match):
url = match.group(0)
open_brackets = close_brackets = 0
if url.startswith('('):
url, open_brackets, close_brackets = (
strip_wrapping_parentheses(url)
)
end = u''
m = re.search(punct_re, url)
if m:
end = m.group(0)
url = url[0:m.start()]
if re.search(proto_re, url):
href = url
else:
href = u''.join([u'http://', url])
repl = u'%s%s%s%s'
attribs = [rel]
if target is not None:
attribs.append('target="%s"' % target)
return repl % ('(' * open_brackets,
filter_url(href), ' '.join(attribs), filter_text(url),
end, ')' * close_brackets)
linkify_nodes(forest)
return _render(forest)
def delinkify(text, allow_domains=None, allow_relative=False):
"""Remove links from text, except those allowed to stay."""
text = force_unicode(text)
if not text:
return u''
parser = html5lib.HTMLParser(tokenizer=HTMLSanitizer)
forest = parser.parseFragment(text)
if allow_domains is None:
allow_domains = []
elif isinstance(allow_domains, basestring):
allow_domains = [allow_domains]
def delinkify_nodes(tree):
"""Remove tags and replace them with their contents."""
for node in tree.childNodes:
if node.name == 'a':
if 'href' not in node.attributes:
continue
parts = urlparse.urlparse(node.attributes['href'])
host = parts.hostname
if any(_domain_match(host, d) for d in allow_domains):
continue
if host is None and allow_relative:
continue
# Replace the node with its children.
# You can't nest tags, and html5lib takes care of that
# for us in the tree-building step.
for n in node.childNodes:
tree.insertBefore(n, node)
tree.removeChild(node)
elif node.type != NODE_TEXT: # Don't try to delinkify text.
delinkify_nodes(node)
delinkify_nodes(forest)
return _render(forest)
def _domain_match(test, compare):
test = test.lower()
compare = compare.lower()
if '*' not in compare:
return test == compare
c = compare.split('.')[::-1]
if '**' in c and (c.count('**') > 1 or not compare.startswith('**')):
raise ValidationError(
'Only 1 ** is allowed, and must start the domain.')
t = test.split('.')[::-1]
z = itertools.izip_longest(c, t)
for c, t in z:
if c == t:
continue
elif c == '*':
continue
elif c == '**':
return True
return False
# Got all the way through and everything matched.
return True
class ValidationError(ValueError):
pass
def _render(tree):
"""Try rendering as HTML, then XML, then give up."""
try:
return force_unicode(_serialize(tree))
except Exception, e:
log.error('HTML: %r' % e, exc_info=sys.exc_info())
try:
return force_unicode(tree.toxml())
except Exception, e:
log.error('XML: %r' % e, exc_info=sys.exc_info())
return u''
def _serialize(domtree):
walker = html5lib.treewalkers.getTreeWalker('simpletree')
stream = walker(domtree)
serializer = HTMLSerializer(quote_attr_values=True,
omit_optional_tags=False)
return serializer.render(stream)