"Useful HTML utilities suitable for global use by World Online projects." import re, string # Configuration for urlize() function LEADING_PUNCTUATION = ['(', '<', '<'] TRAILING_PUNCTUATION = ['.', ',', ')', '>', '\n', '>'] # list of possible strings used for bullets in bulleted lists DOTS = ['·', '*', '\xe2\x80\xa2', '•', '•', '•'] UNENCODED_AMPERSANDS_RE = re.compile(r'&(?!(\w+|#\d+);)') WORD_SPLIT_RE = re.compile(r'(\s+)') PUNCTUATION_RE = re.compile('^(?P(?:%s)*)(?P.*?)(?P(?:%s)*)$' % \ ('|'.join([re.escape(p) for p in LEADING_PUNCTUATION]), '|'.join([re.escape(p) for p in TRAILING_PUNCTUATION]))) SIMPLE_EMAIL_RE = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$') LINK_TARGET_ATTRIBUTE = re.compile(r'(]*?)target=[^\s>]+') HTML_GUNK = re.compile(r'(?:
|<\/i>|<\/b>|<\/em>|<\/strong>|<\/?smallcaps>|<\/?uppercase>)', re.IGNORECASE) HARD_CODED_BULLETS = re.compile(r'((?:

(?:%s).*?[a-zA-Z].*?

\s*)+)' % '|'.join([re.escape(d) for d in DOTS]), re.DOTALL) TRAILING_EMPTY_CONTENT = re.compile(r'(?:

(?: |\s|
)*?

\s*)+\Z') def escape(html): "Returns the given HTML with ampersands, quotes and carets encoded" if not isinstance(html, basestring): html = str(html) return html.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"') def linebreaks(value): "Converts newlines into

and
s" value = re.sub(r'\r\n|\r|\n', '\n', value) # normalize newlines paras = re.split('\n{2,}', value) paras = ['

%s

' % p.strip().replace('\n', '
') for p in paras] return '\n\n'.join(paras) def strip_tags(value): "Returns the given HTML with all tags stripped" return re.sub(r'<[^>]*?>', '', value) def strip_entities(value): "Returns the given HTML with all entities (&something;) stripped" return re.sub(r'&(?:\w+|#\d);', '', value) def fix_ampersands(value): "Returns the given HTML with all unencoded ampersands encoded correctly" return UNENCODED_AMPERSANDS_RE.sub('&', value) def urlize(text, trim_url_limit=None, nofollow=False): """ Converts any URLs in text into clickable links. Works on http://, https:// and www. links. Links can have trailing punctuation (periods, commas, close-parens) and leading punctuation (opening parens) and it'll still do the right thing. If trim_url_limit is not None, the URLs in link text will be limited to trim_url_limit characters. If nofollow is True, the URLs in link text will get a rel="nofollow" attribute. """ trim_url = lambda x, limit=trim_url_limit: limit is not None and (x[:limit] + (len(x) >=limit and '...' or '')) or x words = WORD_SPLIT_RE.split(text) nofollow_attr = nofollow and ' rel="nofollow"' or '' for i, word in enumerate(words): match = PUNCTUATION_RE.match(word) if match: lead, middle, trail = match.groups() if middle.startswith('www.') or ('@' not in middle and not middle.startswith('http://') and \ len(middle) > 0 and middle[0] in string.letters + string.digits and \ (middle.endswith('.org') or middle.endswith('.net') or middle.endswith('.com'))): middle = '
%s' % (middle, nofollow_attr, trim_url(middle)) if middle.startswith('http://') or middle.startswith('https://'): middle = '%s' % (middle, nofollow_attr, trim_url(middle)) if '@' in middle and not middle.startswith('www.') and not ':' in middle \ and SIMPLE_EMAIL_RE.match(middle): middle = '%s' % (middle, middle) if lead + middle + trail != word: words[i] = lead + middle + trail return ''.join(words) def clean_html(text): """ Cleans the given HTML. Specifically, it does the following: * Converts and to and . * Encodes all ampersands correctly. * Removes all "target" attributes from tags. * Removes extraneous HTML, such as presentational tags that open and immediately close and
. * Converts hard-coded bullets into HTML unordered lists. * Removes stuff like "

  

", but only if it's at the bottom of the text. """ from django.utils.text import normalize_newlines text = normalize_newlines(text) text = re.sub(r'<(/?)\s*b\s*>', '<\\1strong>', text) text = re.sub(r'<(/?)\s*i\s*>', '<\\1em>', text) text = fix_ampersands(text) # Remove all target="" attributes from
tags. text = LINK_TARGET_ATTRIBUTE.sub('\\1', text) # Trim stupid HTML such as
. text = HTML_GUNK.sub('', text) # Convert hard-coded bullets into HTML unordered lists. def replace_p_tags(match): s = match.group().replace('

', '') for d in DOTS: s = s.replace('

%s' % d, '

  • ') return '
      \n%s\n
    ' % s text = HARD_CODED_BULLETS.sub(replace_p_tags, text) # Remove stuff like "

      

    ", but only if it's at the bottom of the text. text = TRAILING_EMPTY_CONTENT.sub('', text) return text