Add support for text/html and text/enriched

mbox2web

@@ -1,10 +1,14 @@
 #!/usr/bin/python3
 
-import jinja2
+import html
+import html.parser
 import mailbox
 import os
 import re
 import sys
+import urllib.parse
+
+import jinja2
 
 basedir = "."
 
@@ -43,13 +47,13 @@ def render_message(msg):
 
 def render_body(msg):
     content_type = msg.get_content_type()
+
     if content_type == "text/plain":
         bodytmpl = jenv.get_template("body_text_plain.html")
         context = {
             "body": msg.get_payload()
         }
         bodyhtml = bodytmpl.render(context)
-        return jinja2.Markup(bodyhtml)
     elif content_type == "multipart/mixed":
         partshtml = []
         for part in msg.get_payload():
@@ -59,7 +63,6 @@ def render_body(msg):
             "parts": partshtml
         }
         bodyhtml = bodytmpl.render(context)
-        return jinja2.Markup(bodyhtml)
     elif content_type == "multipart/digest":
         partshtml = []
         for part in msg.get_payload():
@@ -69,7 +72,6 @@ def render_body(msg):
             "parts": partshtml
         }
         bodyhtml = bodytmpl.render(context)
-        return jinja2.Markup(bodyhtml)
     elif content_type == "message/rfc822":
         partshtml = []
         for part in msg.get_payload():
@@ -79,10 +81,18 @@ def render_body(msg):
             "parts": partshtml
         }
         bodyhtml = bodytmpl.render(context)
-        return jinja2.Markup(bodyhtml)
+    elif content_type == "text/html":
+        htmlpart = HTMLPart()
+        htmlpart.feed(msg.get_payload())
+        bodyhtml = htmlpart.as_string()
+    elif content_type == "text/enriched":
+        tepart = TextEnrichedPart(msg.get_payload())
+        bodyhtml = tepart.as_string()
     else:
         raise RuntimeError("Content-type " + content_type + " not implemented yet")
 
+    return jinja2.Markup(bodyhtml)
+
 
 def archive(msg):
     mid = get_message_id(msg)
@@ -104,6 +114,145 @@ def archive(msg):
         hfd.write(msghtml)
 
 
+class HTMLPart(html.parser.HTMLParser):
+    allowed_tags = [ 'h2', 'a', 'wbr', 'hr', 'pre', 'img', 'font', 'i' ]
+    hide_tags = [ 'title' ]
+    ignore_tags = [ 'html', 'head', 'body' ]
+
+    def __init__(self):
+        super().__init__()
+        self.hide = False
+        self.content = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "base":
+            href = [x[1] for x in attrs if x[0] == "href"]
+            if href:
+                self.base = href[0]
+        elif tag in self.allowed_tags:
+            attrstr = "".join(
+                            [' %s="%s"' % (a[0],  html.escape(a[1]))
+                                for a in self.clean_attrs(tag, attrs)
+                            ]
+                         )
+            self.content.append("<%s%s>" % ( tag, attrstr ))
+        elif tag in self.hide_tags:
+            self.hide = True
+        elif tag in self.ignore_tags:
+            pass
+        else:
+            print("Encountered unknown start tag", tag, attrs, file=sys.stderr)
+
+    def handle_endtag(self, tag):
+        if tag in self.allowed_tags:
+            self.content.append("</%s>" % tag)
+        elif tag in self.hide_tags:
+            self.hide = False # XXX - Need stack?
+        elif tag in self.ignore_tags:
+            pass
+        else:
+            print("Encountered unknown end tag", tag, file=sys.stderr)
+
+    def handle_data(self, data):
+        if not self.hide:
+            self.content.append(data)
+
+    def as_string(self):
+        return "".join(self.content)
+
+    def clean_attrs(self, tag, attrs):
+        clean_attrs = []
+        for a in attrs:
+            if a[0] == "href":
+                url = a[1]
+                url = urllib.parse.urljoin(self.base, url)
+                u = urllib.parse.urlparse(url)
+                if u[0] in ['https', 'http', 'ftp']:
+                    clean_attrs.append((a[0], url))
+            elif a[0] == "src":
+                url = a[1]
+                url = urllib.parse.urljoin(self.base, url)
+                u = urllib.parse.urlparse(url)
+                if u[0] == "cid":
+                    print("Encountered src cid attribute", a, file=sys.stderr)
+                    # XXX - implement cid
+                    clean_attrs.append((a[0], url))
+                else:
+                    print("Ignored src attribute", a, file=sys.stderr)
+            elif a[0] == "border":
+                clean_attrs.append(a)
+            elif a[0] == "alt":
+                clean_attrs.append(a)
+            elif a[0] == "size":
+                clean_attrs.append(a)
+
+            else:
+                print("Encountered unknown attribute", a, file=sys.stderr)
+        return clean_attrs
+
+
+class TextEnrichedPart:
+    class TEElement:
+        def __init__(self, t):
+            self.type = t.lower()
+            self.content = []
+            self.filled = True
+        
+        def append_text(self, s):
+            s = s.replace("<<", "<")
+            if self.filled:
+                s = re.sub(r'\n+',
+                           lambda m: m.group(0)[1:] if len(m.group(0)) > 1 else " ",
+                           s)
+            self.content.append(s)
+
+        def as_string(self):
+            if self.type == "":
+                pre = "<div class='text-enriched'>"
+                post = "</div>"
+            elif self.type == "bold":
+                pre = "<b>"
+                post = "</b>"
+            else:
+                raise NotImplementedError("Unknown type " + self.type)
+
+            s = pre
+            for c in self.content:
+                if isinstance(c, type(self)):
+                    s += c.as_string()
+                else:
+                    s += html.escape(c)
+            s += post
+            return s
+
+    def __init__(self, s):
+        self.stack = [ self.TEElement("") ]
+        while s:
+            stack_top = self.stack[-1]
+            m = re.match(r'(.*?)<(/?[A-Za-z0-9-]{,60})>(.*)', s, re.DOTALL)
+            if m:
+                if m.group(2).lower == "param" and re.match(r'\s*', m.group(1)):
+                    stack_top.content.append(TEElement("param"))
+                else:
+                    stack_top.append_text(m.group(1))
+                    if m.group(2)[0] != "/":
+                        new = self.TEElement(m.group(2))
+                        stack_top.content.append(new)
+                        self.stack.append(new)
+                    else:
+                        if stack_top.type == m.group(2)[1:]:
+                            self.stack.pop()
+                        else:
+                            raise RuntimeError("Nesting error: Expected %s, got %s near %s", self.stack[-1].type, m.group(2)[1:], s)
+                s = m.group(3)
+            else:
+                stack_top.append_text(s)
+                s = ""
+
+    def as_string(self):
+        return self.stack[0].as_string()
+
+
 for f in sys.argv[1:]:
     print("F", f)
     mb = mailbox.mbox(f)