Skip to content

Commit ee1cdb6

Browse files
levkowetzlevkowetz
committed
Add our own copy of html5lib
 - Legacy-Id: 2236
1 parent 3db627c commit ee1cdb6

36 files changed

Lines changed: 11444 additions & 0 deletions

‎html5lib/__init__.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
"""
2+
HTML parsing library based on the WHATWG "HTML5"
3+
specification. The parser is designed to be compatible with existing
4+
HTML found in the wild and implements well-defined error recovery that
5+
is largely compatible with modern desktop web browsers.
6+
7+
Example usage:
8+
9+
import html5lib
10+
f = open("my_document.html")
11+
tree = html5lib.parse(f)
12+
"""
13+
__version__ = "0.90"
14+
from html5parser import HTMLParser, parse, parseFragment
15+
from treebuilders import getTreeBuilder
16+
from treewalkers import getTreeWalker
17+
from serializer import serialize

‎html5lib/constants.py‎

Lines changed: 1169 additions & 0 deletions
Large diffs are not rendered by default.

‎html5lib/filters/__init__.py‎

Whitespace-only changes.

‎html5lib/filters/_base.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
2+
class Filter(object):
3+
def __init__(self, source):
4+
self.source = source
5+
6+
def __iter__(self):
7+
return iter(self.source)
8+
9+
def __getattr__(self, name):
10+
return getattr(self.source, name)

‎html5lib/filters/formfiller.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#
2+
# The goal is to finally have a form filler where you pass data for
3+
# each form, using the algorithm for "Seeding a form with initial values"
4+
# See http://www.whatwg.org/specs/web-forms/current-work/#seeding
5+
#
6+
7+
import _base
8+
9+
from html5lib.constants import spaceCharacters
10+
spaceCharacters = u"".join(spaceCharacters)
11+
12+
class SimpleFilter(_base.Filter):
13+
def __init__(self, source, fieldStorage):
14+
_base.Filter.__init__(self, source)
15+
self.fieldStorage = fieldStorage
16+
17+
def __iter__(self):
18+
field_indices = {}
19+
state = None
20+
field_name = None
21+
for token in _base.Filter.__iter__(self):
22+
type = token["type"]
23+
if type in ("StartTag", "EmptyTag"):
24+
name = token["name"].lower()
25+
if name == "input":
26+
field_name = None
27+
field_type = None
28+
input_value_index = -1
29+
input_checked_index = -1
30+
for i,(n,v) in enumerate(token["data"]):
31+
n = n.lower()
32+
if n == u"name":
33+
field_name = v.strip(spaceCharacters)
34+
elif n == u"type":
35+
field_type = v.strip(spaceCharacters)
36+
elif n == u"checked":
37+
input_checked_index = i
38+
elif n == u"value":
39+
input_value_index = i
40+
41+
value_list = self.fieldStorage.getlist(field_name)
42+
field_index = field_indices.setdefault(field_name, 0)
43+
if field_index < len(value_list):
44+
value = value_list[field_index]
45+
else:
46+
value = ""
47+
48+
if field_type in (u"checkbox", u"radio"):
49+
if value_list:
50+
if token["data"][input_value_index][1] == value:
51+
if input_checked_index < 0:
52+
token["data"].append((u"checked", u""))
53+
field_indices[field_name] = field_index + 1
54+
elif input_checked_index >= 0:
55+
del token["data"][input_checked_index]
56+
57+
elif field_type not in (u"button", u"submit", u"reset"):
58+
if input_value_index >= 0:
59+
token["data"][input_value_index] = (u"value", value)
60+
else:
61+
token["data"].append((u"value", value))
62+
field_indices[field_name] = field_index + 1
63+
64+
field_type = None
65+
field_name = None
66+
67+
elif name == "textarea":
68+
field_type = "textarea"
69+
field_name = dict((token["data"])[::-1])["name"]
70+
71+
elif name == "select":
72+
field_type = "select"
73+
attributes = dict(token["data"][::-1])
74+
field_name = attributes.get("name")
75+
is_select_multiple = "multiple" in attributes
76+
is_selected_option_found = False
77+
78+
elif field_type == "select" and field_name and name == "option":
79+
option_selected_index = -1
80+
option_value = None
81+
for i,(n,v) in enumerate(token["data"]):
82+
n = n.lower()
83+
if n == "selected":
84+
option_selected_index = i
85+
elif n == "value":
86+
option_value = v.strip(spaceCharacters)
87+
if option_value is None:
88+
raise NotImplementedError("<option>s without a value= attribute")
89+
else:
90+
value_list = self.fieldStorage.getlist(field_name)
91+
if value_list:
92+
field_index = field_indices.setdefault(field_name, 0)
93+
if field_index < len(value_list):
94+
value = value_list[field_index]
95+
else:
96+
value = ""
97+
if (is_select_multiple or not is_selected_option_found) and option_value == value:
98+
if option_selected_index < 0:
99+
token["data"].append((u"selected", u""))
100+
field_indices[field_name] = field_index + 1
101+
is_selected_option_found = True
102+
elif option_selected_index >= 0:
103+
del token["data"][option_selected_index]
104+
105+
elif field_type is not None and field_name and type == "EndTag":
106+
name = token["name"].lower()
107+
if name == field_type:
108+
if name == "textarea":
109+
value_list = self.fieldStorage.getlist(field_name)
110+
if value_list:
111+
field_index = field_indices.setdefault(field_name, 0)
112+
if field_index < len(value_list):
113+
value = value_list[field_index]
114+
else:
115+
value = ""
116+
yield {"type": "Characters", "data": value}
117+
field_indices[field_name] = field_index + 1
118+
119+
field_name = None
120+
121+
elif name == "option" and field_type == "select":
122+
pass # TODO: part of "option without value= attribute" processing
123+
124+
elif field_type == "textarea":
125+
continue # ignore token
126+
127+
yield token

‎html5lib/filters/inject_meta_charset.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import _base
2+
3+
class Filter(_base.Filter):
4+
def __init__(self, source, encoding):
5+
_base.Filter.__init__(self, source)
6+
self.encoding = encoding
7+
8+
def __iter__(self):
9+
state = "pre_head"
10+
meta_found = (self.encoding is None)
11+
pending = []
12+
13+
for token in _base.Filter.__iter__(self):
14+
type = token["type"]
15+
if type == "StartTag":
16+
if token["name"].lower() == "head":
17+
state = "in_head"
18+
19+
elif type == "EmptyTag":
20+
if token["name"].lower() == "meta":
21+
# replace charset with actual encoding
22+
has_http_equiv_content_type = False
23+
content_index = -1
24+
for i,(name,value) in enumerate(token["data"]):
25+
if name.lower() == 'charset':
26+
token["data"][i] = (u'charset', self.encoding)
27+
meta_found = True
28+
break
29+
elif name == 'http-equiv' and value.lower() == 'content-type':
30+
has_http_equiv_content_type = True
31+
elif name == 'content':
32+
content_index = i
33+
else:
34+
if has_http_equiv_content_type and content_index >= 0:
35+
token["data"][content_index] = (u'content', u'text/html; charset=%s' % self.encoding)
36+
meta_found = True
37+
38+
elif token["name"].lower() == "head" and not meta_found:
39+
# insert meta into empty head
40+
yield {"type": "StartTag", "name": "head",
41+
"data": token["data"]}
42+
yield {"type": "EmptyTag", "name": "meta",
43+
"data": [["charset", self.encoding]]}
44+
yield {"type": "EndTag", "name": "head"}
45+
meta_found = True
46+
continue
47+
48+
elif type == "EndTag":
49+
if token["name"].lower() == "head" and pending:
50+
# insert meta into head (if necessary) and flush pending queue
51+
yield pending.pop(0)
52+
if not meta_found:
53+
yield {"type": "EmptyTag", "name": "meta",
54+
"data": [["charset", self.encoding]]}
55+
while pending:
56+
yield pending.pop(0)
57+
meta_found = True
58+
state = "post_head"
59+
60+
if state == "in_head":
61+
pending.append(token)
62+
else:
63+
yield token

‎html5lib/filters/lint.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
from gettext import gettext
2+
_ = gettext
3+
4+
import _base
5+
from html5lib.constants import cdataElements, rcdataElements, voidElements
6+
7+
from html5lib.constants import spaceCharacters
8+
spaceCharacters = u"".join(spaceCharacters)
9+
10+
class LintError(Exception): pass
11+
12+
class Filter(_base.Filter):
13+
def __iter__(self):
14+
open_elements = []
15+
contentModelFlag = "PCDATA"
16+
for token in _base.Filter.__iter__(self):
17+
type = token["type"]
18+
if type in ("StartTag", "EmptyTag"):
19+
name = token["name"]
20+
if contentModelFlag != "PCDATA":
21+
raise LintError(_("StartTag not in PCDATA content model flag: %s") % name)
22+
if not isinstance(name, unicode):
23+
raise LintError(_(u"Tag name is not a string: %r") % name)
24+
if not name:
25+
raise LintError(_(u"Empty tag name"))
26+
if type == "StartTag" and name in voidElements:
27+
raise LintError(_(u"Void element reported as StartTag token: %s") % name)
28+
elif type == "EmptyTag" and name not in voidElements:
29+
raise LintError(_(u"Non-void element reported as EmptyTag token: %s") % token["name"])
30+
if type == "StartTag":
31+
open_elements.append(name)
32+
for name, value in token["data"]:
33+
if not isinstance(name, unicode):
34+
raise LintError(_("Attribute name is not a string: %r") % name)
35+
if not name:
36+
raise LintError(_(u"Empty attribute name"))
37+
if not isinstance(value, unicode):
38+
raise LintError(_("Attribute value is not a string: %r") % value)
39+
if name in cdataElements:
40+
contentModelFlag = "CDATA"
41+
elif name in rcdataElements:
42+
contentModelFlag = "RCDATA"
43+
elif name == "plaintext":
44+
contentModelFlag = "PLAINTEXT"
45+
46+
elif type == "EndTag":
47+
name = token["name"]
48+
if not isinstance(name, unicode):
49+
raise LintError(_(u"Tag name is not a string: %r") % name)
50+
if not name:
51+
raise LintError(_(u"Empty tag name"))
52+
if name in voidElements:
53+
raise LintError(_(u"Void element reported as EndTag token: %s") % name)
54+
start_name = open_elements.pop()
55+
if start_name != name:
56+
raise LintError(_(u"EndTag (%s) does not match StartTag (%s)") % (name, start_name))
57+
contentModelFlag = "PCDATA"
58+
59+
elif type == "Comment":
60+
if contentModelFlag != "PCDATA":
61+
raise LintError(_("Comment not in PCDATA content model flag"))
62+
63+
elif type in ("Characters", "SpaceCharacters"):
64+
data = token["data"]
65+
if not isinstance(data, unicode):
66+
raise LintError(_("Attribute name is not a string: %r") % data)
67+
if not data:
68+
raise LintError(_(u"%s token with empty data") % type)
69+
if type == "SpaceCharacters":
70+
data = data.strip(spaceCharacters)
71+
if data:
72+
raise LintError(_(u"Non-space character(s) found in SpaceCharacters token: ") % data)
73+
74+
elif type == "Doctype":
75+
name = token["name"]
76+
if contentModelFlag != "PCDATA":
77+
raise LintError(_("Doctype not in PCDATA content model flag: %s") % name)
78+
if not isinstance(name, unicode):
79+
raise LintError(_(u"Tag name is not a string: %r") % name)
80+
# XXX: what to do with token["data"] ?
81+
82+
elif type in ("ParseError", "SerializeError"):
83+
pass
84+
85+
else:
86+
raise LintError(_(u"Unknown token type: %s") % type)
87+
88+
yield token

0 commit comments

Comments
 (0)