entities.py - mozsearch

Enable keyboard shortcuts

import json

import html5lib

def parse(path="html5ents.xml"):

    return html5lib.parse(open(path), treebuilder="lxml")

def entity_table(tree):

    return {entity_name("".join(tr[0].xpath(".//text()"))):

            entity_characters(tr[1].text)

            for tr in tree.xpath("//h:tbody/h:tr",

                                 namespaces={"h": "http://www.w3.org/1999/xhtml"})}

def entity_name(inp):

    return inp.strip()

def entity_characters(inp):

    return "".join(codepoint_to_character(item)

                   for item in inp.split()

                   if item)

def codepoint_to_character(inp):

    return ("\\U000" + inp[2:]).decode("unicode-escape")

def make_tests_json(entities):

    test_list = make_test_list(entities)

    tests_json = {"tests":

                  [make_test(*item) for item in test_list]

    return tests_json

def make_test(name, characters, good):

    return {

        "description": test_description(name, good),

        "input": "&%s" % name,

        "output": test_expected(name, characters, good)

def test_description(name, good):

    with_semicolon = name.endswith(";")

    semicolon_text = {True: "with a semi-colon",

                      False: "without a semi-colon"}[with_semicolon]

    if good:

        text = "Named entity: %s %s" % (name, semicolon_text)

    else:

        text = "Bad named entity: %s %s" % (name, semicolon_text)

    return text

def test_expected(name, characters, good):

    rv = []

    if not good or not name.endswith(";"):

        rv.append("ParseError")

    rv.append(["Character", characters])

    return rv

def make_test_list(entities):

    tests = []

    for entity_name, characters in entities.items():

        if entity_name.endswith(";") and not subentity_exists(entity_name, entities):

            tests.append((entity_name[:-1], "&" + entity_name[:-1], False))

        tests.append((entity_name, characters, True))

    return sorted(tests)

def subentity_exists(entity_name, entities):

    for i in range(1, len(entity_name)):

        if entity_name[:-i] in entities:

            return True

    return False

def make_entities_code(entities):

    entities_text = "\n".join("    \"%s\": u\"%s\"," % (

        name, entities[name].encode(

            "unicode-escape").replace("\"", "\\\""))

        for name in sorted(entities.keys()))

    return """entities = {

%s

}""" % entities_text

def main():

    entities = entity_table(parse())

    tests_json = make_tests_json(entities)

    json.dump(tests_json, open("namedEntities.test", "w"), indent=4)

    code = make_entities_code(entities)

    open("entities_constants.py", "w").write(code)

if __name__ == "__main__":

    main()