"""
Generate the index.html/.md for the examples subdirectory,
based on structured docstrings on each Python example
"""

from __future__ import print_function
import ast
import re
import os
import sys

prog_name, _ = os.path.splitext(os.path.basename(sys.argv[0]))
prog_dir     = os.path.dirname(sys.argv[0])

from argparse import ArgumentParser
parser = ArgumentParser()
subparsers = parser.add_subparsers(dest="command")
subparsers.required = True
w_subparser = subparsers.add_parser("write")
s_subparser = subparsers.add_parser("show")

parser.add_argument("-v", "--verbose", default=False, action="store_true")
w_subparser.add_argument("-e", "--examples-dir", required=True)
w_subparser.add_argument("-t", "--template", required=True)
w_subparser.add_argument("-o", "--output", required=True)
s_subparser.add_argument("-k", "--keywords", default=False, action="store_true")
s_subparser.add_argument("-s", "--search", type=str)
s_subparser.add_argument("-e", "--examples-dir", required=True)

args = parser.parse_args()

#-------------------------------------------------------------------------------
CATEGORY_INFO = [
    (
        "ui",
        "User interface",
        "Creating & manipulating user-interface widgets, prompting the " +
        "user with forms, enriching existing widgets, or creating your " +
        "own UI through Python Qt bindings."
    ),
    (
        "disassembler",
        "Disassembly",
        "Various ways to query, or modify the disassembly listing, " +
        "alter the way analysis is performed, or be notified of changes " +
        "made to the IDB."
    ),
    (
        "decompiler",
        "Decompilation",
        "Querying the decompiler, manipulating the decompilation trees " +
        "(either at the microcode level, or the C-tree), and examples " +
        "showing how to intervene in the decompilation output."
    ),
    (
        "debugger",
        "Debuggers",
        "Driving debugging sessions, be notified of debugging events."
    ),
    (
        "types",
        "Working with types",
        "These samples utilize our Type APIs, which allow you to manage " +
        "the types and perform various operations on them, like creating " +
        "the structures or enums and adding their members programmatically."
    ),
    (
        "misc",
        "Miscellaneous",
        "Miscellaneous examples that don't quite fall into another category, " +
        "but don't really justify one of their own."
    ),
]

#-------------------------------------------------------------------------------
LEVEL_INFO = [
    (
        "beginner",
        "Beginner",
    ),
    (
        "intermediate",
        "Intermediate",
    ),
    (
        "advanced",
        "Advanced",
    ),
]


# --------------------------------------------------------------------------
def verb(msg):
    if args.verbose:
        print(msg)

# --------------------------------------------------------------------------
class ProcessException(Exception):
    pass

# --------------------------------------------------------------------------
def read_config():
    global config

    config_path = os.path.join(prog_dir, prog_name + ".cfg")
    with open(config_path, "r", encoding="UTF-8", errors="surrogateescape") as f:
        config = ast.literal_eval(f.read())

    config["exclude"] = set(config["exclude"])

    for _, key_res in config["auto-keywords"]:
        for i in range(len(key_res)):
            key_res[i] = re.compile(key_res[i] + "$") # anchored for match()

#-------------------------------------------------------------------------------
class Examples(object):
    def __init__(self):
        self.re_indent   = re.compile(r"^ *")
        self.re_tag      = re.compile(r"^ *([a-zA-Z_][a-zA-Z_0-9]*):")
        self.re_wspace   = re.compile(r"[ \t\r\n]*")
        self.re_userlink = re.compile(r"@{(.*?)}")

        self.index = None
        self.shortcut_finder = ShortcutFinder()

    def process(self):
        if args.command == "write":
            # parses the templates, and bails out on template error
            # before going into analyzing the examples
            self.index = ExamplesIndex()

        self.examples = []

        for relpath, path in self._files_with_extension(".py", args.examples_dir):
            verb("Processing \"%s\"" % path)
            with open(path, "r", encoding="UTF-8", errors="surrogateescape") as f:
                self._load_ast(relpath, ast.parse(f.read(), path))

        self._post_read_processing()

        if args.command == "show":
            self._show()
        elif args.command == "write":
            self.index.produce(self.examples, self._collect_all_keywords())

    def _post_read_processing(self):
        self._collect_example_names()  # requisite for other post-read actions

        self._collect_imported_uses()

        # trim down the final collection

        self.examples = [e for e in self.examples \
                         if e.name not in config["exclude"]]
        self.examples = sorted(self.examples, key=lambda x: x.name)

        self._collect_example_names()  # now invalid, redo

        self._parse_description_links()
        self._check_see_also_links()

    def _show(self):
        if args.keywords:
            print("\nKeywords used:")
            for keyword in self._collect_all_keywords():
                print(" ", keyword)

        if args.search:
            print("\nExamples using \"%s\":" % args.search)
            for example in self.examples:
                if args.search in example.keywords:
                    print(" ", example.path)

    def _collect_all_keywords(self):
        all_keywords = set()
        for example in self.examples:
            all_keywords.update(example.keywords)

        return sorted(all_keywords, key=lambda x: x.lower())

    def _load_ast(self, relpath, tree):
        name     = os.path.splitext(os.path.basename(relpath))[0]
        category = relpath.split(os.sep)[0]

        # fix for Windows: paths may contain "\" instead of "/";
        # these ultimately come from os.walk() and the command line
        # (_files_with_extension(), plus args.examples_dir -> rootdir)
        relpath = relpath.replace("\\", "/")

        uses, imports = self._uses_and_imports(tree)
        docstring     = ast.get_docstring(tree, False)

        tags = Tags()
        tags.add(Tags.PATH, relpath)
        tags.add(Tags.NAME, name)
        tags.add(Tags.CATEGORY, category)

        shortcuts = sorted(self.shortcut_finder.find(tree), \
                           key=lambda x: x.lower())
        tags.add(Tags.SHORTCUTS, shortcuts)

        # the next two are kept as sets;
        # they will be sorted lists after _collect_imported_uses()
        tags.add(Tags.USES, uses)
        tags.add(Tags.IMPORTS, imports)

        self._parse_structured_comment(docstring, tags)
        self._add_autokeywords(tags)

        self.examples.append(Example(tags.get()))

    def _add_autokeywords(self, tags):
        def add_keyword(m, key_out):
            groups_dict = {str(n): m.group(n) for n in range(1, m.re.groups+1)}
            new_keyword = key_out % groups_dict

            keywords = tags.tag_so_far(Tags.KEYWORDS)
            keywords.append(new_keyword)

        global config
        autokeys = config["auto-keywords"]

        for use in tags.tag_so_far(Tags.USES):
            for key_out, key_res in autokeys:
                # check for ALL auto-keywords generated by this "use"
                for key_re in key_res:
                    # check for ANY pattern that matches this auto-keyword
                    m = key_re.match(use)
                    if m:
                        add_keyword(m, key_out)
                        break

        keywords = tags.tag_so_far(Tags.KEYWORDS)
        tags.add(Tags.KEYWORDS, sorted(keywords, key=lambda x: x.lower()))

    def _parse_structured_comment(self, docstring, tags):
        if not docstring:
            return

        first_line  = True
        group_tag   = None
        group_lines = []

        def end_group():
            if group_tag:
                self._remove_common_indentation(group_lines)

                if group_tag == Tags.DESCRIPTION:
                    item = "\n".join(group_lines).strip("\n")
                elif group_tag in (Tags.KEYWORDS, Tags.SEE_ALSO):
                    item = []
                    for x in ",".join(group_lines).split(","):
                        x = x.strip()
                        if x:
                            item.append(x)
                else:
                    item = " ".join(group_lines).strip()

                if group_tag in (Tags.SUMMARY, Tags.DESCRIPTION):
                    # capitalize first
                    m = self.re_wspace.match(item)
                    if m and m.end() < len(item):
                        item = item[:m.end()] + item[m.end()].upper() \
                                              + item[m.end()+1:]

                verb("  Read from docstring: tag \"%s\"" % group_tag)
                tags.add(group_tag, item)

        # Blocks of lines are separated by indentation:
        # a block MUST begin with a tag (followed by ":") at indent 0.
        #
        # An attempt is made to preserve the original indentation
        # of each block of lines.

        for line in docstring.split("\n"):
            indent, line = self._indented_line(line.rstrip(), 0)

            if indent == 0 and line:
                end_group()
                first_line  = True
                group_tag   = None
                group_lines = []

            if first_line:
                if not line:
                    # ignore blank lines before the first tag
                    continue
                first_line = False

                m = self.re_tag.match(line)
                if not m:
                    tags.error("No tag in docstring: \"\"\"\n%s\n\"\"\"" % docstring)

                group_tag = m.group(1)
                tags.valid(group_tag)
                tags.can_occur_in_docstring(group_tag)

                indent, line = self._indented_line(line, m.end())

            if group_tag:
                group_lines.append( (indent, line) )

        end_group()

    def _indented_line(self, line, fromPos):
        line = line[fromPos:]

        m = self.re_indent.match(line)

        indent = fromPos + len(m.group())
        line   = line[m.end():]

        return indent, line

    def _remove_common_indentation(self, lines):
        if not lines:
            return

        min_indent = min(t[0] for t in lines if t[1])

        for i in range(len(lines)):
            if lines[i][1]:
                lines[i] = ((lines[i][0] - min_indent) * " ") + lines[i][1]
            else:
                lines[i] = ""

    def _uses_and_imports(self, tree):
        names = set()
        def callback_uses(name):
            verb("  Uses \"%s\"" % name)
            names.add(name)

        imports = set()
        def callback_imports(name):
            verb("  Imports \"%s\"" % name)
            imports.add(name)

        visitor = TreeVisitor(callback_uses,
                              callback_imports,
                              ["idc", "idautils"])
        visitor.generic_visit(tree)

        return names, imports

    def _parse_description_links(self):
        # (all examples have been read already)
        for example in self.examples:
            text = example.description

            out = ""
            while True:
                m = self.re_userlink.search(text)
                if not m:
                    break

                target_name = m.group(1)
                if target_name not in self.examples_set:
                    raise ProcessException(
                        "Link in \"%s\": \"%s\" is not an example name" \
                        % (example.path, target_name))

                try:
                    subs = config["link-format"] % {'1': target_name}
                except Exception as ex:
                    raise ProcessException(
                        "Link format, %s: %s" % (type(ex).__name__, ex))

                out  = text[:m.start()] + subs
                text = text[m.end():]

            example.description = out + text

    def _check_see_also_links(self):
        # do their targets exist?
        # (all examples have been read already)
        for example in self.examples:
            for see_also in example.see_also:
                if see_also not in self.examples_set:
                    raise ProcessException(
                        "%s\n  See-also link to unexistent \"%s\""
                        % (example.path, see_also))

    def _collect_imported_uses(self):
        # if an example imports another,
        # make the parent "use" the API calls that the child "uses"
        # (this has to be done after all examples have been read)

        examples_dict = { e.name: e for e in self.examples }

        # 1. restrict the imports to examples only

        imported_set = set()
        for example in self.examples:
            example.imports &= self.examples_set
            imported_set    |= example.imports

        # 2. topological sort of the imported examples

        graph = {}
        for example in self.examples:
            for imported in example.imports:
                # this example imports other examples
                if example.name in graph:
                    graph[example.name].append(imported)
                else:
                    graph[example.name] = [imported]

        roots = set(graph) - imported_set

        # 3. add children uses to parents

        for name in roots:
            self._add_imported_uses(name, examples_dict, graph, set())

        # 4. now we can make them sorted lists instead of sets
        #    (we leave "uses" of uppercase before lowercase;
        #     the use of constants looks better this way)

        for example in self.examples:
            example.uses    = sorted(example.uses)
            example.imports = sorted(example.imports, key=lambda x: x.lower())

    def _add_imported_uses(self, name, examples_dict, graph, cycle_detect):
        # cycles shouldn't happen if we're reading code that actually works,
        # but we better protect ourselves against bad input
        cycle_detect.add(name)

        example = examples_dict[name]

        for child in graph[name]:
            if child in graph:
                if child in cycle_detect:
                    raise ProcessException("Cycle in import: %s -> %s" % (name, child))

                self._add_imported_uses(child, examples_dict, graph, cycle_detect)

            example.uses |= examples_dict[child].uses

        cycle_detect.remove(name)

    def _collect_example_names(self):
        # (after all examples have been read)
        self.examples_set = set(e.name for e in self.examples)

    def _files_with_extension(self, ext, rootdir):
        for path, _, files in os.walk(rootdir):
            for filename in files:
                if os.path.splitext(filename)[1] == ext:
                    relpath = os.path.relpath(path, rootdir)
                    yield os.path.join(relpath, filename), \
                          os.path.join(path, filename)

#-------------------------------------------------------------------------------
class Tags(object):
    NAME        = "name"
    PATH        = "path"
    SUMMARY     = "summary"
    DESCRIPTION = "description"
    CATEGORY    = "category"
    KEYWORDS    = "keywords"
    SHORTCUTS   = "shortcuts"
    USES        = "uses"
    IMPORTS     = "imports"
    SEE_ALSO    = "see_also"
    AUTHOR      = "author"
    LEVEL       = "level"

    # tags that can be used in the template (with the {{example.XXX}} syntax)
    ALL_TAGS = set([NAME, PATH, SUMMARY, DESCRIPTION, CATEGORY, \
                    KEYWORDS, USES, IMPORTS, SEE_ALSO, AUTHOR, \
                    LEVEL])

    # tags that can be used in the examples' docstrings
    IN_DOCSTRING = set([SUMMARY, DESCRIPTION, CATEGORY, \
                        KEYWORDS, USES, SEE_ALSO, AUTHOR, \
                        LEVEL])

    def __init__(self):
        self.items = {}
        for tag in self.ALL_TAGS:
            if tag in (self.KEYWORDS, self.SEE_ALSO):
                self.items[tag] = []
            elif tag in (self.USES, self.IMPORTS):
                self.items[tag] = set()
            else:
                self.items[tag] = ""

    def error(self, message):
        raise ProcessException("%s:\n  %s" \
                               % (self.items[self.PATH], message))

    def valid(self, tag):
        if tag not in self.ALL_TAGS:
            self.error("Unrecognized tag: \"%s\"" % tag)

    def can_occur_in_docstring(self, tag):
        if tag not in self.IN_DOCSTRING:
            self.error("Not allowed in docstring: \"%s\"" % tag)

    def add(self, tag, item):
        self.items[tag] = item

    def tag_so_far(self, tag):
        return self.items[tag]

    def get(self):
        return self.items

#-------------------------------------------------------------------------------
class Example(object):
    def __init__(self, content):
        self.content = content
        _, self.file_name = os.path.split(self.path)

    def __getattr__(self, key):
        if key in self.content:
            return self.content[key]
        raise AttributeError("Example: \"%s\"" % key)

#-------------------------------------------------------------------------------
class TreeVisitor(ast.NodeVisitor):
    def __init__(self, callback_uses,
                       callback_imports,
                       interesting_names,
                       *args):
        super(TreeVisitor, self).__init__(*args)
        self.callback_uses     = callback_uses
        self.callback_imports  = callback_imports
        self.interesting_names = set(interesting_names)

    def visit_Import(self, node):
        for alias in node.names:
            self.callback_imports(alias.name)

    def visit_ImportFrom(self, node):
        self.callback_imports(node.module)

        if self._is_interesting(node.module):
            self._add_interesting_names(node.names)

    def _add_interesting_names(self, aliases):
        for alias in aliases:
            if isinstance(alias, ast.alias):
                name = alias.asname if alias.asname else alias.name
                self.interesting_names.add(name)

    def _is_interesting(self, name):
        return name.startswith("ida_") or name in self.interesting_names

    def visit_Name(self, node):
        self._callback_if_interesting(node)

    def visit_Attribute(self, node):
        self._callback_if_interesting(node)

    def _callback_if_interesting(self, node):
        names = self._dotted_name(node)
        if not names:
            return
        if self._is_interesting(names[0]) \
           and names[-1] != "__init__":     # removed calls to base constructors
            self.callback_uses(".".join(names))

    def _dotted_name(self, node):
        if isinstance(node, ast.Name):
            return [node.id]
        if isinstance(node, ast.Attribute):
            names = self._dotted_name(node.value)
            if names:
                return names + [node.attr]
        return None

#-------------------------------------------------------------------------------
class ShortcutFinder(ast.NodeVisitor):
    def find(self, tree):
        self._reset()
        self.generic_visit(tree)

        return self.results

    def _reset(self):
        self.results = []
        self.stack   = []

    def generic_visit(self, node):
        # look for "action_desc_t" instances, 4th constructor parameter
        if isinstance(node, ast.Call):
            name = self._name(node.func)

            if name == "ida_kernwin.action_desc_t":
                if len(node.args) >= 4:
                    # search the scopes for value(s) of this parameter
                    values = self._find_values(node.args[3])
                    for value in values:
                        self.results.append(value)

        # assigment - fill in (string) values into the current scope
        elif isinstance(node, ast.Assign):
            names  = [self._name(t) for t in node.targets]
            scope  = self.stack[-1][1]

            value = None
            # save strings
            if isinstance(node.value, ast.Constant):
                value = node.value.value
            # save also lists of names, in case they are class names
            elif isinstance(node.value, ast.List):
                if all(isinstance(elt, ast.Name) for elt in node.value.elts):
                    value = [elt.id for elt in node.value.elts]

            for name in names:
                self._assign(scope, name, value)

            # if our immediate ancestor is a class definition,
            # add this symbol also to the class scope
            if len(self.stack) > 1 and self.stack[-2][0]:
                class_scope = self.stack[-2][0]
                if self.class_name in class_scope:
                    for name in names:
                        self._assign(class_scope[self.class_name], name, value)

        # remember class names
        elif isinstance(node, ast.ClassDef):
            class_scope     = self.stack[-1][0]
            self.class_name = node.name

            class_scope[self.class_name] = {}

        # "for" - treat it as a list assignment
        elif isinstance(node, ast.For):
            # only syntax recognized: "for name in name"
            if isinstance(node.target, ast.Name) and \
               isinstance(node.iter,   ast.Name):
                name   = node.target.id
                scope  = self.stack[-1][1]
                values = self._find_list(node.iter.id)
                # save the variable as having multiple values
                self._assign(scope, name, values)
                scope[name] = values

        # keep a stack of scopes [classes, variables]
        self.stack.append([{}, {}])
        super(ShortcutFinder, self).generic_visit(node)
        self.stack.pop()

    def _assign(self, scope, name, value):
        if value:
            scope[name] = value
        else:
            # symbol overwritten with something I don't understand, forget
            if name in scope:
                del scope[name]

    def _find_values(self, arg):
        if isinstance(arg, ast.Constant):
            return [arg.value]

        # single variable
        if isinstance(arg, ast.Name):
            value = self._variable_in_scope(arg.id)
            if value:
                return [value]

        # variable dot variable - TODO: nested classes
        if isinstance(arg, ast.Attribute) and \
           isinstance(arg.value, ast.Name):
            left_name  = arg.value.id
            right_name = arg.attr

            values = self._find_list(left_name)
            if values:
                # return the values for each class scope
                returned = []
                for c_name in values:
                    class_scope = self._find_class_scope(c_name)
                    if right_name in class_scope:
                        returned.append(class_scope[right_name])
                return returned

        return []

    def _variable_in_scope(self, name):
        for i in range(len(self.stack) - 1, -1, -1):
            scope = self.stack[i][1]
            if name in scope:
                return scope[name]
        return None

    def _find_list(self, name):
        values = self._variable_in_scope(name)
        if isinstance(values, list):
            return values
        return None

    def _find_class_scope(self, name):
        for i in range(len(self.stack) - 1, -1, -1):
            class_scope = self.stack[i][0]
            if name in class_scope:
                return class_scope[name]
        return {}

    def _name(self, node):
        if isinstance(node, ast.Name):
            return node.id
        if isinstance(node, ast.Attribute):
            return self._name(node.value) + "." + node.attr

        # too complex for me
        return "@" + node.__class__.__name__

#-------------------------------------------------------------------------------
class ExamplesIndex(object):
    def __init__(self):
        self.replacer = TemplateReplacer()

        with open(args.template, "r", encoding="UTF-8", errors="surrogateescape") as f:
            self.replacer.template(f.read())

    def produce(self, examples, all_keywords):
        with open(args.output, "w", encoding="UTF-8") as f:
            self.replacer.expand(examples, all_keywords, f)

#-------------------------------------------------------------------------------
class Categories(object):

    class Category(object):

        class Level(object):
            def __init__(self, id):
                self.id = id
                assert(self.id)
                for rank, tpl in enumerate(LEVEL_INFO):
                    if tpl[0] == id:
                        self.rank = rank
                        _, label = tpl
                self.label = label # will bomb if not found
                self.examples = []

            def append(self, example):
                self.examples.append(example)

        def __init__(self, id):
            self.id = id
            for rank, tpl in enumerate(CATEGORY_INFO):
                if tpl[0] == id:
                    self.rank = rank
                    _, label, description = tpl
            self.label = label # will bomb if not found
            self.description = " ".join(description.split("\n"))
            self.levels = []

        def get_level(self, id):
            for l in self.levels:
                if l.id == id:
                    return l
            l = self.Level(id)
            self.levels.append(l)
            return l

        def append(self, example):
            if not example.level:
                raise Exception("Missing level for %s" % example.path)
            self.get_level(example.level).append(example)

    def __init__(self):
        self.categories = []

    def get_category(self, id):
        for c in self.categories:
            if c.id == id:
                return c
        c = self.Category(id)
        self.categories.append(c)
        return c

    def append(self, example):
        self.get_category(example.category).append(example)


#-------------------------------------------------------------------------------
class TemplateReplacer(object):
    def __init__(self):
        self.re_percent_line = re.compile(r" *% *")
        self.re_end_keyword  = re.compile(r"end([a-z]+|})$")
        self.re_variable     = re.compile(r"{{(.*?)}}")

        self.keywords = { # has_body, keyword
            ast.If:          (True,  "if"),
            ast.For:         (True,  "for"),
            ast.FunctionDef: (True,  "def"),
            ast.Break:       (False, "break"),
            ast.Continue:    (False, "continue"),
            ast.Return:      (False, "return")
        }
        # "else" treated separately

    def template(self, content):
        lines = content.rstrip("\n").split("\n")

        # convert the template lines into a Python module

        body = [self._sandwich(lines)]
        self.module = ast.Module(body=body, type_ignores=[])
        self.module = ast.fix_missing_locations(self.module)

        # "import" the module
        self.module_scope = {}
        exec(compile(self.module, "Template", "exec"), self.module_scope)

    def expand(self, examples, all_keywords, f):
        # group by category
        categories = Categories()
        for example in examples:
            assert example.description and example.summary and not example.summary.endswith("."), "Path: %s" % example.path
            categories.append(example)

        # call the template's entry point
        self.module_scope["__mm__entry"](categories, all_keywords, f)

    def _sandwich(self, lines, lineno=0, open_keyword=None):
        in_else = False

        body = []
        text = ""
        while lineno < len(lines):
            line = lines[lineno]
            lineno += 1

            # process "% statement"s in the template
            m = self.re_percent_line.match(line)
            if m:
                line = self._clean_line(line, m.end())

                # text so far
                if text:
                    body.extend(self._substitute_variables(text))
                    text = ""

                # "else" is not an ast.<Node>; parsed separately
                ret = self._is_else(line, lineno, open_keyword)
                if ret:
                    in_else   = True
                    prev_body = body
                    body = []
                    continue

                # code block start "%{"
                if self._is_start_of_code_block(line, lineno):
                    inner_body, _, lineno = self._code_block(lines, lineno)
                    body.extend(inner_body)
                    continue

                if self._is_end_keyword(line, lineno, open_keyword):
                    if not in_else:
                        prev_body, body = body, []
                    return prev_body, body, lineno

                has_body, keyword, statement = self._check_percent_line(line, lineno)
                if has_body:
                    inner_body, \
                    inner_else, lineno = self._sandwich(lines, lineno, keyword)

                    statement.body = inner_body
                    if inner_else:
                        statement.orelse = inner_else

                body.append(statement)
                continue

            text += line + "\n"

        if open_keyword:
            raise ProcessException(
                "Template: unterminated \"%s\"" % open_keyword)

        if text:
            body.extend(self._substitute_variables(text))

        # wrap the final code in a function, so that data can be passed
        return self._function_def("__mm__entry",
                                  ["categories", "all_keywords", "_mm_f"],
                                  body)

    def _code_block(self, lines, lineno):
        start_lineno = lineno

        code = ""
        while lineno < len(lines):
            line = lines[lineno]
            lineno += 1

            # process "% statement"s in the template
            m = self.re_percent_line.match(line)
            if m:
                line = self._clean_line(line, m.end())

                if self._is_end_of_code_block(line, lineno):
                    try:
                        body = ast.parse(code, "Template").body
                        return body, [], lineno
                    except:
                        line_range = "%s" % lineno if lineno == start_lineno \
                                     else "%d-%d" % (start_lineno, lineno)
                        raise ProcessException("Template[%s]: syntax error" % \
                                   line_range)
                break # no other % allowed but %}

            code += line + "\n"

        raise ProcessException("Template: unterminated \"{\"")

    def _is_else(self, line, lineno, open_keyword):
        if line != "else:":
            return False

        # use only in statements that accept "else"
        if open_keyword not in ("if", "for"):
            raise ProcessException(
                "Template[%d]: invalid \"else\"%s" % \
                    (lineno, " in \"%s\"" % open_keyword \
                             if open_keyword else ""))

        return True

    def _is_start_of_code_block(self, line, lineno):
        return line == "{"

    def _is_end_of_code_block(self, line, lineno):
        return line == "}"

    def _is_end_keyword(self, line, lineno, open_keyword):
        m = self.re_end_keyword.match(line)
        if not m:
            return False

        if m.group(1) != open_keyword:
            raise ProcessException(
                "   Template[%d]: expected \"end%s\"" % \
                (lineno, open_keyword))

        return True

    def _check_percent_line(self, line, lineno):
        # return the code for a "% statement" in the template
        if line[-1] == ":":
            line += "\n pass"
        try:
            tree = ast.parse(line, "Template")
        except:
            raise ProcessException("Template[%d]: syntax error" % lineno)

        if type(tree) is ast.Module and \
           len(tree.body) == 1:
            head = type(tree.body[0])
            if head in self.keywords:
                return self.keywords[head] + (tree.body[0],)

        raise ProcessException("Template[%d]: unrecognized keyword" % lineno)

    def _clean_line(self, line, code_pos):
        comment_pos = line.find("#")
        if comment_pos >= 0:
            line = line[:comment_pos]

        return line[code_pos:].rstrip()

    def _substitute_variables(self, input_text):
        # generate code to substitute {{variables}} when the template is run
        code = []
        text = ""
        while input_text:
            m = self.re_variable.search(input_text)
            if not m:
                break

            # text in-between
            text += input_text[:m.start()]
            if text:
                code.append(self._output_text(text))
                text = ""
            input_text = input_text[m.end():]

            code.append(self._output_eval(m.group(1)))

        text += input_text
        if text:
            code.append(self._output_text(text))

        return code

    def _output_text(self, text):
        return ast.Expr(value=self._call(self._dot("_mm_f", "write"),
                                         [ast.Constant(text)]))

    def _output_eval(self, expr):
        # eval(expr)
        value = self._call(ast.Name("eval", ast.Load()), [ast.Constant(expr)])
        # str(eval(expr))
        valueStr = self._call(ast.Name("str", ast.Load()), [value])
        # _mm_f.write(str(eval(expr)))
        call = self._call(self._dot("_mm_f", "write"), [valueStr])

        return ast.Expr(call)

    def _dot(self, name1, name2):
        return ast.Attribute(value=ast.Name(name1, ast.Load()),
                             attr=name2,
                             ctx=ast.Load())

    def _call(self, func, args):
        return ast.Call(
            func=func,
            args=args,
            keywords=[]
        )

    def _function_def(self, fun_name, fun_args, fun_body):
        fun_args = ast.arguments(args=[ast.arg(a, None) \
                                       for a in fun_args],
                                 posonlyargs=[],
                                 kwonlyargs=[],
                                 kw_defaults=[],
                                 defaults=[])
        return ast.FunctionDef(name=fun_name,
                               args=fun_args,
                               body=fun_body,
                               decorator_list=[])

#-------------------------------------------------------------------------------
try:
    read_config()
    e = Examples()
    e.process()
except ProcessException as ex:
    print(ex)
    sys.exit(1)
# other exceptions - let them fail
