Mercurial > repos > shellac > guppy_basecaller
diff env/lib/python3.7/site-packages/jinja2/lexer.py @ 5:9b1c78e6ba9c draft default tip
"planemo upload commit 6c0a8142489327ece472c84e558c47da711a9142"
| author | shellac |
|---|---|
| date | Mon, 01 Jun 2020 08:59:25 -0400 |
| parents | 79f47841a781 |
| children |
line wrap: on
line diff
--- a/env/lib/python3.7/site-packages/jinja2/lexer.py Thu May 14 16:47:39 2020 -0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,848 +0,0 @@ -# -*- coding: utf-8 -*- -"""Implements a Jinja / Python combination lexer. The ``Lexer`` class -is used to do some preprocessing. It filters out invalid operators like -the bitshift operators we don't allow in templates. It separates -template code and python code in expressions. -""" -import re -from ast import literal_eval -from collections import deque -from operator import itemgetter - -from ._compat import implements_iterator -from ._compat import intern -from ._compat import iteritems -from ._compat import text_type -from .exceptions import TemplateSyntaxError -from .utils import LRUCache - -# cache for the lexers. Exists in order to be able to have multiple -# environments with the same lexer -_lexer_cache = LRUCache(50) - -# static regular expressions -whitespace_re = re.compile(r"\s+", re.U) -newline_re = re.compile(r"(\r\n|\r|\n)") -string_re = re.compile( - r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S -) -integer_re = re.compile(r"(\d+_)*\d+") -float_re = re.compile( - r""" - (?<!\.) # doesn't start with a . - (\d+_)*\d+ # digits, possibly _ separated - ( - (\.(\d+_)*\d+)? # optional fractional part - e[+\-]?(\d+_)*\d+ # exponent part - | - \.(\d+_)*\d+ # required fractional part - ) - """, - re.IGNORECASE | re.VERBOSE, -) - -try: - # check if this Python supports Unicode identifiers - compile("föö", "<unknown>", "eval") -except SyntaxError: - # Python 2, no Unicode support, use ASCII identifiers - name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*") - check_ident = False -else: - # Unicode support, import generated re pattern and set flag to use - # str.isidentifier to validate during lexing. - from ._identifier import pattern as name_re - - check_ident = True - -# internal the tokens and keep references to them -TOKEN_ADD = intern("add") -TOKEN_ASSIGN = intern("assign") -TOKEN_COLON = intern("colon") -TOKEN_COMMA = intern("comma") -TOKEN_DIV = intern("div") -TOKEN_DOT = intern("dot") -TOKEN_EQ = intern("eq") -TOKEN_FLOORDIV = intern("floordiv") -TOKEN_GT = intern("gt") -TOKEN_GTEQ = intern("gteq") -TOKEN_LBRACE = intern("lbrace") -TOKEN_LBRACKET = intern("lbracket") -TOKEN_LPAREN = intern("lparen") -TOKEN_LT = intern("lt") -TOKEN_LTEQ = intern("lteq") -TOKEN_MOD = intern("mod") -TOKEN_MUL = intern("mul") -TOKEN_NE = intern("ne") -TOKEN_PIPE = intern("pipe") -TOKEN_POW = intern("pow") -TOKEN_RBRACE = intern("rbrace") -TOKEN_RBRACKET = intern("rbracket") -TOKEN_RPAREN = intern("rparen") -TOKEN_SEMICOLON = intern("semicolon") -TOKEN_SUB = intern("sub") -TOKEN_TILDE = intern("tilde") -TOKEN_WHITESPACE = intern("whitespace") -TOKEN_FLOAT = intern("float") -TOKEN_INTEGER = intern("integer") -TOKEN_NAME = intern("name") -TOKEN_STRING = intern("string") -TOKEN_OPERATOR = intern("operator") -TOKEN_BLOCK_BEGIN = intern("block_begin") -TOKEN_BLOCK_END = intern("block_end") -TOKEN_VARIABLE_BEGIN = intern("variable_begin") -TOKEN_VARIABLE_END = intern("variable_end") -TOKEN_RAW_BEGIN = intern("raw_begin") -TOKEN_RAW_END = intern("raw_end") -TOKEN_COMMENT_BEGIN = intern("comment_begin") -TOKEN_COMMENT_END = intern("comment_end") -TOKEN_COMMENT = intern("comment") -TOKEN_LINESTATEMENT_BEGIN = intern("linestatement_begin") -TOKEN_LINESTATEMENT_END = intern("linestatement_end") -TOKEN_LINECOMMENT_BEGIN = intern("linecomment_begin") -TOKEN_LINECOMMENT_END = intern("linecomment_end") -TOKEN_LINECOMMENT = intern("linecomment") -TOKEN_DATA = intern("data") -TOKEN_INITIAL = intern("initial") -TOKEN_EOF = intern("eof") - -# bind operators to token types -operators = { - "+": TOKEN_ADD, - "-": TOKEN_SUB, - "/": TOKEN_DIV, - "//": TOKEN_FLOORDIV, - "*": TOKEN_MUL, - "%": TOKEN_MOD, - "**": TOKEN_POW, - "~": TOKEN_TILDE, - "[": TOKEN_LBRACKET, - "]": TOKEN_RBRACKET, - "(": TOKEN_LPAREN, - ")": TOKEN_RPAREN, - "{": TOKEN_LBRACE, - "}": TOKEN_RBRACE, - "==": TOKEN_EQ, - "!=": TOKEN_NE, - ">": TOKEN_GT, - ">=": TOKEN_GTEQ, - "<": TOKEN_LT, - "<=": TOKEN_LTEQ, - "=": TOKEN_ASSIGN, - ".": TOKEN_DOT, - ":": TOKEN_COLON, - "|": TOKEN_PIPE, - ",": TOKEN_COMMA, - ";": TOKEN_SEMICOLON, -} - -reverse_operators = dict([(v, k) for k, v in iteritems(operators)]) -assert len(operators) == len(reverse_operators), "operators dropped" -operator_re = re.compile( - "(%s)" % "|".join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x))) -) - -ignored_tokens = frozenset( - [ - TOKEN_COMMENT_BEGIN, - TOKEN_COMMENT, - TOKEN_COMMENT_END, - TOKEN_WHITESPACE, - TOKEN_LINECOMMENT_BEGIN, - TOKEN_LINECOMMENT_END, - TOKEN_LINECOMMENT, - ] -) -ignore_if_empty = frozenset( - [TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT] -) - - -def _describe_token_type(token_type): - if token_type in reverse_operators: - return reverse_operators[token_type] - return { - TOKEN_COMMENT_BEGIN: "begin of comment", - TOKEN_COMMENT_END: "end of comment", - TOKEN_COMMENT: "comment", - TOKEN_LINECOMMENT: "comment", - TOKEN_BLOCK_BEGIN: "begin of statement block", - TOKEN_BLOCK_END: "end of statement block", - TOKEN_VARIABLE_BEGIN: "begin of print statement", - TOKEN_VARIABLE_END: "end of print statement", - TOKEN_LINESTATEMENT_BEGIN: "begin of line statement", - TOKEN_LINESTATEMENT_END: "end of line statement", - TOKEN_DATA: "template data / text", - TOKEN_EOF: "end of template", - }.get(token_type, token_type) - - -def describe_token(token): - """Returns a description of the token.""" - if token.type == TOKEN_NAME: - return token.value - return _describe_token_type(token.type) - - -def describe_token_expr(expr): - """Like `describe_token` but for token expressions.""" - if ":" in expr: - type, value = expr.split(":", 1) - if type == TOKEN_NAME: - return value - else: - type = expr - return _describe_token_type(type) - - -def count_newlines(value): - """Count the number of newline characters in the string. This is - useful for extensions that filter a stream. - """ - return len(newline_re.findall(value)) - - -def compile_rules(environment): - """Compiles all the rules from the environment into a list of rules.""" - e = re.escape - rules = [ - ( - len(environment.comment_start_string), - TOKEN_COMMENT_BEGIN, - e(environment.comment_start_string), - ), - ( - len(environment.block_start_string), - TOKEN_BLOCK_BEGIN, - e(environment.block_start_string), - ), - ( - len(environment.variable_start_string), - TOKEN_VARIABLE_BEGIN, - e(environment.variable_start_string), - ), - ] - - if environment.line_statement_prefix is not None: - rules.append( - ( - len(environment.line_statement_prefix), - TOKEN_LINESTATEMENT_BEGIN, - r"^[ \t\v]*" + e(environment.line_statement_prefix), - ) - ) - if environment.line_comment_prefix is not None: - rules.append( - ( - len(environment.line_comment_prefix), - TOKEN_LINECOMMENT_BEGIN, - r"(?:^|(?<=\S))[^\S\r\n]*" + e(environment.line_comment_prefix), - ) - ) - - return [x[1:] for x in sorted(rules, reverse=True)] - - -class Failure(object): - """Class that raises a `TemplateSyntaxError` if called. - Used by the `Lexer` to specify known errors. - """ - - def __init__(self, message, cls=TemplateSyntaxError): - self.message = message - self.error_class = cls - - def __call__(self, lineno, filename): - raise self.error_class(self.message, lineno, filename) - - -class Token(tuple): - """Token class.""" - - __slots__ = () - lineno, type, value = (property(itemgetter(x)) for x in range(3)) - - def __new__(cls, lineno, type, value): - return tuple.__new__(cls, (lineno, intern(str(type)), value)) - - def __str__(self): - if self.type in reverse_operators: - return reverse_operators[self.type] - elif self.type == "name": - return self.value - return self.type - - def test(self, expr): - """Test a token against a token expression. This can either be a - token type or ``'token_type:token_value'``. This can only test - against string values and types. - """ - # here we do a regular string equality check as test_any is usually - # passed an iterable of not interned strings. - if self.type == expr: - return True - elif ":" in expr: - return expr.split(":", 1) == [self.type, self.value] - return False - - def test_any(self, *iterable): - """Test against multiple token expressions.""" - for expr in iterable: - if self.test(expr): - return True - return False - - def __repr__(self): - return "Token(%r, %r, %r)" % (self.lineno, self.type, self.value) - - -@implements_iterator -class TokenStreamIterator(object): - """The iterator for tokenstreams. Iterate over the stream - until the eof token is reached. - """ - - def __init__(self, stream): - self.stream = stream - - def __iter__(self): - return self - - def __next__(self): - token = self.stream.current - if token.type is TOKEN_EOF: - self.stream.close() - raise StopIteration() - next(self.stream) - return token - - -@implements_iterator -class TokenStream(object): - """A token stream is an iterable that yields :class:`Token`\\s. The - parser however does not iterate over it but calls :meth:`next` to go - one token ahead. The current active token is stored as :attr:`current`. - """ - - def __init__(self, generator, name, filename): - self._iter = iter(generator) - self._pushed = deque() - self.name = name - self.filename = filename - self.closed = False - self.current = Token(1, TOKEN_INITIAL, "") - next(self) - - def __iter__(self): - return TokenStreamIterator(self) - - def __bool__(self): - return bool(self._pushed) or self.current.type is not TOKEN_EOF - - __nonzero__ = __bool__ # py2 - - @property - def eos(self): - """Are we at the end of the stream?""" - return not self - - def push(self, token): - """Push a token back to the stream.""" - self._pushed.append(token) - - def look(self): - """Look at the next token.""" - old_token = next(self) - result = self.current - self.push(result) - self.current = old_token - return result - - def skip(self, n=1): - """Got n tokens ahead.""" - for _ in range(n): - next(self) - - def next_if(self, expr): - """Perform the token test and return the token if it matched. - Otherwise the return value is `None`. - """ - if self.current.test(expr): - return next(self) - - def skip_if(self, expr): - """Like :meth:`next_if` but only returns `True` or `False`.""" - return self.next_if(expr) is not None - - def __next__(self): - """Go one token ahead and return the old one. - - Use the built-in :func:`next` instead of calling this directly. - """ - rv = self.current - if self._pushed: - self.current = self._pushed.popleft() - elif self.current.type is not TOKEN_EOF: - try: - self.current = next(self._iter) - except StopIteration: - self.close() - return rv - - def close(self): - """Close the stream.""" - self.current = Token(self.current.lineno, TOKEN_EOF, "") - self._iter = None - self.closed = True - - def expect(self, expr): - """Expect a given token type and return it. This accepts the same - argument as :meth:`jinja2.lexer.Token.test`. - """ - if not self.current.test(expr): - expr = describe_token_expr(expr) - if self.current.type is TOKEN_EOF: - raise TemplateSyntaxError( - "unexpected end of template, expected %r." % expr, - self.current.lineno, - self.name, - self.filename, - ) - raise TemplateSyntaxError( - "expected token %r, got %r" % (expr, describe_token(self.current)), - self.current.lineno, - self.name, - self.filename, - ) - try: - return self.current - finally: - next(self) - - -def get_lexer(environment): - """Return a lexer which is probably cached.""" - key = ( - environment.block_start_string, - environment.block_end_string, - environment.variable_start_string, - environment.variable_end_string, - environment.comment_start_string, - environment.comment_end_string, - environment.line_statement_prefix, - environment.line_comment_prefix, - environment.trim_blocks, - environment.lstrip_blocks, - environment.newline_sequence, - environment.keep_trailing_newline, - ) - lexer = _lexer_cache.get(key) - if lexer is None: - lexer = Lexer(environment) - _lexer_cache[key] = lexer - return lexer - - -class OptionalLStrip(tuple): - """A special tuple for marking a point in the state that can have - lstrip applied. - """ - - __slots__ = () - - # Even though it looks like a no-op, creating instances fails - # without this. - def __new__(cls, *members, **kwargs): - return super(OptionalLStrip, cls).__new__(cls, members) - - -class Lexer(object): - """Class that implements a lexer for a given environment. Automatically - created by the environment class, usually you don't have to do that. - - Note that the lexer is not automatically bound to an environment. - Multiple environments can share the same lexer. - """ - - def __init__(self, environment): - # shortcuts - e = re.escape - - def c(x): - return re.compile(x, re.M | re.S) - - # lexing rules for tags - tag_rules = [ - (whitespace_re, TOKEN_WHITESPACE, None), - (float_re, TOKEN_FLOAT, None), - (integer_re, TOKEN_INTEGER, None), - (name_re, TOKEN_NAME, None), - (string_re, TOKEN_STRING, None), - (operator_re, TOKEN_OPERATOR, None), - ] - - # assemble the root lexing rule. because "|" is ungreedy - # we have to sort by length so that the lexer continues working - # as expected when we have parsing rules like <% for block and - # <%= for variables. (if someone wants asp like syntax) - # variables are just part of the rules if variable processing - # is required. - root_tag_rules = compile_rules(environment) - - # block suffix if trimming is enabled - block_suffix_re = environment.trim_blocks and "\\n?" or "" - - # If lstrip is enabled, it should not be applied if there is any - # non-whitespace between the newline and block. - self.lstrip_unless_re = c(r"[^ \t]") if environment.lstrip_blocks else None - - self.newline_sequence = environment.newline_sequence - self.keep_trailing_newline = environment.keep_trailing_newline - - # global lexing rules - self.rules = { - "root": [ - # directives - ( - c( - "(.*?)(?:%s)" - % "|".join( - [ - r"(?P<raw_begin>%s(\-|\+|)\s*raw\s*(?:\-%s\s*|%s))" - % ( - e(environment.block_start_string), - e(environment.block_end_string), - e(environment.block_end_string), - ) - ] - + [ - r"(?P<%s>%s(\-|\+|))" % (n, r) - for n, r in root_tag_rules - ] - ) - ), - OptionalLStrip(TOKEN_DATA, "#bygroup"), - "#bygroup", - ), - # data - (c(".+"), TOKEN_DATA, None), - ], - # comments - TOKEN_COMMENT_BEGIN: [ - ( - c( - r"(.*?)((?:\-%s\s*|%s)%s)" - % ( - e(environment.comment_end_string), - e(environment.comment_end_string), - block_suffix_re, - ) - ), - (TOKEN_COMMENT, TOKEN_COMMENT_END), - "#pop", - ), - (c("(.)"), (Failure("Missing end of comment tag"),), None), - ], - # blocks - TOKEN_BLOCK_BEGIN: [ - ( - c( - r"(?:\-%s\s*|%s)%s" - % ( - e(environment.block_end_string), - e(environment.block_end_string), - block_suffix_re, - ) - ), - TOKEN_BLOCK_END, - "#pop", - ), - ] - + tag_rules, - # variables - TOKEN_VARIABLE_BEGIN: [ - ( - c( - r"\-%s\s*|%s" - % ( - e(environment.variable_end_string), - e(environment.variable_end_string), - ) - ), - TOKEN_VARIABLE_END, - "#pop", - ) - ] - + tag_rules, - # raw block - TOKEN_RAW_BEGIN: [ - ( - c( - r"(.*?)((?:%s(\-|\+|))\s*endraw\s*(?:\-%s\s*|%s%s))" - % ( - e(environment.block_start_string), - e(environment.block_end_string), - e(environment.block_end_string), - block_suffix_re, - ) - ), - OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END), - "#pop", - ), - (c("(.)"), (Failure("Missing end of raw directive"),), None), - ], - # line statements - TOKEN_LINESTATEMENT_BEGIN: [ - (c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop") - ] - + tag_rules, - # line comments - TOKEN_LINECOMMENT_BEGIN: [ - ( - c(r"(.*?)()(?=\n|$)"), - (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END), - "#pop", - ) - ], - } - - def _normalize_newlines(self, value): - """Called for strings and template data to normalize it to unicode.""" - return newline_re.sub(self.newline_sequence, value) - - def tokenize(self, source, name=None, filename=None, state=None): - """Calls tokeniter + tokenize and wraps it in a token stream.""" - stream = self.tokeniter(source, name, filename, state) - return TokenStream(self.wrap(stream, name, filename), name, filename) - - def wrap(self, stream, name=None, filename=None): - """This is called with the stream as returned by `tokenize` and wraps - every token in a :class:`Token` and converts the value. - """ - for lineno, token, value in stream: - if token in ignored_tokens: - continue - elif token == TOKEN_LINESTATEMENT_BEGIN: - token = TOKEN_BLOCK_BEGIN - elif token == TOKEN_LINESTATEMENT_END: - token = TOKEN_BLOCK_END - # we are not interested in those tokens in the parser - elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END): - continue - elif token == TOKEN_DATA: - value = self._normalize_newlines(value) - elif token == "keyword": - token = value - elif token == TOKEN_NAME: - value = str(value) - if check_ident and not value.isidentifier(): - raise TemplateSyntaxError( - "Invalid character in identifier", lineno, name, filename - ) - elif token == TOKEN_STRING: - # try to unescape string - try: - value = ( - self._normalize_newlines(value[1:-1]) - .encode("ascii", "backslashreplace") - .decode("unicode-escape") - ) - except Exception as e: - msg = str(e).split(":")[-1].strip() - raise TemplateSyntaxError(msg, lineno, name, filename) - elif token == TOKEN_INTEGER: - value = int(value.replace("_", "")) - elif token == TOKEN_FLOAT: - # remove all "_" first to support more Python versions - value = literal_eval(value.replace("_", "")) - elif token == TOKEN_OPERATOR: - token = operators[value] - yield Token(lineno, token, value) - - def tokeniter(self, source, name, filename=None, state=None): - """This method tokenizes the text and returns the tokens in a - generator. Use this method if you just want to tokenize a template. - """ - source = text_type(source) - lines = source.splitlines() - if self.keep_trailing_newline and source: - for newline in ("\r\n", "\r", "\n"): - if source.endswith(newline): - lines.append("") - break - source = "\n".join(lines) - pos = 0 - lineno = 1 - stack = ["root"] - if state is not None and state != "root": - assert state in ("variable", "block"), "invalid state" - stack.append(state + "_begin") - statetokens = self.rules[stack[-1]] - source_length = len(source) - balancing_stack = [] - lstrip_unless_re = self.lstrip_unless_re - newlines_stripped = 0 - line_starting = True - - while 1: - # tokenizer loop - for regex, tokens, new_state in statetokens: - m = regex.match(source, pos) - # if no match we try again with the next rule - if m is None: - continue - - # we only match blocks and variables if braces / parentheses - # are balanced. continue parsing with the lower rule which - # is the operator rule. do this only if the end tags look - # like operators - if balancing_stack and tokens in ( - TOKEN_VARIABLE_END, - TOKEN_BLOCK_END, - TOKEN_LINESTATEMENT_END, - ): - continue - - # tuples support more options - if isinstance(tokens, tuple): - groups = m.groups() - - if isinstance(tokens, OptionalLStrip): - # Rule supports lstrip. Match will look like - # text, block type, whitespace control, type, control, ... - text = groups[0] - - # Skipping the text and first type, every other group is the - # whitespace control for each type. One of the groups will be - # -, +, or empty string instead of None. - strip_sign = next(g for g in groups[2::2] if g is not None) - - if strip_sign == "-": - # Strip all whitespace between the text and the tag. - stripped = text.rstrip() - newlines_stripped = text[len(stripped) :].count("\n") - groups = (stripped,) + groups[1:] - elif ( - # Not marked for preserving whitespace. - strip_sign != "+" - # lstrip is enabled. - and lstrip_unless_re is not None - # Not a variable expression. - and not m.groupdict().get(TOKEN_VARIABLE_BEGIN) - ): - # The start of text between the last newline and the tag. - l_pos = text.rfind("\n") + 1 - if l_pos > 0 or line_starting: - # If there's only whitespace between the newline and the - # tag, strip it. - if not lstrip_unless_re.search(text, l_pos): - groups = (text[:l_pos],) + groups[1:] - - for idx, token in enumerate(tokens): - # failure group - if token.__class__ is Failure: - raise token(lineno, filename) - # bygroup is a bit more complex, in that case we - # yield for the current token the first named - # group that matched - elif token == "#bygroup": - for key, value in iteritems(m.groupdict()): - if value is not None: - yield lineno, key, value - lineno += value.count("\n") - break - else: - raise RuntimeError( - "%r wanted to resolve " - "the token dynamically" - " but no group matched" % regex - ) - # normal group - else: - data = groups[idx] - if data or token not in ignore_if_empty: - yield lineno, token, data - lineno += data.count("\n") + newlines_stripped - newlines_stripped = 0 - - # strings as token just are yielded as it. - else: - data = m.group() - # update brace/parentheses balance - if tokens == TOKEN_OPERATOR: - if data == "{": - balancing_stack.append("}") - elif data == "(": - balancing_stack.append(")") - elif data == "[": - balancing_stack.append("]") - elif data in ("}", ")", "]"): - if not balancing_stack: - raise TemplateSyntaxError( - "unexpected '%s'" % data, lineno, name, filename - ) - expected_op = balancing_stack.pop() - if expected_op != data: - raise TemplateSyntaxError( - "unexpected '%s', " - "expected '%s'" % (data, expected_op), - lineno, - name, - filename, - ) - # yield items - if data or tokens not in ignore_if_empty: - yield lineno, tokens, data - lineno += data.count("\n") - - line_starting = m.group()[-1:] == "\n" - - # fetch new position into new variable so that we can check - # if there is a internal parsing error which would result - # in an infinite loop - pos2 = m.end() - - # handle state changes - if new_state is not None: - # remove the uppermost state - if new_state == "#pop": - stack.pop() - # resolve the new state by group checking - elif new_state == "#bygroup": - for key, value in iteritems(m.groupdict()): - if value is not None: - stack.append(key) - break - else: - raise RuntimeError( - "%r wanted to resolve the " - "new state dynamically but" - " no group matched" % regex - ) - # direct state name given - else: - stack.append(new_state) - statetokens = self.rules[stack[-1]] - # we are still at the same position and no stack change. - # this means a loop without break condition, avoid that and - # raise error - elif pos2 == pos: - raise RuntimeError( - "%r yielded empty string without stack change" % regex - ) - # publish new function and start again - pos = pos2 - break - # if loop terminated without break we haven't found a single match - # either we are at the end of the file or we have a problem - else: - # end of text - if pos >= source_length: - return - # something went wrong - raise TemplateSyntaxError( - "unexpected char %r at %d" % (source[pos], pos), - lineno, - name, - filename, - )