diff --git a/ipython_pygments_lexers.py b/ipython_pygments_lexers.py index 22e342c..6398dd2 100644 --- a/ipython_pygments_lexers.py +++ b/ipython_pygments_lexers.py @@ -22,13 +22,13 @@ to Pygments. """ -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Copyright (c) 2013, the IPython Development Team. # # Distributed under the terms of the Modified BSD License. # # The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- __version__ = "9.0.0" @@ -37,68 +37,141 @@ # Third party from pygments.lexers import ( - BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer, - Python3Lexer, TexLexer) + BashLexer, + HtmlLexer, + JavascriptLexer, + RubyLexer, + PerlLexer, + PythonLexer, + Python3Lexer, + TexLexer, +) from pygments.lexer import ( - Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using, + Lexer, + DelegatingLexer, + RegexLexer, + do_insertions, + bygroups, + using, ) from pygments.token import ( - Generic, Keyword, Literal, Name, Operator, Other, Text, Error, + Generic, + Keyword, + Literal, + Name, + Operator, + Other, + Text, + Error, ) from pygments.util import get_bool_opt -line_re = re.compile('.*?\n') +line_re = re.compile(".*?\n") __all__ = [ - 'IPython3Lexer', - 'IPythonLexer', - 'IPythonPartialTracebackLexer', - 'IPythonTracebackLexer', - 'IPythonConsoleLexer', - 'IPyLexer' + "IPython3Lexer", + "IPythonLexer", + "IPythonPartialTracebackLexer", + "IPythonTracebackLexer", + "IPythonConsoleLexer", + "IPyLexer", ] ipython_tokens = [ - (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))), - (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))), - (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))), - (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))), - (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))), - (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))), - (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))), - (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), - (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))), + ( + r"(?s)(\s*)(%%capture)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?s)(\s*)(%%debug)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?is)(\s*)(%%html)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(HtmlLexer)), + ), + ( + r"(?s)(\s*)(%%javascript)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(JavascriptLexer)), + ), + ( + r"(?s)(\s*)(%%js)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(JavascriptLexer)), + ), + ( + r"(?s)(\s*)(%%latex)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(TexLexer)), + ), + ( + r"(?s)(\s*)(%%perl)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(PerlLexer)), + ), + ( + r"(?s)(\s*)(%%prun)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?s)(\s*)(%%pypy)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?s)(\s*)(%%python)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?s)(\s*)(%%python2)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(PythonLexer)), + ), + ( + r"(?s)(\s*)(%%python3)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?s)(\s*)(%%ruby)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(RubyLexer)), + ), + ( + r"(?s)(\s*)(%%time)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?s)(\s*)(%%timeit)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?s)(\s*)(%%writefile)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), + ( + r"(?s)(\s*)(%%file)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(Python3Lexer)), + ), (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)), - (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))), - (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)), - (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)), - (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword, - using(BashLexer), Text)), - (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)), - (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)), - (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)), - (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)), - (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)), + ( + r"(?s)(^\s*)(%%!)([^\n]*\n)(.*)", + bygroups(Text, Operator, Text, using(BashLexer)), + ), + (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)), + (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)), + (r"(%)(sx|sc|system)(.*)(\n)", bygroups(Operator, Keyword, using(BashLexer), Text)), + (r"(%)(\w+)(.*\n)", bygroups(Operator, Keyword, Text)), + (r"^(!!)(.+)(\n)", bygroups(Operator, using(BashLexer), Text)), + (r"(!)(?!=)(.+)(\n)", bygroups(Operator, using(BashLexer), Text)), + (r"^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)", bygroups(Text, Operator, Text)), + (r"(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$", bygroups(Text, Operator, Text)), ] class IPython3Lexer(Python3Lexer): """IPython code lexer (based on Python 3)""" + name = "IPython" aliases = ["ipython", "ipython3"] tokens = Python3Lexer.tokens.copy() - tokens['root'] = ipython_tokens + tokens['root'] + tokens["root"] = ipython_tokens + tokens["root"] IPythonLexer = IPython3Lexer @@ -111,10 +184,11 @@ class IPythonPartialTracebackLexer(RegexLexer): Handles all the non-python output. """ - name = 'IPython Partial Traceback' + + name = "IPython Partial Traceback" tokens = { - 'root': [ + "root": [ # Tracebacks for syntax errors have a different style. # For both types of tracebacks, we mark the first line with # Generic.Traceback. For syntax errors, we mark the filename @@ -124,31 +198,43 @@ class IPythonPartialTracebackLexer(RegexLexer): # traceback. # ## Non-syntax traceback - (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)), + (r"^(\^C)?(-+\n)", bygroups(Error, Generic.Traceback)), ## Syntax traceback - (r'^( File)(.*)(, line )(\d+\n)', - bygroups(Generic.Traceback, Name.Namespace, - Generic.Traceback, Literal.Number.Integer)), - + ( + r"^( File)(.*)(, line )(\d+\n)", + bygroups( + Generic.Traceback, + Name.Namespace, + Generic.Traceback, + Literal.Number.Integer, + ), + ), # (Exception Identifier)(Whitespace)(Traceback Message) - (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)', - bygroups(Name.Exception, Generic.Whitespace, Text)), + ( + r"(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)", + bygroups(Name.Exception, Generic.Whitespace, Text), + ), # (Module/Filename)(Text)(Callee)(Function Signature) # Better options for callee and function signature? - (r'(.*)( in )(.*)(\(.*\)\n)', - bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)), + ( + r"(.*)( in )(.*)(\(.*\)\n)", + bygroups(Name.Namespace, Text, Name.Entity, Name.Tag), + ), # Regular line: (Whitespace)(Line Number)(Python Code) - (r'(\s*?)(\d+)(.*?\n)', - bygroups(Generic.Whitespace, Literal.Number.Integer, Other)), + ( + r"(\s*?)(\d+)(.*?\n)", + bygroups(Generic.Whitespace, Literal.Number.Integer, Other), + ), # Emphasized line: (Arrow)(Line Number)(Python Code) # Using Exception token so arrow color matches the Exception. - (r'(-*>?\s?)(\d+)(.*?\n)', - bygroups(Name.Exception, Literal.Number.Integer, Other)), + ( + r"(-*>?\s?)(\d+)(.*?\n)", + bygroups(Name.Exception, Literal.Number.Integer, Other), + ), # (Exception Identifier)(Message) - (r'(?u)(^[^\d\W]\w*)(:.*?\n)', - bygroups(Name.Exception, Text)), + (r"(?u)(^[^\d\W]\w*)(:.*?\n)", bygroups(Name.Exception, Text)), # Tag everything else as Other, will be handled later. - (r'.*\n', Other), + (r".*\n", Other), ], } @@ -163,14 +249,15 @@ class IPythonTracebackLexer(DelegatingLexer): this is the line which lists the File and line number. """ + # The lexer inherits from DelegatingLexer. The "root" lexer is an # appropriate IPython lexer, which depends on the value of the boolean # `python3`. First, we parse with the partial IPython traceback lexer. # Then, any code marked with the "Other" token is delegated to the root # lexer. # - name = 'IPython Traceback' - aliases = ['ipythontb', 'ipython3tb'] + name = "IPython Traceback" + aliases = ["ipythontb", "ipython3tb"] def __init__(self, **options): """ @@ -180,8 +267,10 @@ def __init__(self, **options): # note we need a __init__ doc, as otherwise it inherits the doc from the super class # which will fail the documentation build as it references section of the pygments docs that # do not exists when building IPython's docs. - DelegatingLexer.__init__(self, IPython3Lexer, - IPythonPartialTracebackLexer, **options) + DelegatingLexer.__init__( + self, IPython3Lexer, IPythonPartialTracebackLexer, **options + ) + class IPythonConsoleLexer(Lexer): """ @@ -212,9 +301,10 @@ class IPythonConsoleLexer(Lexer): Exception """ - name = 'IPython console session' - aliases = ['ipythonconsole', 'ipython3console'] - mimetypes = ['text/x-ipython-console'] + + name = "IPython console session" + aliases = ["ipythonconsole", "ipython3console"] + mimetypes = ["text/x-ipython-console"] # The regexps used to determine what is input and what is output. # The default prompts for IPython are: @@ -223,15 +313,15 @@ class IPythonConsoleLexer(Lexer): # continuation = ' .D.: ' # template = 'Out[#]: ' # - # Where '#' is the 'prompt number' or 'execution count' and 'D' - # D is a number of dots matching the width of the execution count + # Where '#' is the 'prompt number' or 'execution count' and 'D' + # D is a number of dots matching the width of the execution count # - in1_regex = r'In \[[0-9]+\]: ' - in2_regex = r' \.\.+\.: ' - out_regex = r'Out\[[0-9]+\]: ' + in1_regex = r"In \[[0-9]+\]: " + in2_regex = r" \.\.+\.: " + out_regex = r"Out\[[0-9]+\]: " #: The regex to determine when a traceback starts. - ipytb_start = re.compile(r'^(\^C)?(-+\n)|^( File)(.*)(, line )(\d+\n)') + ipytb_start = re.compile(r"^(\^C)?(-+\n)|^( File)(.*)(, line )(\d+\n)") def __init__(self, **options): """Initialize the IPython console lexer. @@ -253,9 +343,9 @@ def __init__(self, **options): then the default output prompt is assumed. """ - in1_regex = options.get('in1_regex', self.in1_regex) - in2_regex = options.get('in2_regex', self.in2_regex) - out_regex = options.get('out_regex', self.out_regex) + in1_regex = options.get("in1_regex", self.in1_regex) + in2_regex = options.get("in2_regex", self.in2_regex) + out_regex = options.get("out_regex", self.out_regex) # So that we can work with input and output prompts which have been # rstrip'd (possibly by editors) we also need rstrip'd variants. If @@ -265,13 +355,19 @@ def __init__(self, **options): # with the token. This allows formatted code to be modified so as hide # the appearance of prompts, with the whitespace included. One example # use of this is in copybutton.js from the standard lib Python docs. - in1_regex_rstrip = in1_regex.rstrip() + '\n' - in2_regex_rstrip = in2_regex.rstrip() + '\n' - out_regex_rstrip = out_regex.rstrip() + '\n' + in1_regex_rstrip = in1_regex.rstrip() + "\n" + in2_regex_rstrip = in2_regex.rstrip() + "\n" + out_regex_rstrip = out_regex.rstrip() + "\n" # Compile and save them all. - attrs = ['in1_regex', 'in2_regex', 'out_regex', - 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip'] + attrs = [ + "in1_regex", + "in2_regex", + "out_regex", + "in1_regex_rstrip", + "in2_regex_rstrip", + "out_regex_rstrip", + ] for attr in attrs: self.__setattr__(attr, re.compile(locals()[attr])) @@ -283,9 +379,9 @@ def __init__(self, **options): self.reset() def reset(self): - self.mode = 'output' + self.mode = "output" self.index = 0 - self.buffer = u'' + self.buffer = "" self.insertions = [] def buffered_tokens(self): @@ -294,11 +390,11 @@ def buffered_tokens(self): changing to a new state. """ - if self.mode == 'output': + if self.mode == "output": tokens = [(0, Generic.Output, self.buffer)] - elif self.mode == 'input': + elif self.mode == "input": tokens = self.pylexer.get_tokens_unprocessed(self.buffer) - else: # traceback + else: # traceback tokens = self.tblexer.get_tokens_unprocessed(self.buffer) for i, t, v in do_insertions(self.insertions, tokens): @@ -307,7 +403,7 @@ def buffered_tokens(self): # Clear it all self.index += len(self.buffer) - self.buffer = u'' + self.buffer = "" self.insertions = [] def get_mci(self, line): @@ -336,16 +432,17 @@ def get_mci(self, line): # Check for possible end of input in2_match = self.in2_regex.match(line) in2_match_rstrip = self.in2_regex_rstrip.match(line) - if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \ - in2_match_rstrip: + if ( + in2_match and in2_match.group().rstrip() == line.rstrip() + ) or in2_match_rstrip: end_input = True else: end_input = False - if end_input and self.mode != 'tb': + if end_input and self.mode != "tb": # Only look for an end of input when not in tb mode. # An ellipsis could appear within the traceback. - mode = 'output' - code = u'' + mode = "output" + code = "" insertion = (0, Generic.Prompt, line) return mode, code, insertion @@ -353,7 +450,7 @@ def get_mci(self, line): out_match = self.out_regex.match(line) out_match_rstrip = self.out_regex_rstrip.match(line) if out_match or out_match_rstrip: - mode = 'output' + mode = "output" if out_match: idx = out_match.end() else: @@ -364,17 +461,16 @@ def get_mci(self, line): insertion = (0, Generic.Heading, line[:idx]) return mode, code, insertion - # Check for input or continuation prompt (non stripped version) in1_match = self.in1_regex.match(line) - if in1_match or (in2_match and self.mode != 'tb'): + if in1_match or (in2_match and self.mode != "tb"): # New input or when not in tb, continued input. # We do not check for continued input when in tb since it is # allowable to replace a long stack with an ellipsis. - mode = 'input' + mode = "input" if in1_match: idx = in1_match.end() - else: # in2_match + else: # in2_match idx = in2_match.end() code = line[idx:] insertion = (0, Generic.Prompt, line[:idx]) @@ -382,14 +478,14 @@ def get_mci(self, line): # Check for input or continuation prompt (stripped version) in1_match_rstrip = self.in1_regex_rstrip.match(line) - if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'): + if in1_match_rstrip or (in2_match_rstrip and self.mode != "tb"): # New input or when not in tb, continued input. # We do not check for continued input when in tb since it is # allowable to replace a long stack with an ellipsis. - mode = 'input' + mode = "input" if in1_match_rstrip: idx = in1_match_rstrip.end() - else: # in2_match + else: # in2_match idx = in2_match_rstrip.end() code = line[idx:] insertion = (0, Generic.Prompt, line[:idx]) @@ -397,13 +493,13 @@ def get_mci(self, line): # Check for traceback if self.ipytb_start.match(line): - mode = 'tb' + mode = "tb" code = line insertion = None return mode, code, insertion # All other stuff... - if self.mode in ('input', 'output'): + if self.mode in ("input", "output"): # We assume all other text is output. Multiline input that # does not use the continuation marker cannot be detected. # For example, the 3 in the following is clearly output: @@ -418,9 +514,9 @@ def get_mci(self, line): # # In both cases, the 2nd line will be 'output'. # - mode = 'output' + mode = "output" else: - mode = 'tb' + mode = "tb" code = line insertion = None @@ -446,6 +542,7 @@ def get_tokens_unprocessed(self, text): for token in self.buffered_tokens(): yield token + class IPyLexer(Lexer): r""" Primary lexer for all IPython-like code. @@ -458,8 +555,9 @@ class IPyLexer(Lexer): with Pygments. """ - name = 'IPy session' - aliases = ['ipy', 'ipy3'] + + name = "IPy session" + aliases = ["ipy", "ipy3"] def __init__(self, **options): """ @@ -477,10 +575,9 @@ def __init__(self, **options): def get_tokens_unprocessed(self, text): # Search for the input prompt anywhere...this allows code blocks to # begin with comments as well. - if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL): + if re.match(r".*(In \[[0-9]+\]:)", text.strip(), re.DOTALL): lex = self.IPythonConsoleLexer else: lex = self.IPythonLexer for token in lex.get_tokens_unprocessed(text): yield token - diff --git a/test_ipython_pygments_lexers.py b/test_ipython_pygments_lexers.py index b9323c1..2893014 100644 --- a/test_ipython_pygments_lexers.py +++ b/test_ipython_pygments_lexers.py @@ -15,14 +15,15 @@ class TestLexers(TestCase): """Collection of lexers tests""" + def setUp(self): self.lexer = lexers.IPythonLexer() self.bash_lexer = BashLexer() def testIPythonLexer(self): - fragment = '!echo $HOME\n' + fragment = "!echo $HOME\n" bash_tokens = [ - (Token.Operator, '!'), + (Token.Operator, "!"), ] bash_tokens.extend(self.bash_lexer.get_tokens(fragment[1:])) ipylex_token = list(self.lexer.get_tokens(fragment)) @@ -30,155 +31,155 @@ def testIPythonLexer(self): fragment_2 = "!" + fragment tokens_2 = [ - (Token.Operator, '!!'), + (Token.Operator, "!!"), ] + bash_tokens[1:] assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] - fragment_2 = '\t %%!\n' + fragment[1:] + fragment_2 = "\t %%!\n" + fragment[1:] tokens_2 = [ - (Token.Text, '\t '), - (Token.Operator, '%%!'), - (Token.Text, '\n'), + (Token.Text, "\t "), + (Token.Operator, "%%!"), + (Token.Text, "\n"), ] + bash_tokens[1:] assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) - fragment_2 = 'x = ' + fragment + fragment_2 = "x = " + fragment tokens_2 = [ - (Token.Name, 'x'), - (Token.Text, ' '), - (Token.Operator, '='), - (Token.Text, ' '), + (Token.Name, "x"), + (Token.Text, " "), + (Token.Operator, "="), + (Token.Text, " "), ] + bash_tokens assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] - fragment_2 = 'x, = ' + fragment + fragment_2 = "x, = " + fragment tokens_2 = [ - (Token.Name, 'x'), - (Token.Punctuation, ','), - (Token.Text, ' '), - (Token.Operator, '='), - (Token.Text, ' '), + (Token.Name, "x"), + (Token.Punctuation, ","), + (Token.Text, " "), + (Token.Operator, "="), + (Token.Text, " "), ] + bash_tokens assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] - fragment_2 = 'x, = %sx ' + fragment[1:] + fragment_2 = "x, = %sx " + fragment[1:] tokens_2 = [ - (Token.Name, 'x'), - (Token.Punctuation, ','), - (Token.Text, ' '), - (Token.Operator, '='), - (Token.Text, ' '), - (Token.Operator, '%'), - (Token.Keyword, 'sx'), - (Token.Text, ' '), + (Token.Name, "x"), + (Token.Punctuation, ","), + (Token.Text, " "), + (Token.Operator, "="), + (Token.Text, " "), + (Token.Operator, "%"), + (Token.Keyword, "sx"), + (Token.Text, " "), ] + bash_tokens[1:] if tokens_2[7] == (Token.Text, " ") and pyg214: # pygments 2.14+ tokens_2[7] = (Token.Text.Whitespace, " ") assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] - fragment_2 = 'f = %R function () {}\n' + fragment_2 = "f = %R function () {}\n" tokens_2 = [ - (Token.Name, 'f'), - (Token.Text, ' '), - (Token.Operator, '='), - (Token.Text, ' '), - (Token.Operator, '%'), - (Token.Keyword, 'R'), - (Token.Text, ' function () {}\n'), + (Token.Name, "f"), + (Token.Text, " "), + (Token.Operator, "="), + (Token.Text, " "), + (Token.Operator, "%"), + (Token.Keyword, "R"), + (Token.Text, " function () {}\n"), ] assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) - fragment_2 = '\t%%xyz\n$foo\n' + fragment_2 = "\t%%xyz\n$foo\n" tokens_2 = [ - (Token.Text, '\t'), - (Token.Operator, '%%'), - (Token.Keyword, 'xyz'), - (Token.Text, '\n$foo\n'), + (Token.Text, "\t"), + (Token.Operator, "%%"), + (Token.Keyword, "xyz"), + (Token.Text, "\n$foo\n"), ] assert tokens_2 == list(self.lexer.get_tokens(fragment_2)) - fragment_2 = '%system?\n' + fragment_2 = "%system?\n" tokens_2 = [ - (Token.Operator, '%'), - (Token.Keyword, 'system'), - (Token.Operator, '?'), - (Token.Text, '\n'), + (Token.Operator, "%"), + (Token.Keyword, "system"), + (Token.Operator, "?"), + (Token.Text, "\n"), ] assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] - fragment_2 = 'x != y\n' + fragment_2 = "x != y\n" tokens_2 = [ - (Token.Name, 'x'), - (Token.Text, ' '), - (Token.Operator, '!='), - (Token.Text, ' '), - (Token.Name, 'y'), - (Token.Text, '\n'), + (Token.Name, "x"), + (Token.Text, " "), + (Token.Operator, "!="), + (Token.Text, " "), + (Token.Name, "y"), + (Token.Text, "\n"), ] assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] - fragment_2 = ' ?math.sin\n' + fragment_2 = " ?math.sin\n" tokens_2 = [ - (Token.Text, ' '), - (Token.Operator, '?'), - (Token.Text, 'math.sin'), - (Token.Text, '\n'), + (Token.Text, " "), + (Token.Operator, "?"), + (Token.Text, "math.sin"), + (Token.Text, "\n"), ] assert tokens_2[:-1] == list(self.lexer.get_tokens(fragment_2))[:-1] - fragment = ' *int*?\n' + fragment = " *int*?\n" tokens = [ - (Token.Text, ' *int*'), - (Token.Operator, '?'), - (Token.Text, '\n'), + (Token.Text, " *int*"), + (Token.Operator, "?"), + (Token.Text, "\n"), ] assert tokens == list(self.lexer.get_tokens(fragment)) - fragment = '%%writefile -a foo.py\nif a == b:\n pass' + fragment = "%%writefile -a foo.py\nif a == b:\n pass" tokens = [ - (Token.Operator, '%%writefile'), - (Token.Text, ' -a foo.py\n'), - (Token.Keyword, 'if'), - (Token.Text, ' '), - (Token.Name, 'a'), - (Token.Text, ' '), - (Token.Operator, '=='), - (Token.Text, ' '), - (Token.Name, 'b'), - (Token.Punctuation, ':'), - (Token.Text, '\n'), - (Token.Text, ' '), - (Token.Keyword, 'pass'), - (Token.Text, '\n'), + (Token.Operator, "%%writefile"), + (Token.Text, " -a foo.py\n"), + (Token.Keyword, "if"), + (Token.Text, " "), + (Token.Name, "a"), + (Token.Text, " "), + (Token.Operator, "=="), + (Token.Text, " "), + (Token.Name, "b"), + (Token.Punctuation, ":"), + (Token.Text, "\n"), + (Token.Text, " "), + (Token.Keyword, "pass"), + (Token.Text, "\n"), ] if tokens[10] == (Token.Text, "\n") and pyg214: # pygments 2.14+ tokens[10] = (Token.Text.Whitespace, "\n") assert tokens[:-1] == list(self.lexer.get_tokens(fragment))[:-1] - fragment = '%%timeit\nmath.sin(0)' + fragment = "%%timeit\nmath.sin(0)" tokens = [ - (Token.Operator, '%%timeit\n'), - (Token.Name, 'math'), - (Token.Operator, '.'), - (Token.Name, 'sin'), - (Token.Punctuation, '('), - (Token.Literal.Number.Integer, '0'), - (Token.Punctuation, ')'), - (Token.Text, '\n'), + (Token.Operator, "%%timeit\n"), + (Token.Name, "math"), + (Token.Operator, "."), + (Token.Name, "sin"), + (Token.Punctuation, "("), + (Token.Literal.Number.Integer, "0"), + (Token.Punctuation, ")"), + (Token.Text, "\n"), ] - fragment = '%%HTML\n
foo
' + fragment = "%%HTML\n
foo
" tokens = [ - (Token.Operator, '%%HTML'), - (Token.Text, '\n'), - (Token.Punctuation, '<'), - (Token.Name.Tag, 'div'), - (Token.Punctuation, '>'), - (Token.Text, 'foo'), - (Token.Punctuation, '<'), - (Token.Punctuation, '/'), - (Token.Name.Tag, 'div'), - (Token.Punctuation, '>'), - (Token.Text, '\n'), + (Token.Operator, "%%HTML"), + (Token.Text, "\n"), + (Token.Punctuation, "<"), + (Token.Name.Tag, "div"), + (Token.Punctuation, ">"), + (Token.Text, "foo"), + (Token.Punctuation, "<"), + (Token.Punctuation, "/"), + (Token.Name.Tag, "div"), + (Token.Punctuation, ">"), + (Token.Text, "\n"), ] assert tokens == list(self.lexer.get_tokens(fragment))