pax_global_header00006660000000000000000000000064147424366570014534gustar00rootroot0000000000000052 comment=76f70093e2d354e6a2f4f42a7aed53018eff580c ipython-pygments-lexers-1.1.1/000077500000000000000000000000001474243665700163725ustar00rootroot00000000000000ipython-pygments-lexers-1.1.1/.github/000077500000000000000000000000001474243665700177325ustar00rootroot00000000000000ipython-pygments-lexers-1.1.1/.github/dependabot.yml000066400000000000000000000002411474243665700225570ustar00rootroot00000000000000version: 2 updates: # Maintain dependencies for GitHub Actions - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" ipython-pygments-lexers-1.1.1/.github/workflows/000077500000000000000000000000001474243665700217675ustar00rootroot00000000000000ipython-pygments-lexers-1.1.1/.github/workflows/test.yml000066400000000000000000000027741474243665700235030ustar00rootroot00000000000000name: Tests on: push: branches: [ main ] tags: [ '*' ] pull_request: branches: [ main ] workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: tests: runs-on: ubuntu-latest strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('pyproject.toml') }} - name: Install dependencies run: | python3 -m pip install --upgrade pip python3 -m pip install "." pytest - name: Test with pytest run: | python3 -m pytest -v publish: runs-on: ubuntu-latest if: ${{ startsWith(github.ref, 'refs/tags/') }} needs: tests permissions: id-token: write # OIDC for uploading to PyPI steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.11" - name: Build packages run: | python3 -m pip install build python3 -m build - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 ipython-pygments-lexers-1.1.1/.gitignore000066400000000000000000000000321474243665700203550ustar00rootroot00000000000000build/ dist/ __pycache__/ ipython-pygments-lexers-1.1.1/LICENSE000066400000000000000000000027771474243665700174140ustar00rootroot00000000000000BSD 3-Clause License - Copyright (c) 2012-Present, IPython Development Team All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ipython-pygments-lexers-1.1.1/README.md000066400000000000000000000012301474243665700176450ustar00rootroot00000000000000A [Pygments](https://pygments.org/) plugin for IPython code & console sessions [IPython](https://ipython.org/) is an interactive Python shell. Among other features, it adds some special convenience syntax, including `%magics`, `!shell commands` and `help?`. This package contains lexers for these, to use with the Pygments syntax highlighting package. - The `ipython` lexer should be used where only input code is highlighted - The `ipythonconsole` lexer works for an IPython session, including code, prompts, output and tracebacks. These lexers were previously part of IPython itself (in `IPython.lib.lexers`), but have now been moved to a separate package. ipython-pygments-lexers-1.1.1/ipython_pygments_lexers.py000066400000000000000000000463101474243665700237520ustar00rootroot00000000000000# -*- coding: utf-8 -*- """ Defines a variety of Pygments lexers for highlighting IPython code. This includes: IPythonLexer, IPython3Lexer Lexers for pure IPython (python + magic/shell commands) IPythonPartialTracebackLexer, IPythonTracebackLexer Supports 2.x and 3.x via keyword `python3`. The partial traceback lexer reads everything but the Python code appearing in a traceback. The full lexer combines the partial lexer with an IPython lexer. IPythonConsoleLexer A lexer for IPython console sessions, with support for tracebacks. IPyLexer A friendly lexer which examines the first line of text and from it, decides whether to use an IPython lexer or an IPython console lexer. This is probably the only lexer that needs to be explicitly added to Pygments. """ # ----------------------------------------------------------------------------- # Copyright (c) 2013, the IPython Development Team. # # Distributed under the terms of the Modified BSD License. # # The full license is in the file COPYING.txt, distributed with this software. # ----------------------------------------------------------------------------- __version__ = "1.1.1" # Standard library import re # Third party from pygments.lexers import ( BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, Python2Lexer, Python3Lexer, TexLexer, ) from pygments.lexer import ( Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using, ) from pygments.token import ( Generic, Keyword, Literal, Name, Operator, Other, Text, Error, ) line_re = re.compile(".*?\n") __all__ = [ "IPython3Lexer", "IPythonLexer", "IPythonPartialTracebackLexer", "IPythonTracebackLexer", "IPythonConsoleLexer", "IPyLexer", ] ipython_tokens = [ ( r"(?s)(\s*)(%%capture)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?s)(\s*)(%%debug)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?is)(\s*)(%%html)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(HtmlLexer)), ), ( r"(?s)(\s*)(%%javascript)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(JavascriptLexer)), ), ( r"(?s)(\s*)(%%js)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(JavascriptLexer)), ), ( r"(?s)(\s*)(%%latex)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(TexLexer)), ), ( r"(?s)(\s*)(%%perl)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(PerlLexer)), ), ( r"(?s)(\s*)(%%prun)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?s)(\s*)(%%pypy)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?s)(\s*)(%%python2)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python2Lexer)), ), ( r"(?s)(\s*)(%%python3)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?s)(\s*)(%%python)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?s)(\s*)(%%ruby)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(RubyLexer)), ), ( r"(?s)(\s*)(%%timeit)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?s)(\s*)(%%time)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?s)(\s*)(%%writefile)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), ( r"(?s)(\s*)(%%file)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(Python3Lexer)), ), (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)), ( r"(?s)(^\s*)(%%!)([^\n]*\n)(.*)", bygroups(Text, Operator, Text, using(BashLexer)), ), (r"(%%?)(\w+)(\?\??)$", bygroups(Operator, Keyword, Operator)), (r"\b(\?\??)(\s*)$", bygroups(Operator, Text)), (r"(%)(sx|sc|system)(.*)(\n)", bygroups(Operator, Keyword, using(BashLexer), Text)), (r"(%)(\w+)(.*\n)", bygroups(Operator, Keyword, Text)), (r"^(!!)(.+)(\n)", bygroups(Operator, using(BashLexer), Text)), (r"(!)(?!=)(.+)(\n)", bygroups(Operator, using(BashLexer), Text)), (r"^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)", bygroups(Text, Operator, Text)), (r"(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$", bygroups(Text, Operator, Text)), ] class IPython3Lexer(Python3Lexer): """IPython code lexer (based on Python 3)""" name = "IPython" aliases = ["ipython", "ipython3"] tokens = Python3Lexer.tokens.copy() tokens["root"] = ipython_tokens + tokens["root"] IPythonLexer = IPython3Lexer class IPythonPartialTracebackLexer(RegexLexer): """ Partial lexer for IPython tracebacks. Handles all the non-python output. """ name = "IPython Partial Traceback" tokens = { "root": [ # Tracebacks for syntax errors have a different style. # For both types of tracebacks, we mark the first line with # Generic.Traceback. For syntax errors, we mark the filename # as we mark the filenames for non-syntax tracebacks. # # These two regexps define how IPythonConsoleLexer finds a # traceback. # ## Non-syntax traceback (r"^(\^C)?(-+\n)", bygroups(Error, Generic.Traceback)), ## Syntax traceback ( r"^( File)(.*)(, line )(\d+\n)", bygroups( Generic.Traceback, Name.Namespace, Generic.Traceback, Literal.Number.Integer, ), ), # (Exception Identifier)(Whitespace)(Traceback Message) ( r"(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)", bygroups(Name.Exception, Generic.Whitespace, Text), ), # (Module/Filename)(Text)(Callee)(Function Signature) # Better options for callee and function signature? ( r"(.*)( in )(.*)(\(.*\)\n)", bygroups(Name.Namespace, Text, Name.Entity, Name.Tag), ), # Regular line: (Whitespace)(Line Number)(Python Code) ( r"(\s*?)(\d+)(.*?\n)", bygroups(Generic.Whitespace, Literal.Number.Integer, Other), ), # Emphasized line: (Arrow)(Line Number)(Python Code) # Using Exception token so arrow color matches the Exception. ( r"(-*>?\s?)(\d+)(.*?\n)", bygroups(Name.Exception, Literal.Number.Integer, Other), ), # (Exception Identifier)(Message) (r"(?u)(^[^\d\W]\w*)(:.*?\n)", bygroups(Name.Exception, Text)), # Tag everything else as Other, will be handled later. (r".*\n", Other), ], } class IPythonTracebackLexer(DelegatingLexer): """ IPython traceback lexer. For doctests, the tracebacks can be snipped as much as desired with the exception to the lines that designate a traceback. For non-syntax error tracebacks, this is the line of hyphens. For syntax error tracebacks, this is the line which lists the File and line number. """ # The lexer inherits from DelegatingLexer. The "root" lexer is an # appropriate IPython lexer, which depends on the value of the boolean # `python3`. First, we parse with the partial IPython traceback lexer. # Then, any code marked with the "Other" token is delegated to the root # lexer. # name = "IPython Traceback" aliases = ["ipythontb", "ipython3tb"] def __init__(self, **options): """ A subclass of `DelegatingLexer` which delegates to the appropriate to either IPyLexer, IPythonPartialTracebackLexer. """ # note we need a __init__ doc, as otherwise it inherits the doc from the super class # which will fail the documentation build as it references section of the pygments docs that # do not exists when building IPython's docs. DelegatingLexer.__init__( self, IPython3Lexer, IPythonPartialTracebackLexer, **options ) class IPythonConsoleLexer(Lexer): """ An IPython console lexer for IPython code-blocks and doctests, such as: .. code-block:: rst .. code-block:: ipythonconsole In [1]: a = 'foo' In [2]: a Out[2]: 'foo' In [3]: print(a) foo Support is also provided for IPython exceptions: .. code-block:: rst .. code-block:: ipythonconsole In [1]: raise Exception Traceback (most recent call last): ... Exception """ name = "IPython console session" aliases = ["ipythonconsole", "ipython3console"] mimetypes = ["text/x-ipython-console"] # The regexps used to determine what is input and what is output. # The default prompts for IPython are: # # in = 'In [#]: ' # continuation = ' .D.: ' # template = 'Out[#]: ' # # Where '#' is the 'prompt number' or 'execution count' and 'D' # D is a number of dots matching the width of the execution count # in1_regex = r"In \[[0-9]+\]: " in2_regex = r" \.\.+\.: " out_regex = r"Out\[[0-9]+\]: " #: The regex to determine when a traceback starts. ipytb_start = re.compile(r"^(\^C)?(-+\n)|^( File)(.*)(, line )(\d+\n)") def __init__(self, **options): """Initialize the IPython console lexer. Parameters ---------- in1_regex : RegexObject The compiled regular expression used to detect the start of inputs. Although the IPython configuration setting may have a trailing whitespace, do not include it in the regex. If `None`, then the default input prompt is assumed. in2_regex : RegexObject The compiled regular expression used to detect the continuation of inputs. Although the IPython configuration setting may have a trailing whitespace, do not include it in the regex. If `None`, then the default input prompt is assumed. out_regex : RegexObject The compiled regular expression used to detect outputs. If `None`, then the default output prompt is assumed. """ in1_regex = options.get("in1_regex", self.in1_regex) in2_regex = options.get("in2_regex", self.in2_regex) out_regex = options.get("out_regex", self.out_regex) # So that we can work with input and output prompts which have been # rstrip'd (possibly by editors) we also need rstrip'd variants. If # we do not do this, then such prompts will be tagged as 'output'. # The reason can't just use the rstrip'd variants instead is because # we want any whitespace associated with the prompt to be inserted # with the token. This allows formatted code to be modified so as hide # the appearance of prompts, with the whitespace included. One example # use of this is in copybutton.js from the standard lib Python docs. in1_regex_rstrip = in1_regex.rstrip() + "\n" in2_regex_rstrip = in2_regex.rstrip() + "\n" out_regex_rstrip = out_regex.rstrip() + "\n" # Compile and save them all. attrs = [ "in1_regex", "in2_regex", "out_regex", "in1_regex_rstrip", "in2_regex_rstrip", "out_regex_rstrip", ] for attr in attrs: self.__setattr__(attr, re.compile(locals()[attr])) Lexer.__init__(self, **options) self.pylexer = IPython3Lexer(**options) self.tblexer = IPythonTracebackLexer(**options) self.reset() def reset(self): self.mode = "output" self.index = 0 self.buffer = "" self.insertions = [] def buffered_tokens(self): """ Generator of unprocessed tokens after doing insertions and before changing to a new state. """ if self.mode == "output": tokens = [(0, Generic.Output, self.buffer)] elif self.mode == "input": tokens = self.pylexer.get_tokens_unprocessed(self.buffer) else: # traceback tokens = self.tblexer.get_tokens_unprocessed(self.buffer) for i, t, v in do_insertions(self.insertions, tokens): # All token indexes are relative to the buffer. yield self.index + i, t, v # Clear it all self.index += len(self.buffer) self.buffer = "" self.insertions = [] def get_mci(self, line): """ Parses the line and returns a 3-tuple: (mode, code, insertion). `mode` is the next mode (or state) of the lexer, and is always equal to 'input', 'output', or 'tb'. `code` is a portion of the line that should be added to the buffer corresponding to the next mode and eventually lexed by another lexer. For example, `code` could be Python code if `mode` were 'input'. `insertion` is a 3-tuple (index, token, text) representing an unprocessed "token" that will be inserted into the stream of tokens that are created from the buffer once we change modes. This is usually the input or output prompt. In general, the next mode depends on current mode and on the contents of `line`. """ # To reduce the number of regex match checks, we have multiple # 'if' blocks instead of 'if-elif' blocks. # Check for possible end of input in2_match = self.in2_regex.match(line) in2_match_rstrip = self.in2_regex_rstrip.match(line) if ( in2_match and in2_match.group().rstrip() == line.rstrip() ) or in2_match_rstrip: end_input = True else: end_input = False if end_input and self.mode != "tb": # Only look for an end of input when not in tb mode. # An ellipsis could appear within the traceback. mode = "output" code = "" insertion = (0, Generic.Prompt, line) return mode, code, insertion # Check for output prompt out_match = self.out_regex.match(line) out_match_rstrip = self.out_regex_rstrip.match(line) if out_match or out_match_rstrip: mode = "output" if out_match: idx = out_match.end() else: idx = out_match_rstrip.end() code = line[idx:] # Use the 'heading' token for output. We cannot use Generic.Error # since it would conflict with exceptions. insertion = (0, Generic.Heading, line[:idx]) return mode, code, insertion # Check for input or continuation prompt (non stripped version) in1_match = self.in1_regex.match(line) if in1_match or (in2_match and self.mode != "tb"): # New input or when not in tb, continued input. # We do not check for continued input when in tb since it is # allowable to replace a long stack with an ellipsis. mode = "input" if in1_match: idx = in1_match.end() else: # in2_match idx = in2_match.end() code = line[idx:] insertion = (0, Generic.Prompt, line[:idx]) return mode, code, insertion # Check for input or continuation prompt (stripped version) in1_match_rstrip = self.in1_regex_rstrip.match(line) if in1_match_rstrip or (in2_match_rstrip and self.mode != "tb"): # New input or when not in tb, continued input. # We do not check for continued input when in tb since it is # allowable to replace a long stack with an ellipsis. mode = "input" if in1_match_rstrip: idx = in1_match_rstrip.end() else: # in2_match idx = in2_match_rstrip.end() code = line[idx:] insertion = (0, Generic.Prompt, line[:idx]) return mode, code, insertion # Check for traceback if self.ipytb_start.match(line): mode = "tb" code = line insertion = None return mode, code, insertion # All other stuff... if self.mode in ("input", "output"): # We assume all other text is output. Multiline input that # does not use the continuation marker cannot be detected. # For example, the 3 in the following is clearly output: # # In [1]: print(3) # 3 # # But the following second line is part of the input: # # In [2]: while True: # print(True) # # In both cases, the 2nd line will be 'output'. # mode = "output" else: mode = "tb" code = line insertion = None return mode, code, insertion def get_tokens_unprocessed(self, text): self.reset() for match in line_re.finditer(text): line = match.group() mode, code, insertion = self.get_mci(line) if mode != self.mode: # Yield buffered tokens before transitioning to new mode. for token in self.buffered_tokens(): yield token self.mode = mode if insertion: self.insertions.append((len(self.buffer), [insertion])) self.buffer += code for token in self.buffered_tokens(): yield token class IPyLexer(Lexer): r""" Primary lexer for all IPython-like code. This is a simple helper lexer. If the first line of the text begins with "In \[[0-9]+\]:", then the entire text is parsed with an IPython console lexer. If not, then the entire text is parsed with an IPython lexer. The goal is to reduce the number of lexers that are registered with Pygments. """ name = "IPy session" aliases = ["ipy", "ipy3"] def __init__(self, **options): """ Create a new IPyLexer instance which dispatch to either an IPythonCOnsoleLexer (if In prompts are present) or and IPythonLexer (if In prompts are not present). """ # init docstring is necessary for docs not to fail to build do to parent # docs referenceing a section in pygments docs. Lexer.__init__(self, **options) self.IPythonLexer = IPythonLexer(**options) self.IPythonConsoleLexer = IPythonConsoleLexer(**options) def get_tokens_unprocessed(self, text): # Search for the input prompt anywhere...this allows code blocks to # begin with comments as well. if re.match(r".*(In \[[0-9]+\]:)", text.strip(), re.DOTALL): lex = self.IPythonConsoleLexer else: lex = self.IPythonLexer for token in lex.get_tokens_unprocessed(text): yield token ipython-pygments-lexers-1.1.1/pyproject.toml000066400000000000000000000014271474243665700213120ustar00rootroot00000000000000[build-system] requires = ["flit_core >=3.2,<4"] build-backend = "flit_core.buildapi" [project] name = "ipython_pygments_lexers" authors = [ {name = "The IPython Development Team", email = "ipython-dev@python.org"}, ] readme = "README.md" license = {file = "LICENSE"} classifiers = [ "Framework :: IPython", "License :: OSI Approved :: BSD License", ] requires-python=">=3.8" dependencies = [ "pygments" ] dynamic=["version", "description"] [project.entry-points."pygments.lexers"] ipythonconsole = "ipython_pygments_lexers:IPythonConsoleLexer" ipython = "ipython_pygments_lexers:IPythonLexer" ipython3 = "ipython_pygments_lexers:IPython3Lexer" [project.urls] Source = "https://github.com/ipython/ipython-pygments-lexers" [tool.flit.sdist] include = [ "test_*.py", ] ipython-pygments-lexers-1.1.1/test_ipython_pygments_lexers.py000066400000000000000000000132621474243665700250110ustar00rootroot00000000000000"""Test lexers module""" # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. import pygments.lexers import pytest from pygments import __version__ as pygments_version from pygments.lexers import BashLexer from pygments.token import Token import ipython_pygments_lexers as lexers pyg214 = tuple(int(x) for x in pygments_version.split(".")[:2]) >= (2, 14) TOKEN_WS = Token.Text.Whitespace if pyg214 else Token.Text EXPECTED_LEXER_NAMES = [ cls.name for cls in [lexers.IPythonConsoleLexer, lexers.IPython3Lexer] ] @pytest.mark.parametrize("expected_lexer", EXPECTED_LEXER_NAMES) def test_pygments_entry_points(expected_lexer: str): """Check whether the ``entry_points`` for ``pygments.lexers`` are correct.""" all_pygments_lexer_names = {l[0] for l in pygments.lexers.get_all_lexers()} assert expected_lexer in all_pygments_lexer_names def test_plain_python(): lexer = lexers.IPythonLexer() fragment_2 = "x != y\n" tokens_2 = [ (Token.Name, "x"), (Token.Text, " "), (Token.Operator, "!="), (Token.Text, " "), (Token.Name, "y"), (Token.Text, "\n"), ] assert tokens_2[:-1] == list(lexer.get_tokens(fragment_2))[:-1] def test_shell_commands(): lexer = lexers.IPythonLexer() bash_lexer = BashLexer() fragment = "!echo $HOME\n" bash_tokens = [ (Token.Operator, "!"), ] bash_tokens.extend(bash_lexer.get_tokens(fragment[1:])) ipylex_token = list(lexer.get_tokens(fragment)) assert bash_tokens[:-1] == ipylex_token[:-1] fragment_2 = "!" + fragment tokens_2 = [ (Token.Operator, "!!"), ] + bash_tokens[1:] assert tokens_2[:-1] == list(lexer.get_tokens(fragment_2))[:-1] fragment_2 = "\t %%!\n" + fragment[1:] tokens_2 = [ (Token.Text, "\t "), (Token.Operator, "%%!"), (Token.Text, "\n"), ] + bash_tokens[1:] assert tokens_2 == list(lexer.get_tokens(fragment_2)) fragment_2 = "x = " + fragment tokens_2 = [ (Token.Name, "x"), (Token.Text, " "), (Token.Operator, "="), (Token.Text, " "), ] + bash_tokens assert tokens_2[:-1] == list(lexer.get_tokens(fragment_2))[:-1] fragment_2 = "x, = " + fragment tokens_2 = [ (Token.Name, "x"), (Token.Punctuation, ","), (Token.Text, " "), (Token.Operator, "="), (Token.Text, " "), ] + bash_tokens assert tokens_2[:-1] == list(lexer.get_tokens(fragment_2))[:-1] fragment_2 = "x, = %sx " + fragment[1:] tokens_2 = [ (Token.Name, "x"), (Token.Punctuation, ","), (Token.Text, " "), (Token.Operator, "="), (Token.Text, " "), (Token.Operator, "%"), (Token.Keyword, "sx"), (TOKEN_WS, " "), ] + bash_tokens[1:] assert tokens_2[:-1] == list(lexer.get_tokens(fragment_2))[:-1] def test_magics(): lexer = lexers.IPythonLexer() fragment_2 = "f = %R function () {}\n" tokens_2 = [ (Token.Name, "f"), (Token.Text, " "), (Token.Operator, "="), (Token.Text, " "), (Token.Operator, "%"), (Token.Keyword, "R"), (Token.Text, " function () {}\n"), ] assert tokens_2 == list(lexer.get_tokens(fragment_2)) fragment_2 = "%system?\n" tokens_2 = [ (Token.Operator, "%"), (Token.Keyword, "system"), (Token.Operator, "?"), (Token.Text, "\n"), ] assert tokens_2[:-1] == list(lexer.get_tokens(fragment_2))[:-1] def test_help(): lexer = lexers.IPythonLexer() fragment_2 = " ?math.sin\n" tokens_2 = [ (Token.Text, " "), (Token.Operator, "?"), (Token.Text, "math.sin"), (Token.Text, "\n"), ] assert tokens_2[:-1] == list(lexer.get_tokens(fragment_2))[:-1] fragment = " *int*?\n" tokens = [ (Token.Text, " *int*"), (Token.Operator, "?"), (Token.Text, "\n"), ] assert tokens == list(lexer.get_tokens(fragment)) def test_cell_magics(): lexer = lexers.IPythonLexer() fragment = "%%writefile -a foo.py\nif a == b:\n pass" tokens = [ (Token.Operator, "%%writefile"), (Token.Text, " -a foo.py\n"), (Token.Keyword, "if"), (Token.Text, " "), (Token.Name, "a"), (Token.Text, " "), (Token.Operator, "=="), (Token.Text, " "), (Token.Name, "b"), (Token.Punctuation, ":"), (TOKEN_WS, "\n"), (Token.Text, " "), (Token.Keyword, "pass"), (TOKEN_WS, "\n"), ] assert tokens == list(lexer.get_tokens(fragment)) fragment = "%%timeit\nmath.sin(0)" tokens = [ (Token.Operator, "%%timeit"), (Token.Text, "\n"), (Token.Name, "math"), (Token.Operator, "."), (Token.Name, "sin"), (Token.Punctuation, "("), (Token.Literal.Number.Integer, "0"), (Token.Punctuation, ")"), (TOKEN_WS, "\n"), ] assert tokens == list(lexer.get_tokens(fragment)) fragment = "%%HTML\n
foo
" tokens = [ (Token.Operator, "%%HTML"), (Token.Text, "\n"), (Token.Punctuation, "<"), (Token.Name.Tag, "div"), (Token.Punctuation, ">"), (Token.Text, "foo"), (Token.Punctuation, "<"), (Token.Punctuation, "/"), (Token.Name.Tag, "div"), (Token.Punctuation, ">"), (Token.Text, "\n"), ] assert tokens == list(lexer.get_tokens(fragment)) fragment_2 = "\t%%xyz\n$foo\n" tokens_2 = [ (Token.Text, "\t"), (Token.Operator, "%%"), (Token.Keyword, "xyz"), (Token.Text, "\n$foo\n"), ] assert tokens_2 == list(lexer.get_tokens(fragment_2))