Update Python lexers and add tests for them

lexers/p/python2.go

@@ -56,7 +56,7 @@ func python2Rules() Rules {
 		"builtins": {
 			{Words(`(?<!\.)`, `\b`, `__import__`, `abs`, `all`, `any`, `apply`, `basestring`, `bin`, `bool`, `buffer`, `bytearray`, `bytes`, `callable`, `chr`, `classmethod`, `cmp`, `coerce`, `compile`, `complex`, `delattr`, `dict`, `dir`, `divmod`, `enumerate`, `eval`, `execfile`, `exit`, `file`, `filter`, `float`, `frozenset`, `getattr`, `globals`, `hasattr`, `hash`, `hex`, `id`, `input`, `int`, `intern`, `isinstance`, `issubclass`, `iter`, `len`, `list`, `locals`, `long`, `map`, `max`, `min`, `next`, `object`, `oct`, `open`, `ord`, `pow`, `property`, `range`, `raw_input`, `reduce`, `reload`, `repr`, `reversed`, `round`, `set`, `setattr`, `slice`, `sorted`, `staticmethod`, `str`, `sum`, `super`, `tuple`, `type`, `unichr`, `unicode`, `vars`, `xrange`, `zip`), NameBuiltin, nil},
 			{`(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls)\b`, NameBuiltinPseudo, nil},
-			{Words(`(?<!\.)`, `\b`, `ArithmeticError`, `AssertionError`, `AttributeError`, `BaseException`, `DeprecationWarning`, `EOFError`, `EnvironmentError`, `Exception`, `FloatingPointError`, `FutureWarning`, `GeneratorExit`, `IOError`, `ImportError`, `ImportWarning`, `IndentationError`, `IndexError`, `KeyError`, `KeyboardInterrupt`, `LookupError`, `MemoryError`, `NameError`, `NotImplemented`, `NotImplementedError`, `OSError`, `OverflowError`, `OverflowWarning`, `PendingDeprecationWarning`, `ReferenceError`, `RuntimeError`, `RuntimeWarning`, `StandardError`, `StopIteration`, `SyntaxError`, `SyntaxWarning`, `SystemError`, `SystemExit`, `TabError`, `TypeError`, `UnboundLocalError`, `UnicodeDecodeError`, `UnicodeEncodeError`, `UnicodeError`, `UnicodeTranslateError`, `UnicodeWarning`, `UserWarning`, `ValueError`, `VMSError`, `Warning`, `WindowsError`, `ZeroDivisionError`), NameException, nil},
+			{Words(`(?<!\.)`, `\b`, `ArithmeticError`, `AssertionError`, `AttributeError`, `BaseException`, `DeprecationWarning`, `EOFError`, `EnvironmentError`, `Exception`, `FloatingPointError`, `FutureWarning`, `GeneratorExit`, `IOError`, `ImportError`, `ImportWarning`, `IndentationError`, `IndexError`, `KeyError`, `KeyboardInterrupt`, `LookupError`, `MemoryError`, `NameError`, `NotImplementedError`, `OSError`, `OverflowError`, `OverflowWarning`, `PendingDeprecationWarning`, `ReferenceError`, `RuntimeError`, `RuntimeWarning`, `StandardError`, `StopIteration`, `SyntaxError`, `SyntaxWarning`, `SystemError`, `SystemExit`, `TabError`, `TypeError`, `UnboundLocalError`, `UnicodeDecodeError`, `UnicodeEncodeError`, `UnicodeError`, `UnicodeTranslateError`, `UnicodeWarning`, `UserWarning`, `ValueError`, `VMSError`, `Warning`, `WindowsError`, `ZeroDivisionError`), NameException, nil},
 		},
 		"magicfuncs": {
 			{Words(``, `\b`, `__abs__`, `__add__`, `__and__`, `__call__`, `__cmp__`, `__coerce__`, `__complex__`, `__contains__`, `__del__`, `__delattr__`, `__delete__`, `__delitem__`, `__delslice__`, `__div__`, `__divmod__`, `__enter__`, `__eq__`, `__exit__`, `__float__`, `__floordiv__`, `__ge__`, `__get__`, `__getattr__`, `__getattribute__`, `__getitem__`, `__getslice__`, `__gt__`, `__hash__`, `__hex__`, `__iadd__`, `__iand__`, `__idiv__`, `__ifloordiv__`, `__ilshift__`, `__imod__`, `__imul__`, `__index__`, `__init__`, `__instancecheck__`, `__int__`, `__invert__`, `__iop__`, `__ior__`, `__ipow__`, `__irshift__`, `__isub__`, `__iter__`, `__itruediv__`, `__ixor__`, `__le__`, `__len__`, `__long__`, `__lshift__`, `__lt__`, `__missing__`, `__mod__`, `__mul__`, `__ne__`, `__neg__`, `__new__`, `__nonzero__`, `__oct__`, `__op__`, `__or__`, `__pos__`, `__pow__`, `__radd__`, `__rand__`, `__rcmp__`, `__rdiv__`, `__rdivmod__`, `__repr__`, `__reversed__`, `__rfloordiv__`, `__rlshift__`, `__rmod__`, `__rmul__`, `__rop__`, `__ror__`, `__rpow__`, `__rrshift__`, `__rshift__`, `__rsub__`, `__rtruediv__`, `__rxor__`, `__set__`, `__setattr__`, `__setitem__`, `__setslice__`, `__str__`, `__sub__`, `__subclasscheck__`, `__truediv__`, `__unicode__`, `__xor__`), NameFunctionMagic, nil},
@@ -69,9 +69,9 @@ func python2Rules() Rules {
 			{`\d+[eE][+-]?[0-9]+j?`, LiteralNumberFloat, nil},
 			{`0[0-7]+j?`, LiteralNumberOct, nil},
 			{`0[bB][01]+`, LiteralNumberBin, nil},
-			{`0[xX][a-fA-F0-9_]+`, LiteralNumberHex, nil},
+			{`0[xX][a-fA-F0-9]+`, LiteralNumberHex, nil},
 			{`\d+L`, LiteralNumberIntegerLong, nil},
-			{`[\d_]+j?`, LiteralNumberInteger, nil},
+			{`\d+j?`, LiteralNumberInteger, nil},
 		},
 		"backtick": {
 			{"`.*?`", LiteralStringBacktick, nil},

lexers/testdata/python/test_complex_file1.actual

@@ -0,0 +1,271 @@
+# From CPython (Lib/asyncio/coroutines.py)
+__all__ = 'coroutine', 'iscoroutinefunction', 'iscoroutine'
+
+import collections.abc
+import functools
+import inspect
+import os
+import sys
+import traceback
+import types
+import warnings
+
+from . import base_futures
+from . import constants
+from . import format_helpers
+from .log import logger
+
+
+def _is_debug_mode():
+    # If you set _DEBUG to true, @coroutine will wrap the resulting
+    # generator objects in a CoroWrapper instance (defined below).  That
+    # instance will log a message when the generator is never iterated
+    # over, which may happen when you forget to use "await" or "yield from"
+    # with a coroutine call.
+    # Note that the value of the _DEBUG flag is taken
+    # when the decorator is used, so to be of any use it must be set
+    # before you define your coroutines.  A downside of using this feature
+    # is that tracebacks show entries for the CoroWrapper.__next__ method
+    # when _DEBUG is true.
+    return sys.flags.dev_mode or (not sys.flags.ignore_environment and
+                                  bool(os.environ.get('PYTHONASYNCIODEBUG')))
+
+
+_DEBUG = _is_debug_mode()
+
+
+class CoroWrapper:
+    # Wrapper for coroutine object in _DEBUG mode.
+
+    def __init__(self, gen, func=None):
+        assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen
+        self.gen = gen
+        self.func = func  # Used to unwrap @coroutine decorator
+        self._source_traceback = format_helpers.extract_stack(sys._getframe(1))
+        self.__name__ = getattr(gen, '__name__', None)
+        self.__qualname__ = getattr(gen, '__qualname__', None)
+
+    def __repr__(self):
+        coro_repr = _format_coroutine(self)
+        if self._source_traceback:
+            frame = self._source_traceback[-1]
+            coro_repr += f', created at {frame[0]}:{frame[1]}'
+
+        return f'<{self.__class__.__name__} {coro_repr}>'
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return self.gen.send(None)
+
+    def send(self, value):
+        return self.gen.send(value)
+
+    def throw(self, type, value=None, traceback=None):
+        return self.gen.throw(type, value, traceback)
+
+    def close(self):
+        return self.gen.close()
+
+    @property
+    def gi_frame(self):
+        return self.gen.gi_frame
+
+    @property
+    def gi_running(self):
+        return self.gen.gi_running
+
+    @property
+    def gi_code(self):
+        return self.gen.gi_code
+
+    def __await__(self):
+        return self
+
+    @property
+    def gi_yieldfrom(self):
+        return self.gen.gi_yieldfrom
+
+    def __del__(self):
+        # Be careful accessing self.gen.frame -- self.gen might not exist.
+        gen = getattr(self, 'gen', None)
+        frame = getattr(gen, 'gi_frame', None)
+        if frame is not None and frame.f_lasti == -1:
+            msg = f'{self!r} was never yielded from'
+            tb = getattr(self, '_source_traceback', ())
+            if tb:
+                tb = ''.join(traceback.format_list(tb))
+                msg += (f'\nCoroutine object created at '
+                        f'(most recent call last, truncated to '
+                        f'{constants.DEBUG_STACK_DEPTH} last lines):\n')
+                msg += tb.rstrip()
+            logger.error(msg)
+
+
+def coroutine(func):
+    """Decorator to mark coroutines.
+
+    If the coroutine is not yielded from before it is destroyed,
+    an error message is logged.
+    """
+    warnings.warn('"@coroutine" decorator is deprecated since Python 3.8, use "async def" instead',
+                  DeprecationWarning,
+                  stacklevel=2)
+    if inspect.iscoroutinefunction(func):
+        # In Python 3.5 that's all we need to do for coroutines
+        # defined with "async def".
+        return func
+
+    if inspect.isgeneratorfunction(func):
+        coro = func
+    else:
+        @functools.wraps(func)
+        def coro(*args, **kw):
+            res = func(*args, **kw)
+            if (base_futures.isfuture(res) or inspect.isgenerator(res) or
+                    isinstance(res, CoroWrapper)):
+                res = yield from res
+            else:
+                # If 'res' is an awaitable, run it.
+                try:
+                    await_meth = res.__await__
+                except AttributeError:
+                    pass
+                else:
+                    if isinstance(res, collections.abc.Awaitable):
+                        res = yield from await_meth()
+            return res
+
+    coro = types.coroutine(coro)
+    if not _DEBUG:
+        wrapper = coro
+    else:
+        @functools.wraps(func)
+        def wrapper(*args, **kwds):
+            w = CoroWrapper(coro(*args, **kwds), func=func)
+            if w._source_traceback:
+                del w._source_traceback[-1]
+            # Python < 3.5 does not implement __qualname__
+            # on generator objects, so we set it manually.
+            # We use getattr as some callables (such as
+            # functools.partial may lack __qualname__).
+            w.__name__ = getattr(func, '__name__', None)
+            w.__qualname__ = getattr(func, '__qualname__', None)
+            return w
+
+    wrapper._is_coroutine = _is_coroutine  # For iscoroutinefunction().
+    return wrapper
+
+
+# A marker for iscoroutinefunction.
+_is_coroutine = object()
+
+
+def iscoroutinefunction(func):
+    """Return True if func is a decorated coroutine function."""
+    return (inspect.iscoroutinefunction(func) or
+            getattr(func, '_is_coroutine', None) is _is_coroutine)
+
+
+# Prioritize native coroutine check to speed-up
+# asyncio.iscoroutine.
+_COROUTINE_TYPES = (types.CoroutineType, types.GeneratorType,
+                    collections.abc.Coroutine, CoroWrapper)
+_iscoroutine_typecache = set()
+
+
+def iscoroutine(obj):
+    """Return True if obj is a coroutine object."""
+    if type(obj) in _iscoroutine_typecache:
+        return True
+
+    if isinstance(obj, _COROUTINE_TYPES):
+        # Just in case we don't want to cache more than 100
+        # positive types.  That shouldn't ever happen, unless
+        # someone stressing the system on purpose.
+        if len(_iscoroutine_typecache) < 100:
+            _iscoroutine_typecache.add(type(obj))
+        return True
+    else:
+        return False
+
+
+def _format_coroutine(coro):
+    assert iscoroutine(coro)
+
+    is_corowrapper = isinstance(coro, CoroWrapper)
+
+    def get_name(coro):
+        # Coroutines compiled with Cython sometimes don't have
+        # proper __qualname__ or __name__.  While that is a bug
+        # in Cython, asyncio shouldn't crash with an AttributeError
+        # in its __repr__ functions.
+        if is_corowrapper:
+            return format_helpers._format_callback(coro.func, (), {})
+
+        if hasattr(coro, '__qualname__') and coro.__qualname__:
+            coro_name = coro.__qualname__
+        elif hasattr(coro, '__name__') and coro.__name__:
+            coro_name = coro.__name__
+        else:
+            # Stop masking Cython bugs, expose them in a friendly way.
+            coro_name = f'<{type(coro).__name__} without __name__>'
+        return f'{coro_name}()'
+
+    def is_running(coro):
+        try:
+            return coro.cr_running
+        except AttributeError:
+            try:
+                return coro.gi_running
+            except AttributeError:
+                return False
+
+    coro_code = None
+    if hasattr(coro, 'cr_code') and coro.cr_code:
+        coro_code = coro.cr_code
+    elif hasattr(coro, 'gi_code') and coro.gi_code:
+        coro_code = coro.gi_code
+
+    coro_name = get_name(coro)
+
+    if not coro_code:
+        # Built-in types might not have __qualname__ or __name__.
+        if is_running(coro):
+            return f'{coro_name} running'
+        else:
+            return coro_name
+
+    coro_frame = None
+    if hasattr(coro, 'gi_frame') and coro.gi_frame:
+        coro_frame = coro.gi_frame
+    elif hasattr(coro, 'cr_frame') and coro.cr_frame:
+        coro_frame = coro.cr_frame
+
+    # If Cython's coroutine has a fake code object without proper
+    # co_filename -- expose that.
+    filename = coro_code.co_filename or '<empty co_filename>'
+
+    lineno = 0
+    if (is_corowrapper and
+            coro.func is not None and
+            not inspect.isgeneratorfunction(coro.func)):
+        source = format_helpers._get_function_source(coro.func)
+        if source is not None:
+            filename, lineno = source
+        if coro_frame is None:
+            coro_repr = f'{coro_name} done, defined at {filename}:{lineno}'
+        else:
+            coro_repr = f'{coro_name} running, defined at {filename}:{lineno}'
+
+    elif coro_frame is not None:
+        lineno = coro_frame.f_lineno
+        coro_repr = f'{coro_name} running at {filename}:{lineno}'
+
+    else:
+        lineno = coro_code.co_firstlineno
+        coro_repr = f'{coro_name} done, defined at {filename}:{lineno}'
+
+    return coro_repr
+

lexers/testdata/python/test_complex_file2.actual

@@ -0,0 +1,224 @@
+# From CPython (Lib/asyncio/subprocess.py)
+__all__ = 'create_subprocess_exec', 'create_subprocess_shell'
+
+import subprocess
+
+from . import events
+from . import protocols
+from . import streams
+from . import tasks
+from .log import logger
+
+
+PIPE = subprocess.PIPE
+STDOUT = subprocess.STDOUT
+DEVNULL = subprocess.DEVNULL
+
+
+class SubprocessStreamProtocol(streams.FlowControlMixin,
+                               protocols.SubprocessProtocol):
+    """Like StreamReaderProtocol, but for a subprocess."""
+
+    def __init__(self, limit, loop):
+        super().__init__(loop=loop)
+        self._limit = limit
+        self.stdin = self.stdout = self.stderr = None
+        self._transport = None
+        self._process_exited = False
+        self._pipe_fds = []
+        self._stdin_closed = self._loop.create_future()
+
+    def __repr__(self):
+        info = [self.__class__.__name__]
+        if self.stdin is not None:
+            info.append(f'stdin={self.stdin!r}')
+        if self.stdout is not None:
+            info.append(f'stdout={self.stdout!r}')
+        if self.stderr is not None:
+            info.append(f'stderr={self.stderr!r}')
+        return '<{}>'.format(' '.join(info))
+
+    def connection_made(self, transport):
+        self._transport = transport
+
+        stdout_transport = transport.get_pipe_transport(1)
+        if stdout_transport is not None:
+            self.stdout = streams.StreamReader(limit=self._limit,
+                                               loop=self._loop)
+            self.stdout.set_transport(stdout_transport)
+            self._pipe_fds.append(1)
+
+        stderr_transport = transport.get_pipe_transport(2)
+        if stderr_transport is not None:
+            self.stderr = streams.StreamReader(limit=self._limit,
+                                               loop=self._loop)
+            self.stderr.set_transport(stderr_transport)
+            self._pipe_fds.append(2)
+
+        stdin_transport = transport.get_pipe_transport(0)
+        if stdin_transport is not None:
+            self.stdin = streams.StreamWriter(stdin_transport,
+                                              protocol=self,
+                                              reader=None,
+                                              loop=self._loop)
+
+    def pipe_data_received(self, fd, data):
+        if fd == 1:
+            reader = self.stdout
+        elif fd == 2:
+            reader = self.stderr
+        else:
+            reader = None
+        if reader is not None:
+            reader.feed_data(data)
+
+    def pipe_connection_lost(self, fd, exc):
+        if fd == 0:
+            pipe = self.stdin
+            if pipe is not None:
+                pipe.close()
+            self.connection_lost(exc)
+            if exc is None:
+                self._stdin_closed.set_result(None)
+            else:
+                self._stdin_closed.set_exception(exc)
+            return
+        if fd == 1:
+            reader = self.stdout
+        elif fd == 2:
+            reader = self.stderr
+        else:
+            reader = None
+        if reader is not None:
+            if exc is None:
+                reader.feed_eof()
+            else:
+                reader.set_exception(exc)
+
+        if fd in self._pipe_fds:
+            self._pipe_fds.remove(fd)
+        self._maybe_close_transport()
+
+    def process_exited(self):
+        self._process_exited = True
+        self._maybe_close_transport()
+
+    def _maybe_close_transport(self):
+        if len(self._pipe_fds) == 0 and self._process_exited:
+            self._transport.close()
+            self._transport = None
+
+    def _get_close_waiter(self, stream):
+        if stream is self.stdin:
+            return self._stdin_closed
+
+
+class Process:
+    def __init__(self, transport, protocol, loop):
+        self._transport = transport
+        self._protocol = protocol
+        self._loop = loop
+        self.stdin = protocol.stdin
+        self.stdout = protocol.stdout
+        self.stderr = protocol.stderr
+        self.pid = transport.get_pid()
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__} {self.pid}>'
+
+    @property
+    def returncode(self):
+        return self._transport.get_returncode()
+
+    async def wait(self):
+        """Wait until the process exit and return the process return code."""
+        return await self._transport._wait()
+
+    def send_signal(self, signal):
+        self._transport.send_signal(signal)
+
+    def terminate(self):
+        self._transport.terminate()
+
+    def kill(self):
+        self._transport.kill()
+
+    async def _feed_stdin(self, input):
+        debug = self._loop.get_debug()
+        self.stdin.write(input)
+        if debug:
+            logger.debug(
+                '%r communicate: feed stdin (%s bytes)', self, len(input))
+        try:
+            await self.stdin.drain()
+        except (BrokenPipeError, ConnectionResetError) as exc:
+            # communicate() ignores BrokenPipeError and ConnectionResetError
+            if debug:
+                logger.debug('%r communicate: stdin got %r', self, exc)
+
+        if debug:
+            logger.debug('%r communicate: close stdin', self)
+        self.stdin.close()
+
+    async def _noop(self):
+        return None
+
+    async def _read_stream(self, fd):
+        transport = self._transport.get_pipe_transport(fd)
+        if fd == 2:
+            stream = self.stderr
+        else:
+            assert fd == 1
+            stream = self.stdout
+        if self._loop.get_debug():
+            name = 'stdout' if fd == 1 else 'stderr'
+            logger.debug('%r communicate: read %s', self, name)
+        output = await stream.read()
+        if self._loop.get_debug():
+            name = 'stdout' if fd == 1 else 'stderr'
+            logger.debug('%r communicate: close %s', self, name)
+        transport.close()
+        return output
+
+    async def communicate(self, input=None):
+        if input is not None:
+            stdin = self._feed_stdin(input)
+        else:
+            stdin = self._noop()
+        if self.stdout is not None:
+            stdout = self._read_stream(1)
+        else:
+            stdout = self._noop()
+        if self.stderr is not None:
+            stderr = self._read_stream(2)
+        else:
+            stderr = self._noop()
+        stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr)
+        await self.wait()
+        return (stdout, stderr)
+
+
+async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
+                                  limit=streams._DEFAULT_LIMIT, **kwds):
+    loop = events.get_running_loop()
+    protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
+                                                        loop=loop)
+    transport, protocol = await loop.subprocess_shell(
+        protocol_factory,
+        cmd, stdin=stdin, stdout=stdout,
+        stderr=stderr, **kwds)
+    return Process(transport, protocol, loop)
+
+
+async def create_subprocess_exec(program, *args, stdin=None, stdout=None,
+                                 stderr=None, limit=streams._DEFAULT_LIMIT,
+                                 **kwds):
+    loop = events.get_running_loop()
+    protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
+                                                        loop=loop)
+    transport, protocol = await loop.subprocess_exec(
+        protocol_factory,
+        program, *args,
+        stdin=stdin, stdout=stdout,
+        stderr=stderr, **kwds)
+    return Process(transport, protocol, loop)

lexers/testdata/python/test_floats.actual

@@ -0,0 +1 @@
+123 -11 0 -0 0.5 .5 1. -0.5 +0.5 -.5 -1. 2e1 -2e1 2e -2e +2e e.3 -e.3 11.2e-3 -11.2e-3 5_6 5__6 _5 6_ 5.6_7 5.67_

lexers/testdata/python/test_floats.expected

@@ -0,0 +1,73 @@
+[
+  {"type":"LiteralNumberInteger","value":"123"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"LiteralNumberInteger","value":"11"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"0.5"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":".5"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1."},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"LiteralNumberFloat","value":"0.5"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"LiteralNumberFloat","value":"0.5"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"LiteralNumberFloat","value":".5"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"LiteralNumberFloat","value":"1."},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"2e1"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"LiteralNumberFloat","value":"2e1"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Name","value":"e"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Name","value":"e"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Name","value":"e"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"e"},
+  {"type":"LiteralNumberFloat","value":".3"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"Name","value":"e"},
+  {"type":"LiteralNumberFloat","value":".3"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"11.2e-3"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"LiteralNumberFloat","value":"11.2e-3"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"5_6"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"5"},
+  {"type":"Name","value":"__6"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_5"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"6"},
+  {"type":"Name","value":"_"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"5.6_7"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"5.67"},
+  {"type":"Name","value":"_"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_01a.actual

@@ -0,0 +1 @@
+f'My name is {name}, my age next year is {age+1}, my anniversary is {anniversary:%A, %B %d, %Y}.'

lexers/testdata/python/test_fstring_01a.expected

@@ -0,0 +1,21 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'My name is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"name"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":", my age next year is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"age"},
+  {"type":"Operator","value":"+"},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":", my anniversary is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"anniversary"},
+  {"type":"LiteralStringInterpol","value":":"},
+  {"type":"LiteralStringSingle","value":"%A, %B %d, %Y"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":".'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_01b.actual

@@ -0,0 +1 @@
+f"My name is {name}, my age next year is {age+1}, my anniversary is {anniversary:%A, %B %d, %Y}."

lexers/testdata/python/test_fstring_01b.expected

@@ -0,0 +1,21 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"My name is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"name"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":", my age next year is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"age"},
+  {"type":"Operator","value":"+"},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":", my anniversary is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"anniversary"},
+  {"type":"LiteralStringInterpol","value":":"},
+  {"type":"LiteralStringDouble","value":"%A, %B %d, %Y"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":".\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_02a.actual

@@ -0,0 +1 @@
+f'He said his name is {name!r}.'

lexers/testdata/python/test_fstring_02a.expected

@@ -0,0 +1,9 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'He said his name is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"name"},
+  {"type":"LiteralStringInterpol","value":"!r}"},
+  {"type":"LiteralStringSingle","value":".'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_02b.actual

@@ -0,0 +1 @@
+f"He said his name is {name!r}."

lexers/testdata/python/test_fstring_02b.expected

@@ -0,0 +1,9 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"He said his name is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"name"},
+  {"type":"LiteralStringInterpol","value":"!r}"},
+  {"type":"LiteralStringDouble","value":".\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_03a.actual

@@ -0,0 +1 @@
+f'input={value:#06x}'

lexers/testdata/python/test_fstring_03a.expected

@@ -0,0 +1,11 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'input="},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"value"},
+  {"type":"LiteralStringInterpol","value":":"},
+  {"type":"LiteralStringSingle","value":"#06x"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_03b.actual

@@ -0,0 +1 @@
+f"input={value:#06x}"

lexers/testdata/python/test_fstring_03b.expected

@@ -0,0 +1,11 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"input="},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"value"},
+  {"type":"LiteralStringInterpol","value":":"},
+  {"type":"LiteralStringDouble","value":"#06x"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_04a.actual

@@ -0,0 +1 @@
+f'{"quoted string"}'

lexers/testdata/python/test_fstring_04a.expected

@@ -0,0 +1,9 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralStringDouble","value":"\"quoted string\""},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_04b.actual

@@ -0,0 +1 @@
+f"{'quoted string'}"

lexers/testdata/python/test_fstring_04b.expected

@@ -0,0 +1,9 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralStringSingle","value":"'quoted string'"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_05a.actual

@@ -0,0 +1 @@
+f'{f"{inner}"}'

lexers/testdata/python/test_fstring_05a.expected

@@ -0,0 +1,14 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"inner"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_05b.actual

@@ -0,0 +1 @@
+f"{f'{inner}'}"

lexers/testdata/python/test_fstring_05b.expected

@@ -0,0 +1,14 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"inner"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_06a.actual

@@ -0,0 +1 @@
+f'{\'quoted string\'}'

lexers/testdata/python/test_fstring_06a.expected

@@ -0,0 +1,10 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Error","value":"\\"},
+  {"type":"LiteralStringSingle","value":"'quoted string"},
+  {"type":"LiteralStringEscape","value":"\\'"},
+  {"type":"LiteralStringSingle","value":"}'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_06b.actual

@@ -0,0 +1 @@
+f"{\"quoted string\"}"

lexers/testdata/python/test_fstring_06b.expected

@@ -0,0 +1,10 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Error","value":"\\"},
+  {"type":"LiteralStringDouble","value":"\"quoted string"},
+  {"type":"LiteralStringEscape","value":"\\\""},
+  {"type":"LiteralStringDouble","value":"}\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_07a.actual

@@ -0,0 +1 @@
+f'{{ {4*10} }}'

lexers/testdata/python/test_fstring_07a.expected

@@ -0,0 +1,15 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringEscape","value":"{{"},
+  {"type":"LiteralStringSingle","value":" "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Operator","value":"*"},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":" "},
+  {"type":"LiteralStringEscape","value":"}}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_07b.actual

@@ -0,0 +1 @@
+f"{{ {4*10} }}"

lexers/testdata/python/test_fstring_07b.expected

@@ -0,0 +1,15 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringEscape","value":"{{"},
+  {"type":"LiteralStringDouble","value":" "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Operator","value":"*"},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":" "},
+  {"type":"LiteralStringEscape","value":"}}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_08a.actual

@@ -0,0 +1 @@
+f'{{{4*10}}}'

lexers/testdata/python/test_fstring_08a.expected

@@ -0,0 +1,13 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringEscape","value":"{{"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Operator","value":"*"},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringEscape","value":"}}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_08b.actual

@@ -0,0 +1 @@
+f"{{{4*10}}}"

lexers/testdata/python/test_fstring_08b.expected

@@ -0,0 +1,13 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringEscape","value":"{{"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Operator","value":"*"},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringEscape","value":"}}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_09a.actual

@@ -0,0 +1 @@
+fr'x={4*10}'

lexers/testdata/python/test_fstring_09a.expected

@@ -0,0 +1,11 @@
+[
+  {"type":"LiteralStringAffix","value":"fr"},
+  {"type":"LiteralStringSingle","value":"'x="},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Operator","value":"*"},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_09b.actual

@@ -0,0 +1 @@
+fr"x={4*10}"

lexers/testdata/python/test_fstring_09b.expected

@@ -0,0 +1,11 @@
+[
+  {"type":"LiteralStringAffix","value":"fr"},
+  {"type":"LiteralStringDouble","value":"\"x="},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Operator","value":"*"},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_10a.actual

@@ -0,0 +1 @@
+f'abc {a["x"]} def'

lexers/testdata/python/test_fstring_10a.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'abc "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringDouble","value":"\"x\""},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":" def'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_10b.actual

@@ -0,0 +1 @@
+f"abc {a['x']} def"

lexers/testdata/python/test_fstring_10b.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"abc "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringSingle","value":"'x'"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":" def\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_11a.actual

@@ -0,0 +1 @@
+f'''abc {a['x']} def'''

lexers/testdata/python/test_fstring_11a.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'''abc "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringSingle","value":"'x'"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":" def'''"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_11b.actual

@@ -0,0 +1 @@
+f"""abc {a["x"]} def"""

lexers/testdata/python/test_fstring_11b.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"\"\"abc "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringDouble","value":"\"x\""},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":" def\"\"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_12a.actual

@@ -0,0 +1,2 @@
+f'''{x
++1}'''

lexers/testdata/python/test_fstring_12a.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'''"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"x"},
+  {"type":"Text","value":"\n"},
+  {"type":"Operator","value":"+"},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'''"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_12b.actual

@@ -0,0 +1,2 @@
+f"""{x
++1}"""

lexers/testdata/python/test_fstring_12b.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"\"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"x"},
+  {"type":"Text","value":"\n"},
+  {"type":"Operator","value":"+"},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\"\"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_13a.actual

@@ -0,0 +1,2 @@
+f'''{d[0
+]}'''

lexers/testdata/python/test_fstring_13a.expected

@@ -0,0 +1,13 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'''"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"d"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'''"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_13b.actual

@@ -0,0 +1,2 @@
+f"""{d[0
+]}"""

lexers/testdata/python/test_fstring_13b.expected

@@ -0,0 +1,13 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"\"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"d"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\"\"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_14a.actual

@@ -0,0 +1 @@
+f'result: {value:{width}.{precision}}'

lexers/testdata/python/test_fstring_14a.expected

@@ -0,0 +1,15 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'result: "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"value"},
+  {"type":"LiteralStringInterpol","value":":{"},
+  {"type":"Name","value":"width"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"."},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"precision"},
+  {"type":"LiteralStringInterpol","value":"}}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_14b.actual

@@ -0,0 +1 @@
+f"result: {value:{width}.{precision}}"

lexers/testdata/python/test_fstring_14b.expected

@@ -0,0 +1,15 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"result: "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"value"},
+  {"type":"LiteralStringInterpol","value":":{"},
+  {"type":"Name","value":"width"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"."},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"precision"},
+  {"type":"LiteralStringInterpol","value":"}}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_15a.actual

@@ -0,0 +1 @@
+'a' 'b' f'{x}' '{c}' f'str<{y:^4}>' 'd' 'e'

lexers/testdata/python/test_fstring_15a.expected

@@ -0,0 +1,30 @@
+[
+  {"type":"LiteralStringSingle","value":"'a'"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringSingle","value":"'b'"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"x"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{c}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'str\u003c"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"y"},
+  {"type":"LiteralStringInterpol","value":":"},
+  {"type":"LiteralStringSingle","value":"^4"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"\u003e'"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringSingle","value":"'d'"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringSingle","value":"'e'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_15b.actual

@@ -0,0 +1 @@
+"a" "b" f"{x}" "{c}" f"str<{y:^4}>" "d" "e"

lexers/testdata/python/test_fstring_15b.expected

@@ -0,0 +1,30 @@
+[
+  {"type":"LiteralStringDouble","value":"\"a\""},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"b\""},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"x"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{c}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"str\u003c"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"y"},
+  {"type":"LiteralStringInterpol","value":":"},
+  {"type":"LiteralStringDouble","value":"^4"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\u003e\""},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"d\""},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"e\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_16a.actual

@@ -0,0 +1 @@
+f'{i}:{d[i]}'

lexers/testdata/python/test_fstring_16a.expected

@@ -0,0 +1,16 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"i"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":":"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"d"},
+  {"type":"Punctuation","value":"["},
+  {"type":"Name","value":"i"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_16b.actual

@@ -0,0 +1 @@
+f"{i}:{d[i]}"

lexers/testdata/python/test_fstring_16b.expected

@@ -0,0 +1,16 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"i"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":":"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"d"},
+  {"type":"Punctuation","value":"["},
+  {"type":"Name","value":"i"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_17a.actual

@@ -0,0 +1 @@
+f'x = {x:+3}'

lexers/testdata/python/test_fstring_17a.expected

@@ -0,0 +1,11 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'x = "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"x"},
+  {"type":"LiteralStringInterpol","value":":"},
+  {"type":"LiteralStringSingle","value":"+3"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_17b.actual

@@ -0,0 +1 @@
+f"x = {x:+3}"

lexers/testdata/python/test_fstring_17b.expected

@@ -0,0 +1,11 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"x = "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"x"},
+  {"type":"LiteralStringInterpol","value":":"},
+  {"type":"LiteralStringDouble","value":"+3"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_18a.actual

@@ -0,0 +1 @@
+f'{fn(lst,2)} {fn(lst,3)}'

lexers/testdata/python/test_fstring_18a.expected

@@ -0,0 +1,23 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"fn"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"lst"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Punctuation","value":")"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":" "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"fn"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"lst"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberInteger","value":"3"},
+  {"type":"Punctuation","value":")"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_18b.actual

@@ -0,0 +1 @@
+f"{fn(lst,2)} {fn(lst,3)}"

lexers/testdata/python/test_fstring_18b.expected

@@ -0,0 +1,23 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"fn"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"lst"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Punctuation","value":")"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":" "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"fn"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"lst"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberInteger","value":"3"},
+  {"type":"Punctuation","value":")"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_19a.actual

@@ -0,0 +1 @@
+f'mapping is { {a:b for (a, b) in ((1, 2), (3, 4))} }'

lexers/testdata/python/test_fstring_19a.expected

@@ -0,0 +1,39 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'mapping is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Name","value":"b"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"for"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"b"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"OperatorWord","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"(("},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberInteger","value":"3"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Punctuation","value":"))}"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_19b.actual

@@ -0,0 +1 @@
+f"mapping is { {a:b for (a, b) in ((1, 2), (3, 4))} }"

lexers/testdata/python/test_fstring_19b.expected

@@ -0,0 +1,39 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"mapping is "},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Name","value":"b"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"for"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"b"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"OperatorWord","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"(("},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberInteger","value":"3"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Punctuation","value":"))}"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_20a.actual

@@ -0,0 +1 @@
+f'a={d["a"]}'

lexers/testdata/python/test_fstring_20a.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'a="},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"d"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringDouble","value":"\"a\""},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_20b.actual

@@ -0,0 +1 @@
+f"a={d['a']}"

lexers/testdata/python/test_fstring_20b.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"a="},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"d"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringSingle","value":"'a'"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_21a.actual

@@ -0,0 +1 @@
+f'a={d[a]}'

lexers/testdata/python/test_fstring_21a.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'a="},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"d"},
+  {"type":"Punctuation","value":"["},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_21b.actual

@@ -0,0 +1 @@
+f"a={d[a]}"

lexers/testdata/python/test_fstring_21b.expected

@@ -0,0 +1,12 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\"a="},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"d"},
+  {"type":"Punctuation","value":"["},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_22a.actual

@@ -0,0 +1 @@
+fr'{header}:\s+'

lexers/testdata/python/test_fstring_22a.expected

@@ -0,0 +1,9 @@
+[
+  {"type":"LiteralStringAffix","value":"fr"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"header"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringSingle","value":":\\s+'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_22b.actual

@@ -0,0 +1 @@
+fr"{header}:\s+"

lexers/testdata/python/test_fstring_22b.expected

@@ -0,0 +1,9 @@
+[
+  {"type":"LiteralStringAffix","value":"fr"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"header"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":":\\s+\""},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_23a.actual

@@ -0,0 +1 @@
+f'{a!r}'

lexers/testdata/python/test_fstring_23a.expected

@@ -0,0 +1,9 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"a"},
+  {"type":"LiteralStringInterpol","value":"!r}"},
+  {"type":"LiteralStringSingle","value":"'"},
+  {"type":"Text","value":"\n"}
+]

lexers/testdata/python/test_fstring_23b.actual

@@ -0,0 +1 @@
+f"{a!r}"

lexers/testdata/python/test_fstring_23b.expected

@@ -0,0 +1,9 @@
+[
+  {"type":"LiteralStringAffix","value":"f"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"LiteralStringInterpol","value":"{"},
+  {"type":"Name","value":"a"},
+  {"type":"LiteralStringInterpol","value":"!r}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"}
+]