完美

python/py/Lib/site-packages/click-8.3.2.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

python/py/Lib/site-packages/click-8.3.2.dist-info/METADATA

@@ -0,0 +1,84 @@
+Metadata-Version: 2.4
+Name: click
+Version: 8.3.2
+Summary: Composable command line interface toolkit
+Maintainer-email: Pallets <contact@palletsprojects.com>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-Expression: BSD-3-Clause
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Typing :: Typed
+License-File: LICENSE.txt
+Requires-Dist: colorama; platform_system == 'Windows'
+Project-URL: Changes, https://click.palletsprojects.com/page/changes/
+Project-URL: Chat, https://discord.gg/pallets
+Project-URL: Documentation, https://click.palletsprojects.com/
+Project-URL: Donate, https://palletsprojects.com/donate
+Project-URL: Source, https://github.com/pallets/click/
+
+<div align="center"><img src="https://raw.githubusercontent.com/pallets/click/refs/heads/stable/docs/_static/click-name.svg" alt="" height="150"></div>
+
+# Click
+
+Click is a Python package for creating beautiful command line interfaces
+in a composable way with as little code as necessary. It's the "Command
+Line Interface Creation Kit". It's highly configurable but comes with
+sensible defaults out of the box.
+
+It aims to make the process of writing command line tools quick and fun
+while also preventing any frustration caused by the inability to
+implement an intended CLI API.
+
+Click in three points:
+
+-   Arbitrary nesting of commands
+-   Automatic help page generation
+-   Supports lazy loading of subcommands at runtime
+
+
+## A Simple Example
+
+```python
+import click
+
+@click.command()
+@click.option("--count", default=1, help="Number of greetings.")
+@click.option("--name", prompt="Your name", help="The person to greet.")
+def hello(count, name):
+    """Simple program that greets NAME for a total of COUNT times."""
+    for _ in range(count):
+        click.echo(f"Hello, {name}!")
+
+if __name__ == '__main__':
+    hello()
+```
+
+```
+$ python hello.py --count=3
+Your name: Click
+Hello, Click!
+Hello, Click!
+Hello, Click!
+```
+
+
+## Donate
+
+The Pallets organization develops and supports Click and other popular
+packages. In order to grow the community of contributors and users, and
+allow the maintainers to devote more time to the projects, [please
+donate today][].
+
+[please donate today]: https://palletsprojects.com/donate
+
+## Contributing
+
+See our [detailed contributing documentation][contrib] for many ways to
+contribute, including reporting issues, requesting features, asking or answering
+questions, and making PRs.
+
+[contrib]: https://palletsprojects.com/contributing/
+

python/py/Lib/site-packages/click-8.3.2.dist-info/RECORD

@@ -0,0 +1,40 @@
+click-8.3.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+click-8.3.2.dist-info/METADATA,sha256=yA3Hu5bMxtntTd4QrI8hTFAc58rHSPhDbP6bklbUQkA,2621
+click-8.3.2.dist-info/RECORD,,
+click-8.3.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+click-8.3.2.dist-info/licenses/LICENSE.txt,sha256=morRBqOU6FO_4h9C9OctWSgZoigF2ZG18ydQKSkrZY0,1475
+click/__init__.py,sha256=6YyS1aeyknZ0LYweWozNZy0A9nZ_11wmYIhv3cbQrYo,4473
+click/__pycache__/__init__.cpython-312.pyc,,
+click/__pycache__/_compat.cpython-312.pyc,,
+click/__pycache__/_termui_impl.cpython-312.pyc,,
+click/__pycache__/_textwrap.cpython-312.pyc,,
+click/__pycache__/_utils.cpython-312.pyc,,
+click/__pycache__/_winconsole.cpython-312.pyc,,
+click/__pycache__/core.cpython-312.pyc,,
+click/__pycache__/decorators.cpython-312.pyc,,
+click/__pycache__/exceptions.cpython-312.pyc,,
+click/__pycache__/formatting.cpython-312.pyc,,
+click/__pycache__/globals.cpython-312.pyc,,
+click/__pycache__/parser.cpython-312.pyc,,
+click/__pycache__/shell_completion.cpython-312.pyc,,
+click/__pycache__/termui.cpython-312.pyc,,
+click/__pycache__/testing.cpython-312.pyc,,
+click/__pycache__/types.cpython-312.pyc,,
+click/__pycache__/utils.cpython-312.pyc,,
+click/_compat.py,sha256=v3xBZkFbvA1BXPRkFfBJc6-pIwPI7345m-kQEnpVAs4,18693
+click/_termui_impl.py,sha256=rgCb3On8X5A4200rA5L6i13u5iapmFer7sru57Jy6zA,27093
+click/_textwrap.py,sha256=BOae0RQ6vg3FkNgSJyOoGzG1meGMxJ_ukWVZKx_v-0o,1400
+click/_utils.py,sha256=kZwtTf5gMuCilJJceS2iTCvRvCY-0aN5rJq8gKw7p8g,943
+click/_winconsole.py,sha256=_vxUuUaxwBhoR0vUWCNuHY8VUefiMdCIyU2SXPqoF-A,8465
+click/core.py,sha256=7db9qr_wqXbQriDHCDc26OK0MsaLCSt4yrz14Kn7AEQ,132905
+click/decorators.py,sha256=5P7abhJtAQYp_KHgjUvhMv464ERwOzrv2enNknlwHyQ,18461
+click/exceptions.py,sha256=8utf8w6V5hJXMnO_ic1FNrtbwuEn1NUu1aDwV8UqnG4,9954
+click/formatting.py,sha256=RVfwwr0rwWNpgGr8NaHodPzkIr7_tUyVh_nDdanLMNc,9730
+click/globals.py,sha256=gM-Nh6A4M0HB_SgkaF5M4ncGGMDHc_flHXu9_oh4GEU,1923
+click/parser.py,sha256=Q31pH0FlQZEq-UXE_ABRzlygEfvxPTuZbWNh4xfXmzw,19010
+click/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+click/shell_completion.py,sha256=Cc4GQUFuWpfQBa9sF5qXeeYI7n3tI_1k6ZdSn4BZbT0,20994
+click/termui.py,sha256=hqCEjNndU-nzW08nRAkBaVgfZp_FdCA9KxfIWlKYaMc,31037
+click/testing.py,sha256=LjNfHqNctxc3GfRkLgifO6gnRetblh8yGXzjw4FPFCQ,18978
+click/types.py,sha256=ek54BNSFwPKsqtfT7jsqcc4WHui8AIFVMKM4oVZIXhc,39927
+click/utils.py,sha256=gCUoewdAhA-QLBUUHxrLh4uj6m7T1WjZZMNPvR0I7YA,20257

python/py/Lib/site-packages/click-8.3.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: flit 3.12.0
+Root-Is-Purelib: true
+Tag: py3-none-any

python/py/Lib/site-packages/click-8.3.2.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,28 @@
+Copyright 2014 Pallets
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1.  Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+2.  Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+
+3.  Neither the name of the copyright holder nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

python/py/Lib/site-packages/click/__init__.py

@@ -0,0 +1,123 @@
+"""
+Click is a simple Python module inspired by the stdlib optparse to make
+writing command line scripts fun. Unlike other modules, it's based
+around a simple API that does not come with too much magic and is
+composable.
+"""
+
+from __future__ import annotations
+
+from .core import Argument as Argument
+from .core import Command as Command
+from .core import CommandCollection as CommandCollection
+from .core import Context as Context
+from .core import Group as Group
+from .core import Option as Option
+from .core import Parameter as Parameter
+from .decorators import argument as argument
+from .decorators import command as command
+from .decorators import confirmation_option as confirmation_option
+from .decorators import group as group
+from .decorators import help_option as help_option
+from .decorators import make_pass_decorator as make_pass_decorator
+from .decorators import option as option
+from .decorators import pass_context as pass_context
+from .decorators import pass_obj as pass_obj
+from .decorators import password_option as password_option
+from .decorators import version_option as version_option
+from .exceptions import Abort as Abort
+from .exceptions import BadArgumentUsage as BadArgumentUsage
+from .exceptions import BadOptionUsage as BadOptionUsage
+from .exceptions import BadParameter as BadParameter
+from .exceptions import ClickException as ClickException
+from .exceptions import FileError as FileError
+from .exceptions import MissingParameter as MissingParameter
+from .exceptions import NoSuchOption as NoSuchOption
+from .exceptions import UsageError as UsageError
+from .formatting import HelpFormatter as HelpFormatter
+from .formatting import wrap_text as wrap_text
+from .globals import get_current_context as get_current_context
+from .termui import clear as clear
+from .termui import confirm as confirm
+from .termui import echo_via_pager as echo_via_pager
+from .termui import edit as edit
+from .termui import getchar as getchar
+from .termui import launch as launch
+from .termui import pause as pause
+from .termui import progressbar as progressbar
+from .termui import prompt as prompt
+from .termui import secho as secho
+from .termui import style as style
+from .termui import unstyle as unstyle
+from .types import BOOL as BOOL
+from .types import Choice as Choice
+from .types import DateTime as DateTime
+from .types import File as File
+from .types import FLOAT as FLOAT
+from .types import FloatRange as FloatRange
+from .types import INT as INT
+from .types import IntRange as IntRange
+from .types import ParamType as ParamType
+from .types import Path as Path
+from .types import STRING as STRING
+from .types import Tuple as Tuple
+from .types import UNPROCESSED as UNPROCESSED
+from .types import UUID as UUID
+from .utils import echo as echo
+from .utils import format_filename as format_filename
+from .utils import get_app_dir as get_app_dir
+from .utils import get_binary_stream as get_binary_stream
+from .utils import get_text_stream as get_text_stream
+from .utils import open_file as open_file
+
+
+def __getattr__(name: str) -> object:
+    import warnings
+
+    if name == "BaseCommand":
+        from .core import _BaseCommand
+
+        warnings.warn(
+            "'BaseCommand' is deprecated and will be removed in Click 9.0. Use"
+            " 'Command' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return _BaseCommand
+
+    if name == "MultiCommand":
+        from .core import _MultiCommand
+
+        warnings.warn(
+            "'MultiCommand' is deprecated and will be removed in Click 9.0. Use"
+            " 'Group' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return _MultiCommand
+
+    if name == "OptionParser":
+        from .parser import _OptionParser
+
+        warnings.warn(
+            "'OptionParser' is deprecated and will be removed in Click 9.0. The"
+            " old parser is available in 'optparse'.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return _OptionParser
+
+    if name == "__version__":
+        import importlib.metadata
+        import warnings
+
+        warnings.warn(
+            "The '__version__' attribute is deprecated and will be removed in"
+            " Click 9.1. Use feature detection or"
+            " 'importlib.metadata.version(\"click\")' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return importlib.metadata.version("click")
+
+    raise AttributeError(name)

python/py/Lib/site-packages/click/_compat.py

@@ -0,0 +1,622 @@
+from __future__ import annotations
+
+import codecs
+import collections.abc as cabc
+import io
+import os
+import re
+import sys
+import typing as t
+from types import TracebackType
+from weakref import WeakKeyDictionary
+
+CYGWIN = sys.platform.startswith("cygwin")
+WIN = sys.platform.startswith("win")
+auto_wrap_for_ansi: t.Callable[[t.TextIO], t.TextIO] | None = None
+_ansi_re = re.compile(r"\033\[[;?0-9]*[a-zA-Z]")
+
+
+def _make_text_stream(
+    stream: t.BinaryIO,
+    encoding: str | None,
+    errors: str | None,
+    force_readable: bool = False,
+    force_writable: bool = False,
+) -> t.TextIO:
+    if encoding is None:
+        encoding = get_best_encoding(stream)
+    if errors is None:
+        errors = "replace"
+    return _NonClosingTextIOWrapper(
+        stream,
+        encoding,
+        errors,
+        line_buffering=True,
+        force_readable=force_readable,
+        force_writable=force_writable,
+    )
+
+
+def is_ascii_encoding(encoding: str) -> bool:
+    """Checks if a given encoding is ascii."""
+    try:
+        return codecs.lookup(encoding).name == "ascii"
+    except LookupError:
+        return False
+
+
+def get_best_encoding(stream: t.IO[t.Any]) -> str:
+    """Returns the default stream encoding if not found."""
+    rv = getattr(stream, "encoding", None) or sys.getdefaultencoding()
+    if is_ascii_encoding(rv):
+        return "utf-8"
+    return rv
+
+
+class _NonClosingTextIOWrapper(io.TextIOWrapper):
+    def __init__(
+        self,
+        stream: t.BinaryIO,
+        encoding: str | None,
+        errors: str | None,
+        force_readable: bool = False,
+        force_writable: bool = False,
+        **extra: t.Any,
+    ) -> None:
+        self._stream = stream = t.cast(
+            t.BinaryIO, _FixupStream(stream, force_readable, force_writable)
+        )
+        super().__init__(stream, encoding, errors, **extra)
+
+    def __del__(self) -> None:
+        try:
+            self.detach()
+        except Exception:
+            pass
+
+    def isatty(self) -> bool:
+        # https://bitbucket.org/pypy/pypy/issue/1803
+        return self._stream.isatty()
+
+
+class _FixupStream:
+    """The new io interface needs more from streams than streams
+    traditionally implement.  As such, this fix-up code is necessary in
+    some circumstances.
+
+    The forcing of readable and writable flags are there because some tools
+    put badly patched objects on sys (one such offender are certain version
+    of jupyter notebook).
+    """
+
+    def __init__(
+        self,
+        stream: t.BinaryIO,
+        force_readable: bool = False,
+        force_writable: bool = False,
+    ):
+        self._stream = stream
+        self._force_readable = force_readable
+        self._force_writable = force_writable
+
+    def __getattr__(self, name: str) -> t.Any:
+        return getattr(self._stream, name)
+
+    def read1(self, size: int) -> bytes:
+        f = getattr(self._stream, "read1", None)
+
+        if f is not None:
+            return t.cast(bytes, f(size))
+
+        return self._stream.read(size)
+
+    def readable(self) -> bool:
+        if self._force_readable:
+            return True
+        x = getattr(self._stream, "readable", None)
+        if x is not None:
+            return t.cast(bool, x())
+        try:
+            self._stream.read(0)
+        except Exception:
+            return False
+        return True
+
+    def writable(self) -> bool:
+        if self._force_writable:
+            return True
+        x = getattr(self._stream, "writable", None)
+        if x is not None:
+            return t.cast(bool, x())
+        try:
+            self._stream.write(b"")
+        except Exception:
+            try:
+                self._stream.write(b"")
+            except Exception:
+                return False
+        return True
+
+    def seekable(self) -> bool:
+        x = getattr(self._stream, "seekable", None)
+        if x is not None:
+            return t.cast(bool, x())
+        try:
+            self._stream.seek(self._stream.tell())
+        except Exception:
+            return False
+        return True
+
+
+def _is_binary_reader(stream: t.IO[t.Any], default: bool = False) -> bool:
+    try:
+        return isinstance(stream.read(0), bytes)
+    except Exception:
+        return default
+        # This happens in some cases where the stream was already
+        # closed.  In this case, we assume the default.
+
+
+def _is_binary_writer(stream: t.IO[t.Any], default: bool = False) -> bool:
+    try:
+        stream.write(b"")
+    except Exception:
+        try:
+            stream.write("")
+            return False
+        except Exception:
+            pass
+        return default
+    return True
+
+
+def _find_binary_reader(stream: t.IO[t.Any]) -> t.BinaryIO | None:
+    # We need to figure out if the given stream is already binary.
+    # This can happen because the official docs recommend detaching
+    # the streams to get binary streams.  Some code might do this, so
+    # we need to deal with this case explicitly.
+    if _is_binary_reader(stream, False):
+        return t.cast(t.BinaryIO, stream)
+
+    buf = getattr(stream, "buffer", None)
+
+    # Same situation here; this time we assume that the buffer is
+    # actually binary in case it's closed.
+    if buf is not None and _is_binary_reader(buf, True):
+        return t.cast(t.BinaryIO, buf)
+
+    return None
+
+
+def _find_binary_writer(stream: t.IO[t.Any]) -> t.BinaryIO | None:
+    # We need to figure out if the given stream is already binary.
+    # This can happen because the official docs recommend detaching
+    # the streams to get binary streams.  Some code might do this, so
+    # we need to deal with this case explicitly.
+    if _is_binary_writer(stream, False):
+        return t.cast(t.BinaryIO, stream)
+
+    buf = getattr(stream, "buffer", None)
+
+    # Same situation here; this time we assume that the buffer is
+    # actually binary in case it's closed.
+    if buf is not None and _is_binary_writer(buf, True):
+        return t.cast(t.BinaryIO, buf)
+
+    return None
+
+
+def _stream_is_misconfigured(stream: t.TextIO) -> bool:
+    """A stream is misconfigured if its encoding is ASCII."""
+    # If the stream does not have an encoding set, we assume it's set
+    # to ASCII.  This appears to happen in certain unittest
+    # environments.  It's not quite clear what the correct behavior is
+    # but this at least will force Click to recover somehow.
+    return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii")
+
+
+def _is_compat_stream_attr(stream: t.TextIO, attr: str, value: str | None) -> bool:
+    """A stream attribute is compatible if it is equal to the
+    desired value or the desired value is unset and the attribute
+    has a value.
+    """
+    stream_value = getattr(stream, attr, None)
+    return stream_value == value or (value is None and stream_value is not None)
+
+
+def _is_compatible_text_stream(
+    stream: t.TextIO, encoding: str | None, errors: str | None
+) -> bool:
+    """Check if a stream's encoding and errors attributes are
+    compatible with the desired values.
+    """
+    return _is_compat_stream_attr(
+        stream, "encoding", encoding
+    ) and _is_compat_stream_attr(stream, "errors", errors)
+
+
+def _force_correct_text_stream(
+    text_stream: t.IO[t.Any],
+    encoding: str | None,
+    errors: str | None,
+    is_binary: t.Callable[[t.IO[t.Any], bool], bool],
+    find_binary: t.Callable[[t.IO[t.Any]], t.BinaryIO | None],
+    force_readable: bool = False,
+    force_writable: bool = False,
+) -> t.TextIO:
+    if is_binary(text_stream, False):
+        binary_reader = t.cast(t.BinaryIO, text_stream)
+    else:
+        text_stream = t.cast(t.TextIO, text_stream)
+        # If the stream looks compatible, and won't default to a
+        # misconfigured ascii encoding, return it as-is.
+        if _is_compatible_text_stream(text_stream, encoding, errors) and not (
+            encoding is None and _stream_is_misconfigured(text_stream)
+        ):
+            return text_stream
+
+        # Otherwise, get the underlying binary reader.
+        possible_binary_reader = find_binary(text_stream)
+
+        # If that's not possible, silently use the original reader
+        # and get mojibake instead of exceptions.
+        if possible_binary_reader is None:
+            return text_stream
+
+        binary_reader = possible_binary_reader
+
+    # Default errors to replace instead of strict in order to get
+    # something that works.
+    if errors is None:
+        errors = "replace"
+
+    # Wrap the binary stream in a text stream with the correct
+    # encoding parameters.
+    return _make_text_stream(
+        binary_reader,
+        encoding,
+        errors,
+        force_readable=force_readable,
+        force_writable=force_writable,
+    )
+
+
+def _force_correct_text_reader(
+    text_reader: t.IO[t.Any],
+    encoding: str | None,
+    errors: str | None,
+    force_readable: bool = False,
+) -> t.TextIO:
+    return _force_correct_text_stream(
+        text_reader,
+        encoding,
+        errors,
+        _is_binary_reader,
+        _find_binary_reader,
+        force_readable=force_readable,
+    )
+
+
+def _force_correct_text_writer(
+    text_writer: t.IO[t.Any],
+    encoding: str | None,
+    errors: str | None,
+    force_writable: bool = False,
+) -> t.TextIO:
+    return _force_correct_text_stream(
+        text_writer,
+        encoding,
+        errors,
+        _is_binary_writer,
+        _find_binary_writer,
+        force_writable=force_writable,
+    )
+
+
+def get_binary_stdin() -> t.BinaryIO:
+    reader = _find_binary_reader(sys.stdin)
+    if reader is None:
+        raise RuntimeError("Was not able to determine binary stream for sys.stdin.")
+    return reader
+
+
+def get_binary_stdout() -> t.BinaryIO:
+    writer = _find_binary_writer(sys.stdout)
+    if writer is None:
+        raise RuntimeError("Was not able to determine binary stream for sys.stdout.")
+    return writer
+
+
+def get_binary_stderr() -> t.BinaryIO:
+    writer = _find_binary_writer(sys.stderr)
+    if writer is None:
+        raise RuntimeError("Was not able to determine binary stream for sys.stderr.")
+    return writer
+
+
+def get_text_stdin(encoding: str | None = None, errors: str | None = None) -> t.TextIO:
+    rv = _get_windows_console_stream(sys.stdin, encoding, errors)
+    if rv is not None:
+        return rv
+    return _force_correct_text_reader(sys.stdin, encoding, errors, force_readable=True)
+
+
+def get_text_stdout(encoding: str | None = None, errors: str | None = None) -> t.TextIO:
+    rv = _get_windows_console_stream(sys.stdout, encoding, errors)
+    if rv is not None:
+        return rv
+    return _force_correct_text_writer(sys.stdout, encoding, errors, force_writable=True)
+
+
+def get_text_stderr(encoding: str | None = None, errors: str | None = None) -> t.TextIO:
+    rv = _get_windows_console_stream(sys.stderr, encoding, errors)
+    if rv is not None:
+        return rv
+    return _force_correct_text_writer(sys.stderr, encoding, errors, force_writable=True)
+
+
+def _wrap_io_open(
+    file: str | os.PathLike[str] | int,
+    mode: str,
+    encoding: str | None,
+    errors: str | None,
+) -> t.IO[t.Any]:
+    """Handles not passing ``encoding`` and ``errors`` in binary mode."""
+    if "b" in mode:
+        return open(file, mode)
+
+    return open(file, mode, encoding=encoding, errors=errors)
+
+
+def open_stream(
+    filename: str | os.PathLike[str],
+    mode: str = "r",
+    encoding: str | None = None,
+    errors: str | None = "strict",
+    atomic: bool = False,
+) -> tuple[t.IO[t.Any], bool]:
+    binary = "b" in mode
+    filename = os.fspath(filename)
+
+    # Standard streams first. These are simple because they ignore the
+    # atomic flag. Use fsdecode to handle Path("-").
+    if os.fsdecode(filename) == "-":
+        if any(m in mode for m in ["w", "a", "x"]):
+            if binary:
+                return get_binary_stdout(), False
+            return get_text_stdout(encoding=encoding, errors=errors), False
+        if binary:
+            return get_binary_stdin(), False
+        return get_text_stdin(encoding=encoding, errors=errors), False
+
+    # Non-atomic writes directly go out through the regular open functions.
+    if not atomic:
+        return _wrap_io_open(filename, mode, encoding, errors), True
+
+    # Some usability stuff for atomic writes
+    if "a" in mode:
+        raise ValueError(
+            "Appending to an existing file is not supported, because that"
+            " would involve an expensive `copy`-operation to a temporary"
+            " file. Open the file in normal `w`-mode and copy explicitly"
+            " if that's what you're after."
+        )
+    if "x" in mode:
+        raise ValueError("Use the `overwrite`-parameter instead.")
+    if "w" not in mode:
+        raise ValueError("Atomic writes only make sense with `w`-mode.")
+
+    # Atomic writes are more complicated.  They work by opening a file
+    # as a proxy in the same folder and then using the fdopen
+    # functionality to wrap it in a Python file.  Then we wrap it in an
+    # atomic file that moves the file over on close.
+    import errno
+    import random
+
+    try:
+        perm: int | None = os.stat(filename).st_mode
+    except OSError:
+        perm = None
+
+    flags = os.O_RDWR | os.O_CREAT | os.O_EXCL
+
+    if binary:
+        flags |= getattr(os, "O_BINARY", 0)
+
+    while True:
+        tmp_filename = os.path.join(
+            os.path.dirname(filename),
+            f".__atomic-write{random.randrange(1 << 32):08x}",
+        )
+        try:
+            fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm)
+            break
+        except OSError as e:
+            if e.errno == errno.EEXIST or (
+                os.name == "nt"
+                and e.errno == errno.EACCES
+                and os.path.isdir(e.filename)
+                and os.access(e.filename, os.W_OK)
+            ):
+                continue
+            raise
+
+    if perm is not None:
+        os.chmod(tmp_filename, perm)  # in case perm includes bits in umask
+
+    f = _wrap_io_open(fd, mode, encoding, errors)
+    af = _AtomicFile(f, tmp_filename, os.path.realpath(filename))
+    return t.cast(t.IO[t.Any], af), True
+
+
+class _AtomicFile:
+    def __init__(self, f: t.IO[t.Any], tmp_filename: str, real_filename: str) -> None:
+        self._f = f
+        self._tmp_filename = tmp_filename
+        self._real_filename = real_filename
+        self.closed = False
+
+    @property
+    def name(self) -> str:
+        return self._real_filename
+
+    def close(self, delete: bool = False) -> None:
+        if self.closed:
+            return
+        self._f.close()
+        os.replace(self._tmp_filename, self._real_filename)
+        self.closed = True
+
+    def __getattr__(self, name: str) -> t.Any:
+        return getattr(self._f, name)
+
+    def __enter__(self) -> _AtomicFile:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close(delete=exc_type is not None)
+
+    def __repr__(self) -> str:
+        return repr(self._f)
+
+
+def strip_ansi(value: str) -> str:
+    return _ansi_re.sub("", value)
+
+
+def _is_jupyter_kernel_output(stream: t.IO[t.Any]) -> bool:
+    while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)):
+        stream = stream._stream
+
+    return stream.__class__.__module__.startswith("ipykernel.")
+
+
+def should_strip_ansi(
+    stream: t.IO[t.Any] | None = None, color: bool | None = None
+) -> bool:
+    if color is None:
+        if stream is None:
+            stream = sys.stdin
+        return not isatty(stream) and not _is_jupyter_kernel_output(stream)
+    return not color
+
+
+# On Windows, wrap the output streams with colorama to support ANSI
+# color codes.
+# NOTE: double check is needed so mypy does not analyze this on Linux
+if sys.platform.startswith("win") and WIN:
+    from ._winconsole import _get_windows_console_stream
+
+    def _get_argv_encoding() -> str:
+        import locale
+
+        return locale.getpreferredencoding()
+
+    _ansi_stream_wrappers: cabc.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary()
+
+    def auto_wrap_for_ansi(stream: t.TextIO, color: bool | None = None) -> t.TextIO:
+        """Support ANSI color and style codes on Windows by wrapping a
+        stream with colorama.
+        """
+        try:
+            cached = _ansi_stream_wrappers.get(stream)
+        except Exception:
+            cached = None
+
+        if cached is not None:
+            return cached
+
+        import colorama
+
+        strip = should_strip_ansi(stream, color)
+        ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip)
+        rv = t.cast(t.TextIO, ansi_wrapper.stream)
+        _write = rv.write
+
+        def _safe_write(s: str) -> int:
+            try:
+                return _write(s)
+            except BaseException:
+                ansi_wrapper.reset_all()
+                raise
+
+        rv.write = _safe_write  # type: ignore[method-assign]
+
+        try:
+            _ansi_stream_wrappers[stream] = rv
+        except Exception:
+            pass
+
+        return rv
+
+else:
+
+    def _get_argv_encoding() -> str:
+        return getattr(sys.stdin, "encoding", None) or sys.getfilesystemencoding()
+
+    def _get_windows_console_stream(
+        f: t.TextIO, encoding: str | None, errors: str | None
+    ) -> t.TextIO | None:
+        return None
+
+
+def term_len(x: str) -> int:
+    return len(strip_ansi(x))
+
+
+def isatty(stream: t.IO[t.Any]) -> bool:
+    try:
+        return stream.isatty()
+    except Exception:
+        return False
+
+
+def _make_cached_stream_func(
+    src_func: t.Callable[[], t.TextIO | None],
+    wrapper_func: t.Callable[[], t.TextIO],
+) -> t.Callable[[], t.TextIO | None]:
+    cache: cabc.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary()
+
+    def func() -> t.TextIO | None:
+        stream = src_func()
+
+        if stream is None:
+            return None
+
+        try:
+            rv = cache.get(stream)
+        except Exception:
+            rv = None
+        if rv is not None:
+            return rv
+        rv = wrapper_func()
+        try:
+            cache[stream] = rv
+        except Exception:
+            pass
+        return rv
+
+    return func
+
+
+_default_text_stdin = _make_cached_stream_func(lambda: sys.stdin, get_text_stdin)
+_default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_stdout)
+_default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr)
+
+
+binary_streams: cabc.Mapping[str, t.Callable[[], t.BinaryIO]] = {
+    "stdin": get_binary_stdin,
+    "stdout": get_binary_stdout,
+    "stderr": get_binary_stderr,
+}
+
+text_streams: cabc.Mapping[str, t.Callable[[str | None, str | None], t.TextIO]] = {
+    "stdin": get_text_stdin,
+    "stdout": get_text_stdout,
+    "stderr": get_text_stderr,
+}

python/py/Lib/site-packages/click/_termui_impl.py

@@ -0,0 +1,852 @@
+"""
+This module contains implementations for the termui module. To keep the
+import time of Click down, some infrequently used functionality is
+placed in this module and only imported as needed.
+"""
+
+from __future__ import annotations
+
+import collections.abc as cabc
+import contextlib
+import math
+import os
+import shlex
+import sys
+import time
+import typing as t
+from gettext import gettext as _
+from io import StringIO
+from pathlib import Path
+from types import TracebackType
+
+from ._compat import _default_text_stdout
+from ._compat import CYGWIN
+from ._compat import get_best_encoding
+from ._compat import isatty
+from ._compat import open_stream
+from ._compat import strip_ansi
+from ._compat import term_len
+from ._compat import WIN
+from .exceptions import ClickException
+from .utils import echo
+
+V = t.TypeVar("V")
+
+if os.name == "nt":
+    BEFORE_BAR = "\r"
+    AFTER_BAR = "\n"
+else:
+    BEFORE_BAR = "\r\033[?25l"
+    AFTER_BAR = "\033[?25h\n"
+
+
+class ProgressBar(t.Generic[V]):
+    def __init__(
+        self,
+        iterable: cabc.Iterable[V] | None,
+        length: int | None = None,
+        fill_char: str = "#",
+        empty_char: str = " ",
+        bar_template: str = "%(bar)s",
+        info_sep: str = "  ",
+        hidden: bool = False,
+        show_eta: bool = True,
+        show_percent: bool | None = None,
+        show_pos: bool = False,
+        item_show_func: t.Callable[[V | None], str | None] | None = None,
+        label: str | None = None,
+        file: t.TextIO | None = None,
+        color: bool | None = None,
+        update_min_steps: int = 1,
+        width: int = 30,
+    ) -> None:
+        self.fill_char = fill_char
+        self.empty_char = empty_char
+        self.bar_template = bar_template
+        self.info_sep = info_sep
+        self.hidden = hidden
+        self.show_eta = show_eta
+        self.show_percent = show_percent
+        self.show_pos = show_pos
+        self.item_show_func = item_show_func
+        self.label: str = label or ""
+
+        if file is None:
+            file = _default_text_stdout()
+
+            # There are no standard streams attached to write to. For example,
+            # pythonw on Windows.
+            if file is None:
+                file = StringIO()
+
+        self.file = file
+        self.color = color
+        self.update_min_steps = update_min_steps
+        self._completed_intervals = 0
+        self.width: int = width
+        self.autowidth: bool = width == 0
+
+        if length is None:
+            from operator import length_hint
+
+            length = length_hint(iterable, -1)
+
+            if length == -1:
+                length = None
+        if iterable is None:
+            if length is None:
+                raise TypeError("iterable or length is required")
+            iterable = t.cast("cabc.Iterable[V]", range(length))
+        self.iter: cabc.Iterable[V] = iter(iterable)
+        self.length = length
+        self.pos: int = 0
+        self.avg: list[float] = []
+        self.last_eta: float
+        self.start: float
+        self.start = self.last_eta = time.time()
+        self.eta_known: bool = False
+        self.finished: bool = False
+        self.max_width: int | None = None
+        self.entered: bool = False
+        self.current_item: V | None = None
+        self._is_atty = isatty(self.file)
+        self._last_line: str | None = None
+
+    def __enter__(self) -> ProgressBar[V]:
+        self.entered = True
+        self.render_progress()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.render_finish()
+
+    def __iter__(self) -> cabc.Iterator[V]:
+        if not self.entered:
+            raise RuntimeError("You need to use progress bars in a with block.")
+        self.render_progress()
+        return self.generator()
+
+    def __next__(self) -> V:
+        # Iteration is defined in terms of a generator function,
+        # returned by iter(self); use that to define next(). This works
+        # because `self.iter` is an iterable consumed by that generator,
+        # so it is re-entry safe. Calling `next(self.generator())`
+        # twice works and does "what you want".
+        return next(iter(self))
+
+    def render_finish(self) -> None:
+        if self.hidden or not self._is_atty:
+            return
+        self.file.write(AFTER_BAR)
+        self.file.flush()
+
+    @property
+    def pct(self) -> float:
+        if self.finished:
+            return 1.0
+        return min(self.pos / (float(self.length or 1) or 1), 1.0)
+
+    @property
+    def time_per_iteration(self) -> float:
+        if not self.avg:
+            return 0.0
+        return sum(self.avg) / float(len(self.avg))
+
+    @property
+    def eta(self) -> float:
+        if self.length is not None and not self.finished:
+            return self.time_per_iteration * (self.length - self.pos)
+        return 0.0
+
+    def format_eta(self) -> str:
+        if self.eta_known:
+            t = int(self.eta)
+            seconds = t % 60
+            t //= 60
+            minutes = t % 60
+            t //= 60
+            hours = t % 24
+            t //= 24
+            if t > 0:
+                return f"{t}d {hours:02}:{minutes:02}:{seconds:02}"
+            else:
+                return f"{hours:02}:{minutes:02}:{seconds:02}"
+        return ""
+
+    def format_pos(self) -> str:
+        pos = str(self.pos)
+        if self.length is not None:
+            pos += f"/{self.length}"
+        return pos
+
+    def format_pct(self) -> str:
+        return f"{int(self.pct * 100): 4}%"[1:]
+
+    def format_bar(self) -> str:
+        if self.length is not None:
+            bar_length = int(self.pct * self.width)
+            bar = self.fill_char * bar_length
+            bar += self.empty_char * (self.width - bar_length)
+        elif self.finished:
+            bar = self.fill_char * self.width
+        else:
+            chars = list(self.empty_char * (self.width or 1))
+            if self.time_per_iteration != 0:
+                chars[
+                    int(
+                        (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5)
+                        * self.width
+                    )
+                ] = self.fill_char
+            bar = "".join(chars)
+        return bar
+
+    def format_progress_line(self) -> str:
+        show_percent = self.show_percent
+
+        info_bits = []
+        if self.length is not None and show_percent is None:
+            show_percent = not self.show_pos
+
+        if self.show_pos:
+            info_bits.append(self.format_pos())
+        if show_percent:
+            info_bits.append(self.format_pct())
+        if self.show_eta and self.eta_known and not self.finished:
+            info_bits.append(self.format_eta())
+        if self.item_show_func is not None:
+            item_info = self.item_show_func(self.current_item)
+            if item_info is not None:
+                info_bits.append(item_info)
+
+        return (
+            self.bar_template
+            % {
+                "label": self.label,
+                "bar": self.format_bar(),
+                "info": self.info_sep.join(info_bits),
+            }
+        ).rstrip()
+
+    def render_progress(self) -> None:
+        if self.hidden:
+            return
+
+        if not self._is_atty:
+            # Only output the label once if the output is not a TTY.
+            if self._last_line != self.label:
+                self._last_line = self.label
+                echo(self.label, file=self.file, color=self.color)
+            return
+
+        buf = []
+        # Update width in case the terminal has been resized
+        if self.autowidth:
+            import shutil
+
+            old_width = self.width
+            self.width = 0
+            clutter_length = term_len(self.format_progress_line())
+            new_width = max(0, shutil.get_terminal_size().columns - clutter_length)
+            if new_width < old_width and self.max_width is not None:
+                buf.append(BEFORE_BAR)
+                buf.append(" " * self.max_width)
+                self.max_width = new_width
+            self.width = new_width
+
+        clear_width = self.width
+        if self.max_width is not None:
+            clear_width = self.max_width
+
+        buf.append(BEFORE_BAR)
+        line = self.format_progress_line()
+        line_len = term_len(line)
+        if self.max_width is None or self.max_width < line_len:
+            self.max_width = line_len
+
+        buf.append(line)
+        buf.append(" " * (clear_width - line_len))
+        line = "".join(buf)
+        # Render the line only if it changed.
+
+        if line != self._last_line:
+            self._last_line = line
+            echo(line, file=self.file, color=self.color, nl=False)
+            self.file.flush()
+
+    def make_step(self, n_steps: int) -> None:
+        self.pos += n_steps
+        if self.length is not None and self.pos >= self.length:
+            self.finished = True
+
+        if (time.time() - self.last_eta) < 1.0:
+            return
+
+        self.last_eta = time.time()
+
+        # self.avg is a rolling list of length <= 7 of steps where steps are
+        # defined as time elapsed divided by the total progress through
+        # self.length.
+        if self.pos:
+            step = (time.time() - self.start) / self.pos
+        else:
+            step = time.time() - self.start
+
+        self.avg = self.avg[-6:] + [step]
+
+        self.eta_known = self.length is not None
+
+    def update(self, n_steps: int, current_item: V | None = None) -> None:
+        """Update the progress bar by advancing a specified number of
+        steps, and optionally set the ``current_item`` for this new
+        position.
+
+        :param n_steps: Number of steps to advance.
+        :param current_item: Optional item to set as ``current_item``
+            for the updated position.
+
+        .. versionchanged:: 8.0
+            Added the ``current_item`` optional parameter.
+
+        .. versionchanged:: 8.0
+            Only render when the number of steps meets the
+            ``update_min_steps`` threshold.
+        """
+        if current_item is not None:
+            self.current_item = current_item
+
+        self._completed_intervals += n_steps
+
+        if self._completed_intervals >= self.update_min_steps:
+            self.make_step(self._completed_intervals)
+            self.render_progress()
+            self._completed_intervals = 0
+
+    def finish(self) -> None:
+        self.eta_known = False
+        self.current_item = None
+        self.finished = True
+
+    def generator(self) -> cabc.Iterator[V]:
+        """Return a generator which yields the items added to the bar
+        during construction, and updates the progress bar *after* the
+        yielded block returns.
+        """
+        # WARNING: the iterator interface for `ProgressBar` relies on
+        # this and only works because this is a simple generator which
+        # doesn't create or manage additional state. If this function
+        # changes, the impact should be evaluated both against
+        # `iter(bar)` and `next(bar)`. `next()` in particular may call
+        # `self.generator()` repeatedly, and this must remain safe in
+        # order for that interface to work.
+        if not self.entered:
+            raise RuntimeError("You need to use progress bars in a with block.")
+
+        if not self._is_atty:
+            yield from self.iter
+        else:
+            for rv in self.iter:
+                self.current_item = rv
+
+                # This allows show_item_func to be updated before the
+                # item is processed. Only trigger at the beginning of
+                # the update interval.
+                if self._completed_intervals == 0:
+                    self.render_progress()
+
+                yield rv
+                self.update(1)
+
+            self.finish()
+            self.render_progress()
+
+
+def pager(generator: cabc.Iterable[str], color: bool | None = None) -> None:
+    """Decide what method to use for paging through text."""
+    stdout = _default_text_stdout()
+
+    # There are no standard streams attached to write to. For example,
+    # pythonw on Windows.
+    if stdout is None:
+        stdout = StringIO()
+
+    if not isatty(sys.stdin) or not isatty(stdout):
+        return _nullpager(stdout, generator, color)
+
+    # Split and normalize the pager command into parts.
+    pager_cmd_parts = shlex.split(os.environ.get("PAGER", ""), posix=False)
+    if pager_cmd_parts:
+        if WIN:
+            if _tempfilepager(generator, pager_cmd_parts, color):
+                return
+        elif _pipepager(generator, pager_cmd_parts, color):
+            return
+
+    if os.environ.get("TERM") in ("dumb", "emacs"):
+        return _nullpager(stdout, generator, color)
+    if (WIN or sys.platform.startswith("os2")) and _tempfilepager(
+        generator, ["more"], color
+    ):
+        return
+    if _pipepager(generator, ["less"], color):
+        return
+
+    import tempfile
+
+    fd, filename = tempfile.mkstemp()
+    os.close(fd)
+    try:
+        if _pipepager(generator, ["more"], color):
+            return
+        return _nullpager(stdout, generator, color)
+    finally:
+        os.unlink(filename)
+
+
+def _pipepager(
+    generator: cabc.Iterable[str], cmd_parts: list[str], color: bool | None
+) -> bool:
+    """Page through text by feeding it to another program. Invoking a
+    pager through this might support colors.
+
+    Returns `True` if the command was found, `False` otherwise and thus another
+    pager should be attempted.
+    """
+    # Split the command into the invoked CLI and its parameters.
+    if not cmd_parts:
+        return False
+
+    import shutil
+
+    cmd = cmd_parts[0]
+    cmd_params = cmd_parts[1:]
+
+    cmd_filepath = shutil.which(cmd)
+    if not cmd_filepath:
+        return False
+
+    # Produces a normalized absolute path string.
+    # multi-call binaries such as busybox derive their identity from the symlink
+    # less -> busybox. resolve() causes them to misbehave. (eg. less becomes busybox)
+    cmd_path = Path(cmd_filepath).absolute()
+    cmd_name = cmd_path.name
+
+    import subprocess
+
+    # Make a local copy of the environment to not affect the global one.
+    env = dict(os.environ)
+
+    # If we're piping to less and the user hasn't decided on colors, we enable
+    # them by default we find the -R flag in the command line arguments.
+    if color is None and cmd_name == "less":
+        less_flags = f"{os.environ.get('LESS', '')}{' '.join(cmd_params)}"
+        if not less_flags:
+            env["LESS"] = "-R"
+            color = True
+        elif "r" in less_flags or "R" in less_flags:
+            color = True
+
+    c = subprocess.Popen(
+        [str(cmd_path)] + cmd_params,
+        shell=False,
+        stdin=subprocess.PIPE,
+        env=env,
+        errors="replace",
+        text=True,
+    )
+    assert c.stdin is not None
+    try:
+        for text in generator:
+            if not color:
+                text = strip_ansi(text)
+
+            c.stdin.write(text)
+    except BrokenPipeError:
+        # In case the pager exited unexpectedly, ignore the broken pipe error.
+        pass
+    except Exception as e:
+        # In case there is an exception we want to close the pager immediately
+        # and let the caller handle it.
+        # Otherwise the pager will keep running, and the user may not notice
+        # the error message, or worse yet it may leave the terminal in a broken state.
+        c.terminate()
+        raise e
+    finally:
+        # We must close stdin and wait for the pager to exit before we continue
+        try:
+            c.stdin.close()
+        # Close implies flush, so it might throw a BrokenPipeError if the pager
+        # process exited already.
+        except BrokenPipeError:
+            pass
+
+        # Less doesn't respect ^C, but catches it for its own UI purposes (aborting
+        # search or other commands inside less).
+        #
+        # That means when the user hits ^C, the parent process (click) terminates,
+        # but less is still alive, paging the output and messing up the terminal.
+        #
+        # If the user wants to make the pager exit on ^C, they should set
+        # `LESS='-K'`. It's not our decision to make.
+        while True:
+            try:
+                c.wait()
+            except KeyboardInterrupt:
+                pass
+            else:
+                break
+
+    return True
+
+
+def _tempfilepager(
+    generator: cabc.Iterable[str], cmd_parts: list[str], color: bool | None
+) -> bool:
+    """Page through text by invoking a program on a temporary file.
+
+    Returns `True` if the command was found, `False` otherwise and thus another
+    pager should be attempted.
+    """
+    # Split the command into the invoked CLI and its parameters.
+    if not cmd_parts:
+        return False
+
+    import shutil
+
+    cmd = cmd_parts[0]
+
+    cmd_filepath = shutil.which(cmd)
+    if not cmd_filepath:
+        return False
+    # Produces a normalized absolute path string.
+    # multi-call binaries such as busybox derive their identity from the symlink
+    # less -> busybox. resolve() causes them to misbehave. (eg. less becomes busybox)
+    cmd_path = Path(cmd_filepath).absolute()
+
+    import subprocess
+    import tempfile
+
+    fd, filename = tempfile.mkstemp()
+    # TODO: This never terminates if the passed generator never terminates.
+    text = "".join(generator)
+    if not color:
+        text = strip_ansi(text)
+    encoding = get_best_encoding(sys.stdout)
+    with open_stream(filename, "wb")[0] as f:
+        f.write(text.encode(encoding))
+    try:
+        subprocess.call([str(cmd_path), filename])
+    except OSError:
+        # Command not found
+        pass
+    finally:
+        os.close(fd)
+        os.unlink(filename)
+
+    return True
+
+
+def _nullpager(
+    stream: t.TextIO, generator: cabc.Iterable[str], color: bool | None
+) -> None:
+    """Simply print unformatted text.  This is the ultimate fallback."""
+    for text in generator:
+        if not color:
+            text = strip_ansi(text)
+        stream.write(text)
+
+
+class Editor:
+    def __init__(
+        self,
+        editor: str | None = None,
+        env: cabc.Mapping[str, str] | None = None,
+        require_save: bool = True,
+        extension: str = ".txt",
+    ) -> None:
+        self.editor = editor
+        self.env = env
+        self.require_save = require_save
+        self.extension = extension
+
+    def get_editor(self) -> str:
+        if self.editor is not None:
+            return self.editor
+        for key in "VISUAL", "EDITOR":
+            rv = os.environ.get(key)
+            if rv:
+                return rv
+        if WIN:
+            return "notepad"
+
+        from shutil import which
+
+        for editor in "sensible-editor", "vim", "nano":
+            if which(editor) is not None:
+                return editor
+        return "vi"
+
+    def edit_files(self, filenames: cabc.Iterable[str]) -> None:
+        import subprocess
+
+        editor = self.get_editor()
+        environ: dict[str, str] | None = None
+
+        if self.env:
+            environ = os.environ.copy()
+            environ.update(self.env)
+
+        exc_filename = " ".join(f'"{filename}"' for filename in filenames)
+
+        try:
+            c = subprocess.Popen(
+                args=f"{editor} {exc_filename}", env=environ, shell=True
+            )
+            exit_code = c.wait()
+            if exit_code != 0:
+                raise ClickException(
+                    _("{editor}: Editing failed").format(editor=editor)
+                )
+        except OSError as e:
+            raise ClickException(
+                _("{editor}: Editing failed: {e}").format(editor=editor, e=e)
+            ) from e
+
+    @t.overload
+    def edit(self, text: bytes | bytearray) -> bytes | None: ...
+
+    # We cannot know whether or not the type expected is str or bytes when None
+    # is passed, so str is returned as that was what was done before.
+    @t.overload
+    def edit(self, text: str | None) -> str | None: ...
+
+    def edit(self, text: str | bytes | bytearray | None) -> str | bytes | None:
+        import tempfile
+
+        if text is None:
+            data: bytes | bytearray = b""
+        elif isinstance(text, (bytes, bytearray)):
+            data = text
+        else:
+            if text and not text.endswith("\n"):
+                text += "\n"
+
+            if WIN:
+                data = text.replace("\n", "\r\n").encode("utf-8-sig")
+            else:
+                data = text.encode("utf-8")
+
+        fd, name = tempfile.mkstemp(prefix="editor-", suffix=self.extension)
+        f: t.BinaryIO
+
+        try:
+            with os.fdopen(fd, "wb") as f:
+                f.write(data)
+
+            # If the filesystem resolution is 1 second, like Mac OS
+            # 10.12 Extended, or 2 seconds, like FAT32, and the editor
+            # closes very fast, require_save can fail. Set the modified
+            # time to be 2 seconds in the past to work around this.
+            os.utime(name, (os.path.getatime(name), os.path.getmtime(name) - 2))
+            # Depending on the resolution, the exact value might not be
+            # recorded, so get the new recorded value.
+            timestamp = os.path.getmtime(name)
+
+            self.edit_files((name,))
+
+            if self.require_save and os.path.getmtime(name) == timestamp:
+                return None
+
+            with open(name, "rb") as f:
+                rv = f.read()
+
+            if isinstance(text, (bytes, bytearray)):
+                return rv
+
+            return rv.decode("utf-8-sig").replace("\r\n", "\n")
+        finally:
+            os.unlink(name)
+
+
+def open_url(url: str, wait: bool = False, locate: bool = False) -> int:
+    import subprocess
+
+    def _unquote_file(url: str) -> str:
+        from urllib.parse import unquote
+
+        if url.startswith("file://"):
+            url = unquote(url[7:])
+
+        return url
+
+    if sys.platform == "darwin":
+        args = ["open"]
+        if wait:
+            args.append("-W")
+        if locate:
+            args.append("-R")
+        args.append(_unquote_file(url))
+        null = open("/dev/null", "w")
+        try:
+            return subprocess.Popen(args, stderr=null).wait()
+        finally:
+            null.close()
+    elif WIN:
+        if locate:
+            url = _unquote_file(url)
+            args = ["explorer", f"/select,{url}"]
+        else:
+            args = ["start"]
+            if wait:
+                args.append("/WAIT")
+            args.append("")
+            args.append(url)
+        try:
+            return subprocess.call(args)
+        except OSError:
+            # Command not found
+            return 127
+    elif CYGWIN:
+        if locate:
+            url = _unquote_file(url)
+            args = ["cygstart", os.path.dirname(url)]
+        else:
+            args = ["cygstart"]
+            if wait:
+                args.append("-w")
+            args.append(url)
+        try:
+            return subprocess.call(args)
+        except OSError:
+            # Command not found
+            return 127
+
+    try:
+        if locate:
+            url = os.path.dirname(_unquote_file(url)) or "."
+        else:
+            url = _unquote_file(url)
+        c = subprocess.Popen(["xdg-open", url])
+        if wait:
+            return c.wait()
+        return 0
+    except OSError:
+        if url.startswith(("http://", "https://")) and not locate and not wait:
+            import webbrowser
+
+            webbrowser.open(url)
+            return 0
+        return 1
+
+
+def _translate_ch_to_exc(ch: str) -> None:
+    if ch == "\x03":
+        raise KeyboardInterrupt()
+
+    if ch == "\x04" and not WIN:  # Unix-like, Ctrl+D
+        raise EOFError()
+
+    if ch == "\x1a" and WIN:  # Windows, Ctrl+Z
+        raise EOFError()
+
+    return None
+
+
+if sys.platform == "win32":
+    import msvcrt
+
+    @contextlib.contextmanager
+    def raw_terminal() -> cabc.Iterator[int]:
+        yield -1
+
+    def getchar(echo: bool) -> str:
+        # The function `getch` will return a bytes object corresponding to
+        # the pressed character. Since Windows 10 build 1803, it will also
+        # return \x00 when called a second time after pressing a regular key.
+        #
+        # `getwch` does not share this probably-bugged behavior. Moreover, it
+        # returns a Unicode object by default, which is what we want.
+        #
+        # Either of these functions will return \x00 or \xe0 to indicate
+        # a special key, and you need to call the same function again to get
+        # the "rest" of the code. The fun part is that \u00e0 is
+        # "latin small letter a with grave", so if you type that on a French
+        # keyboard, you _also_ get a \xe0.
+        # E.g., consider the Up arrow. This returns \xe0 and then \x48. The
+        # resulting Unicode string reads as "a with grave" + "capital H".
+        # This is indistinguishable from when the user actually types
+        # "a with grave" and then "capital H".
+        #
+        # When \xe0 is returned, we assume it's part of a special-key sequence
+        # and call `getwch` again, but that means that when the user types
+        # the \u00e0 character, `getchar` doesn't return until a second
+        # character is typed.
+        # The alternative is returning immediately, but that would mess up
+        # cross-platform handling of arrow keys and others that start with
+        # \xe0. Another option is using `getch`, but then we can't reliably
+        # read non-ASCII characters, because return values of `getch` are
+        # limited to the current 8-bit codepage.
+        #
+        # Anyway, Click doesn't claim to do this Right(tm), and using `getwch`
+        # is doing the right thing in more situations than with `getch`.
+
+        if echo:
+            func = t.cast(t.Callable[[], str], msvcrt.getwche)
+        else:
+            func = t.cast(t.Callable[[], str], msvcrt.getwch)
+
+        rv = func()
+
+        if rv in ("\x00", "\xe0"):
+            # \x00 and \xe0 are control characters that indicate special key,
+            # see above.
+            rv += func()
+
+        _translate_ch_to_exc(rv)
+        return rv
+
+else:
+    import termios
+    import tty
+
+    @contextlib.contextmanager
+    def raw_terminal() -> cabc.Iterator[int]:
+        f: t.TextIO | None
+        fd: int
+
+        if not isatty(sys.stdin):
+            f = open("/dev/tty")
+            fd = f.fileno()
+        else:
+            fd = sys.stdin.fileno()
+            f = None
+
+        try:
+            old_settings = termios.tcgetattr(fd)
+
+            try:
+                tty.setraw(fd)
+                yield fd
+            finally:
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+                sys.stdout.flush()
+
+                if f is not None:
+                    f.close()
+        except termios.error:
+            pass
+
+    def getchar(echo: bool) -> str:
+        with raw_terminal() as fd:
+            ch = os.read(fd, 32).decode(get_best_encoding(sys.stdin), "replace")
+
+            if echo and isatty(sys.stdout):
+                sys.stdout.write(ch)
+
+            _translate_ch_to_exc(ch)
+            return ch

python/py/Lib/site-packages/click/_textwrap.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+import textwrap
+from contextlib import contextmanager
+
+
+class TextWrapper(textwrap.TextWrapper):
+    def _handle_long_word(
+        self,
+        reversed_chunks: list[str],
+        cur_line: list[str],
+        cur_len: int,
+        width: int,
+    ) -> None:
+        space_left = max(width - cur_len, 1)
+
+        if self.break_long_words:
+            last = reversed_chunks[-1]
+            cut = last[:space_left]
+            res = last[space_left:]
+            cur_line.append(cut)
+            reversed_chunks[-1] = res
+        elif not cur_line:
+            cur_line.append(reversed_chunks.pop())
+
+    @contextmanager
+    def extra_indent(self, indent: str) -> cabc.Iterator[None]:
+        old_initial_indent = self.initial_indent
+        old_subsequent_indent = self.subsequent_indent
+        self.initial_indent += indent
+        self.subsequent_indent += indent
+
+        try:
+            yield
+        finally:
+            self.initial_indent = old_initial_indent
+            self.subsequent_indent = old_subsequent_indent
+
+    def indent_only(self, text: str) -> str:
+        rv = []
+
+        for idx, line in enumerate(text.splitlines()):
+            indent = self.initial_indent
+
+            if idx > 0:
+                indent = self.subsequent_indent
+
+            rv.append(f"{indent}{line}")
+
+        return "\n".join(rv)

python/py/Lib/site-packages/click/_utils.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import enum
+import typing as t
+
+
+class Sentinel(enum.Enum):
+    """Enum used to define sentinel values.
+
+    .. seealso::
+
+        `PEP 661 - Sentinel Values <https://peps.python.org/pep-0661/>`_.
+    """
+
+    UNSET = object()
+    FLAG_NEEDS_VALUE = object()
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}.{self.name}"
+
+
+UNSET = Sentinel.UNSET
+"""Sentinel used to indicate that a value is not set."""
+
+FLAG_NEEDS_VALUE = Sentinel.FLAG_NEEDS_VALUE
+"""Sentinel used to indicate an option was passed as a flag without a
+value but is not a flag option.
+
+``Option.consume_value`` uses this to prompt or use the ``flag_value``.
+"""
+
+T_UNSET = t.Literal[UNSET]  # type: ignore[valid-type]
+"""Type hint for the :data:`UNSET` sentinel value."""
+
+T_FLAG_NEEDS_VALUE = t.Literal[FLAG_NEEDS_VALUE]  # type: ignore[valid-type]
+"""Type hint for the :data:`FLAG_NEEDS_VALUE` sentinel value."""

python/py/Lib/site-packages/click/_winconsole.py

@@ -0,0 +1,296 @@
+# This module is based on the excellent work by Adam Bartoš who
+# provided a lot of what went into the implementation here in
+# the discussion to issue1602 in the Python bug tracker.
+#
+# There are some general differences in regards to how this works
+# compared to the original patches as we do not need to patch
+# the entire interpreter but just work in our little world of
+# echo and prompt.
+from __future__ import annotations
+
+import collections.abc as cabc
+import io
+import sys
+import time
+import typing as t
+from ctypes import Array
+from ctypes import byref
+from ctypes import c_char
+from ctypes import c_char_p
+from ctypes import c_int
+from ctypes import c_ssize_t
+from ctypes import c_ulong
+from ctypes import c_void_p
+from ctypes import POINTER
+from ctypes import py_object
+from ctypes import Structure
+from ctypes.wintypes import DWORD
+from ctypes.wintypes import HANDLE
+from ctypes.wintypes import LPCWSTR
+from ctypes.wintypes import LPWSTR
+
+from ._compat import _NonClosingTextIOWrapper
+
+assert sys.platform == "win32"
+import msvcrt  # noqa: E402
+from ctypes import windll  # noqa: E402
+from ctypes import WINFUNCTYPE  # noqa: E402
+
+c_ssize_p = POINTER(c_ssize_t)
+
+kernel32 = windll.kernel32
+GetStdHandle = kernel32.GetStdHandle
+ReadConsoleW = kernel32.ReadConsoleW
+WriteConsoleW = kernel32.WriteConsoleW
+GetConsoleMode = kernel32.GetConsoleMode
+GetLastError = kernel32.GetLastError
+GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32))
+CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(
+    ("CommandLineToArgvW", windll.shell32)
+)
+LocalFree = WINFUNCTYPE(c_void_p, c_void_p)(("LocalFree", windll.kernel32))
+
+STDIN_HANDLE = GetStdHandle(-10)
+STDOUT_HANDLE = GetStdHandle(-11)
+STDERR_HANDLE = GetStdHandle(-12)
+
+PyBUF_SIMPLE = 0
+PyBUF_WRITABLE = 1
+
+ERROR_SUCCESS = 0
+ERROR_NOT_ENOUGH_MEMORY = 8
+ERROR_OPERATION_ABORTED = 995
+
+STDIN_FILENO = 0
+STDOUT_FILENO = 1
+STDERR_FILENO = 2
+
+EOF = b"\x1a"
+MAX_BYTES_WRITTEN = 32767
+
+if t.TYPE_CHECKING:
+    try:
+        # Using `typing_extensions.Buffer` instead of `collections.abc`
+        # on Windows for some reason does not have `Sized` implemented.
+        from collections.abc import Buffer  # type: ignore
+    except ImportError:
+        from typing_extensions import Buffer
+
+try:
+    from ctypes import pythonapi
+except ImportError:
+    # On PyPy we cannot get buffers so our ability to operate here is
+    # severely limited.
+    get_buffer = None
+else:
+
+    class Py_buffer(Structure):
+        _fields_ = [  # noqa: RUF012
+            ("buf", c_void_p),
+            ("obj", py_object),
+            ("len", c_ssize_t),
+            ("itemsize", c_ssize_t),
+            ("readonly", c_int),
+            ("ndim", c_int),
+            ("format", c_char_p),
+            ("shape", c_ssize_p),
+            ("strides", c_ssize_p),
+            ("suboffsets", c_ssize_p),
+            ("internal", c_void_p),
+        ]
+
+    PyObject_GetBuffer = pythonapi.PyObject_GetBuffer
+    PyBuffer_Release = pythonapi.PyBuffer_Release
+
+    def get_buffer(obj: Buffer, writable: bool = False) -> Array[c_char]:
+        buf = Py_buffer()
+        flags: int = PyBUF_WRITABLE if writable else PyBUF_SIMPLE
+        PyObject_GetBuffer(py_object(obj), byref(buf), flags)
+
+        try:
+            buffer_type = c_char * buf.len
+            out: Array[c_char] = buffer_type.from_address(buf.buf)
+            return out
+        finally:
+            PyBuffer_Release(byref(buf))
+
+
+class _WindowsConsoleRawIOBase(io.RawIOBase):
+    def __init__(self, handle: int | None) -> None:
+        self.handle = handle
+
+    def isatty(self) -> t.Literal[True]:
+        super().isatty()
+        return True
+
+
+class _WindowsConsoleReader(_WindowsConsoleRawIOBase):
+    def readable(self) -> t.Literal[True]:
+        return True
+
+    def readinto(self, b: Buffer) -> int:
+        bytes_to_be_read = len(b)
+        if not bytes_to_be_read:
+            return 0
+        elif bytes_to_be_read % 2:
+            raise ValueError(
+                "cannot read odd number of bytes from UTF-16-LE encoded console"
+            )
+
+        buffer = get_buffer(b, writable=True)
+        code_units_to_be_read = bytes_to_be_read // 2
+        code_units_read = c_ulong()
+
+        rv = ReadConsoleW(
+            HANDLE(self.handle),
+            buffer,
+            code_units_to_be_read,
+            byref(code_units_read),
+            None,
+        )
+        if GetLastError() == ERROR_OPERATION_ABORTED:
+            # wait for KeyboardInterrupt
+            time.sleep(0.1)
+        if not rv:
+            raise OSError(f"Windows error: {GetLastError()}")
+
+        if buffer[0] == EOF:
+            return 0
+        return 2 * code_units_read.value
+
+
+class _WindowsConsoleWriter(_WindowsConsoleRawIOBase):
+    def writable(self) -> t.Literal[True]:
+        return True
+
+    @staticmethod
+    def _get_error_message(errno: int) -> str:
+        if errno == ERROR_SUCCESS:
+            return "ERROR_SUCCESS"
+        elif errno == ERROR_NOT_ENOUGH_MEMORY:
+            return "ERROR_NOT_ENOUGH_MEMORY"
+        return f"Windows error {errno}"
+
+    def write(self, b: Buffer) -> int:
+        bytes_to_be_written = len(b)
+        buf = get_buffer(b)
+        code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2
+        code_units_written = c_ulong()
+
+        WriteConsoleW(
+            HANDLE(self.handle),
+            buf,
+            code_units_to_be_written,
+            byref(code_units_written),
+            None,
+        )
+        bytes_written = 2 * code_units_written.value
+
+        if bytes_written == 0 and bytes_to_be_written > 0:
+            raise OSError(self._get_error_message(GetLastError()))
+        return bytes_written
+
+
+class ConsoleStream:
+    def __init__(self, text_stream: t.TextIO, byte_stream: t.BinaryIO) -> None:
+        self._text_stream = text_stream
+        self.buffer = byte_stream
+
+    @property
+    def name(self) -> str:
+        return self.buffer.name
+
+    def write(self, x: t.AnyStr) -> int:
+        if isinstance(x, str):
+            return self._text_stream.write(x)
+        try:
+            self.flush()
+        except Exception:
+            pass
+        return self.buffer.write(x)
+
+    def writelines(self, lines: cabc.Iterable[t.AnyStr]) -> None:
+        for line in lines:
+            self.write(line)
+
+    def __getattr__(self, name: str) -> t.Any:
+        return getattr(self._text_stream, name)
+
+    def isatty(self) -> bool:
+        return self.buffer.isatty()
+
+    def __repr__(self) -> str:
+        return f"<ConsoleStream name={self.name!r} encoding={self.encoding!r}>"
+
+
+def _get_text_stdin(buffer_stream: t.BinaryIO) -> t.TextIO:
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),
+        "utf-16-le",
+        "strict",
+        line_buffering=True,
+    )
+    return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream))
+
+
+def _get_text_stdout(buffer_stream: t.BinaryIO) -> t.TextIO:
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)),
+        "utf-16-le",
+        "strict",
+        line_buffering=True,
+    )
+    return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream))
+
+
+def _get_text_stderr(buffer_stream: t.BinaryIO) -> t.TextIO:
+    text_stream = _NonClosingTextIOWrapper(
+        io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)),
+        "utf-16-le",
+        "strict",
+        line_buffering=True,
+    )
+    return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream))
+
+
+_stream_factories: cabc.Mapping[int, t.Callable[[t.BinaryIO], t.TextIO]] = {
+    0: _get_text_stdin,
+    1: _get_text_stdout,
+    2: _get_text_stderr,
+}
+
+
+def _is_console(f: t.TextIO) -> bool:
+    if not hasattr(f, "fileno"):
+        return False
+
+    try:
+        fileno = f.fileno()
+    except (OSError, io.UnsupportedOperation):
+        return False
+
+    handle = msvcrt.get_osfhandle(fileno)
+    return bool(GetConsoleMode(handle, byref(DWORD())))
+
+
+def _get_windows_console_stream(
+    f: t.TextIO, encoding: str | None, errors: str | None
+) -> t.TextIO | None:
+    if (
+        get_buffer is None
+        or encoding not in {"utf-16-le", None}
+        or errors not in {"strict", None}
+        or not _is_console(f)
+    ):
+        return None
+
+    func = _stream_factories.get(f.fileno())
+    if func is None:
+        return None
+
+    b = getattr(f, "buffer", None)
+
+    if b is None:
+        return None
+
+    return func(b)

python/py/Lib/site-packages/click/core.py

@@ -0,0 +1,3437 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+import enum
+import errno
+import inspect
+import os
+import sys
+import typing as t
+from collections import abc
+from collections import Counter
+from contextlib import AbstractContextManager
+from contextlib import contextmanager
+from contextlib import ExitStack
+from functools import update_wrapper
+from gettext import gettext as _
+from gettext import ngettext
+from itertools import repeat
+from types import TracebackType
+
+from . import types
+from ._utils import FLAG_NEEDS_VALUE
+from ._utils import UNSET
+from .exceptions import Abort
+from .exceptions import BadParameter
+from .exceptions import ClickException
+from .exceptions import Exit
+from .exceptions import MissingParameter
+from .exceptions import NoArgsIsHelpError
+from .exceptions import UsageError
+from .formatting import HelpFormatter
+from .formatting import join_options
+from .globals import pop_context
+from .globals import push_context
+from .parser import _OptionParser
+from .parser import _split_opt
+from .termui import confirm
+from .termui import prompt
+from .termui import style
+from .utils import _detect_program_name
+from .utils import _expand_args
+from .utils import echo
+from .utils import make_default_short_help
+from .utils import make_str
+from .utils import PacifyFlushWrapper
+
+if t.TYPE_CHECKING:
+    from .shell_completion import CompletionItem
+
+F = t.TypeVar("F", bound="t.Callable[..., t.Any]")
+V = t.TypeVar("V")
+
+
+def _complete_visible_commands(
+    ctx: Context, incomplete: str
+) -> cabc.Iterator[tuple[str, Command]]:
+    """List all the subcommands of a group that start with the
+    incomplete value and aren't hidden.
+
+    :param ctx: Invocation context for the group.
+    :param incomplete: Value being completed. May be empty.
+    """
+    multi = t.cast(Group, ctx.command)
+
+    for name in multi.list_commands(ctx):
+        if name.startswith(incomplete):
+            command = multi.get_command(ctx, name)
+
+            if command is not None and not command.hidden:
+                yield name, command
+
+
+def _check_nested_chain(
+    base_command: Group, cmd_name: str, cmd: Command, register: bool = False
+) -> None:
+    if not base_command.chain or not isinstance(cmd, Group):
+        return
+
+    if register:
+        message = (
+            f"It is not possible to add the group {cmd_name!r} to another"
+            f" group {base_command.name!r} that is in chain mode."
+        )
+    else:
+        message = (
+            f"Found the group {cmd_name!r} as subcommand to another group "
+            f" {base_command.name!r} that is in chain mode. This is not supported."
+        )
+
+    raise RuntimeError(message)
+
+
+def batch(iterable: cabc.Iterable[V], batch_size: int) -> list[tuple[V, ...]]:
+    return list(zip(*repeat(iter(iterable), batch_size), strict=False))
+
+
+@contextmanager
+def augment_usage_errors(
+    ctx: Context, param: Parameter | None = None
+) -> cabc.Iterator[None]:
+    """Context manager that attaches extra information to exceptions."""
+    try:
+        yield
+    except BadParameter as e:
+        if e.ctx is None:
+            e.ctx = ctx
+        if param is not None and e.param is None:
+            e.param = param
+        raise
+    except UsageError as e:
+        if e.ctx is None:
+            e.ctx = ctx
+        raise
+
+
+def iter_params_for_processing(
+    invocation_order: cabc.Sequence[Parameter],
+    declaration_order: cabc.Sequence[Parameter],
+) -> list[Parameter]:
+    """Returns all declared parameters in the order they should be processed.
+
+    The declared parameters are re-shuffled depending on the order in which
+    they were invoked, as well as the eagerness of each parameters.
+
+    The invocation order takes precedence over the declaration order. I.e. the
+    order in which the user provided them to the CLI is respected.
+
+    This behavior and its effect on callback evaluation is detailed at:
+    https://click.palletsprojects.com/en/stable/advanced/#callback-evaluation-order
+    """
+
+    def sort_key(item: Parameter) -> tuple[bool, float]:
+        try:
+            idx: float = invocation_order.index(item)
+        except ValueError:
+            idx = float("inf")
+
+        return not item.is_eager, idx
+
+    return sorted(declaration_order, key=sort_key)
+
+
+class ParameterSource(enum.Enum):
+    """This is an :class:`~enum.Enum` that indicates the source of a
+    parameter's value.
+
+    Use :meth:`click.Context.get_parameter_source` to get the
+    source for a parameter by name.
+
+    .. versionchanged:: 8.0
+        Use :class:`~enum.Enum` and drop the ``validate`` method.
+
+    .. versionchanged:: 8.0
+        Added the ``PROMPT`` value.
+    """
+
+    COMMANDLINE = enum.auto()
+    """The value was provided by the command line args."""
+    ENVIRONMENT = enum.auto()
+    """The value was provided with an environment variable."""
+    DEFAULT = enum.auto()
+    """Used the default specified by the parameter."""
+    DEFAULT_MAP = enum.auto()
+    """Used a default provided by :attr:`Context.default_map`."""
+    PROMPT = enum.auto()
+    """Used a prompt to confirm a default or provide a value."""
+
+
+class Context:
+    """The context is a special internal object that holds state relevant
+    for the script execution at every single level.  It's normally invisible
+    to commands unless they opt-in to getting access to it.
+
+    The context is useful as it can pass internal objects around and can
+    control special execution features such as reading data from
+    environment variables.
+
+    A context can be used as context manager in which case it will call
+    :meth:`close` on teardown.
+
+    :param command: the command class for this context.
+    :param parent: the parent context.
+    :param info_name: the info name for this invocation.  Generally this
+                      is the most descriptive name for the script or
+                      command.  For the toplevel script it is usually
+                      the name of the script, for commands below it it's
+                      the name of the script.
+    :param obj: an arbitrary object of user data.
+    :param auto_envvar_prefix: the prefix to use for automatic environment
+                               variables.  If this is `None` then reading
+                               from environment variables is disabled.  This
+                               does not affect manually set environment
+                               variables which are always read.
+    :param default_map: a dictionary (like object) with default values
+                        for parameters.
+    :param terminal_width: the width of the terminal.  The default is
+                           inherit from parent context.  If no context
+                           defines the terminal width then auto
+                           detection will be applied.
+    :param max_content_width: the maximum width for content rendered by
+                              Click (this currently only affects help
+                              pages).  This defaults to 80 characters if
+                              not overridden.  In other words: even if the
+                              terminal is larger than that, Click will not
+                              format things wider than 80 characters by
+                              default.  In addition to that, formatters might
+                              add some safety mapping on the right.
+    :param resilient_parsing: if this flag is enabled then Click will
+                              parse without any interactivity or callback
+                              invocation.  Default values will also be
+                              ignored.  This is useful for implementing
+                              things such as completion support.
+    :param allow_extra_args: if this is set to `True` then extra arguments
+                             at the end will not raise an error and will be
+                             kept on the context.  The default is to inherit
+                             from the command.
+    :param allow_interspersed_args: if this is set to `False` then options
+                                    and arguments cannot be mixed.  The
+                                    default is to inherit from the command.
+    :param ignore_unknown_options: instructs click to ignore options it does
+                                   not know and keeps them for later
+                                   processing.
+    :param help_option_names: optionally a list of strings that define how
+                              the default help parameter is named.  The
+                              default is ``['--help']``.
+    :param token_normalize_func: an optional function that is used to
+                                 normalize tokens (options, choices,
+                                 etc.).  This for instance can be used to
+                                 implement case insensitive behavior.
+    :param color: controls if the terminal supports ANSI colors or not.  The
+                  default is autodetection.  This is only needed if ANSI
+                  codes are used in texts that Click prints which is by
+                  default not the case.  This for instance would affect
+                  help output.
+    :param show_default: Show the default value for commands. If this
+        value is not set, it defaults to the value from the parent
+        context. ``Command.show_default`` overrides this default for the
+        specific command.
+
+    .. versionchanged:: 8.2
+        The ``protected_args`` attribute is deprecated and will be removed in
+        Click 9.0. ``args`` will contain remaining unparsed tokens.
+
+    .. versionchanged:: 8.1
+        The ``show_default`` parameter is overridden by
+        ``Command.show_default``, instead of the other way around.
+
+    .. versionchanged:: 8.0
+        The ``show_default`` parameter defaults to the value from the
+        parent context.
+
+    .. versionchanged:: 7.1
+       Added the ``show_default`` parameter.
+
+    .. versionchanged:: 4.0
+        Added the ``color``, ``ignore_unknown_options``, and
+        ``max_content_width`` parameters.
+
+    .. versionchanged:: 3.0
+        Added the ``allow_extra_args`` and ``allow_interspersed_args``
+        parameters.
+
+    .. versionchanged:: 2.0
+        Added the ``resilient_parsing``, ``help_option_names``, and
+        ``token_normalize_func`` parameters.
+    """
+
+    #: The formatter class to create with :meth:`make_formatter`.
+    #:
+    #: .. versionadded:: 8.0
+    formatter_class: type[HelpFormatter] = HelpFormatter
+
+    def __init__(
+        self,
+        command: Command,
+        parent: Context | None = None,
+        info_name: str | None = None,
+        obj: t.Any | None = None,
+        auto_envvar_prefix: str | None = None,
+        default_map: cabc.MutableMapping[str, t.Any] | None = None,
+        terminal_width: int | None = None,
+        max_content_width: int | None = None,
+        resilient_parsing: bool = False,
+        allow_extra_args: bool | None = None,
+        allow_interspersed_args: bool | None = None,
+        ignore_unknown_options: bool | None = None,
+        help_option_names: list[str] | None = None,
+        token_normalize_func: t.Callable[[str], str] | None = None,
+        color: bool | None = None,
+        show_default: bool | None = None,
+    ) -> None:
+        #: the parent context or `None` if none exists.
+        self.parent = parent
+        #: the :class:`Command` for this context.
+        self.command = command
+        #: the descriptive information name
+        self.info_name = info_name
+        #: Map of parameter names to their parsed values. Parameters
+        #: with ``expose_value=False`` are not stored.
+        self.params: dict[str, t.Any] = {}
+        #: the leftover arguments.
+        self.args: list[str] = []
+        #: protected arguments.  These are arguments that are prepended
+        #: to `args` when certain parsing scenarios are encountered but
+        #: must be never propagated to another arguments.  This is used
+        #: to implement nested parsing.
+        self._protected_args: list[str] = []
+        #: the collected prefixes of the command's options.
+        self._opt_prefixes: set[str] = set(parent._opt_prefixes) if parent else set()
+
+        if obj is None and parent is not None:
+            obj = parent.obj
+
+        #: the user object stored.
+        self.obj: t.Any = obj
+        self._meta: dict[str, t.Any] = getattr(parent, "meta", {})
+
+        #: A dictionary (-like object) with defaults for parameters.
+        if (
+            default_map is None
+            and info_name is not None
+            and parent is not None
+            and parent.default_map is not None
+        ):
+            default_map = parent.default_map.get(info_name)
+
+        self.default_map: cabc.MutableMapping[str, t.Any] | None = default_map
+
+        #: This flag indicates if a subcommand is going to be executed. A
+        #: group callback can use this information to figure out if it's
+        #: being executed directly or because the execution flow passes
+        #: onwards to a subcommand. By default it's None, but it can be
+        #: the name of the subcommand to execute.
+        #:
+        #: If chaining is enabled this will be set to ``'*'`` in case
+        #: any commands are executed.  It is however not possible to
+        #: figure out which ones.  If you require this knowledge you
+        #: should use a :func:`result_callback`.
+        self.invoked_subcommand: str | None = None
+
+        if terminal_width is None and parent is not None:
+            terminal_width = parent.terminal_width
+
+        #: The width of the terminal (None is autodetection).
+        self.terminal_width: int | None = terminal_width
+
+        if max_content_width is None and parent is not None:
+            max_content_width = parent.max_content_width
+
+        #: The maximum width of formatted content (None implies a sensible
+        #: default which is 80 for most things).
+        self.max_content_width: int | None = max_content_width
+
+        if allow_extra_args is None:
+            allow_extra_args = command.allow_extra_args
+
+        #: Indicates if the context allows extra args or if it should
+        #: fail on parsing.
+        #:
+        #: .. versionadded:: 3.0
+        self.allow_extra_args = allow_extra_args
+
+        if allow_interspersed_args is None:
+            allow_interspersed_args = command.allow_interspersed_args
+
+        #: Indicates if the context allows mixing of arguments and
+        #: options or not.
+        #:
+        #: .. versionadded:: 3.0
+        self.allow_interspersed_args: bool = allow_interspersed_args
+
+        if ignore_unknown_options is None:
+            ignore_unknown_options = command.ignore_unknown_options
+
+        #: Instructs click to ignore options that a command does not
+        #: understand and will store it on the context for later
+        #: processing.  This is primarily useful for situations where you
+        #: want to call into external programs.  Generally this pattern is
+        #: strongly discouraged because it's not possibly to losslessly
+        #: forward all arguments.
+        #:
+        #: .. versionadded:: 4.0
+        self.ignore_unknown_options: bool = ignore_unknown_options
+
+        if help_option_names is None:
+            if parent is not None:
+                help_option_names = parent.help_option_names
+            else:
+                help_option_names = ["--help"]
+
+        #: The names for the help options.
+        self.help_option_names: list[str] = help_option_names
+
+        if token_normalize_func is None and parent is not None:
+            token_normalize_func = parent.token_normalize_func
+
+        #: An optional normalization function for tokens.  This is
+        #: options, choices, commands etc.
+        self.token_normalize_func: t.Callable[[str], str] | None = token_normalize_func
+
+        #: Indicates if resilient parsing is enabled.  In that case Click
+        #: will do its best to not cause any failures and default values
+        #: will be ignored. Useful for completion.
+        self.resilient_parsing: bool = resilient_parsing
+
+        # If there is no envvar prefix yet, but the parent has one and
+        # the command on this level has a name, we can expand the envvar
+        # prefix automatically.
+        if auto_envvar_prefix is None:
+            if (
+                parent is not None
+                and parent.auto_envvar_prefix is not None
+                and self.info_name is not None
+            ):
+                auto_envvar_prefix = (
+                    f"{parent.auto_envvar_prefix}_{self.info_name.upper()}"
+                )
+        else:
+            auto_envvar_prefix = auto_envvar_prefix.upper()
+
+        if auto_envvar_prefix is not None:
+            auto_envvar_prefix = auto_envvar_prefix.replace("-", "_")
+
+        self.auto_envvar_prefix: str | None = auto_envvar_prefix
+
+        if color is None and parent is not None:
+            color = parent.color
+
+        #: Controls if styling output is wanted or not.
+        self.color: bool | None = color
+
+        if show_default is None and parent is not None:
+            show_default = parent.show_default
+
+        #: Show option default values when formatting help text.
+        self.show_default: bool | None = show_default
+
+        self._close_callbacks: list[t.Callable[[], t.Any]] = []
+        self._depth = 0
+        self._parameter_source: dict[str, ParameterSource] = {}
+        self._exit_stack = ExitStack()
+
+    @property
+    def protected_args(self) -> list[str]:
+        import warnings
+
+        warnings.warn(
+            "'protected_args' is deprecated and will be removed in Click 9.0."
+            " 'args' will contain remaining unparsed tokens.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self._protected_args
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        """Gather information that could be useful for a tool generating
+        user-facing documentation. This traverses the entire CLI
+        structure.
+
+        .. code-block:: python
+
+            with Context(cli) as ctx:
+                info = ctx.to_info_dict()
+
+        .. versionadded:: 8.0
+        """
+        return {
+            "command": self.command.to_info_dict(self),
+            "info_name": self.info_name,
+            "allow_extra_args": self.allow_extra_args,
+            "allow_interspersed_args": self.allow_interspersed_args,
+            "ignore_unknown_options": self.ignore_unknown_options,
+            "auto_envvar_prefix": self.auto_envvar_prefix,
+        }
+
+    def __enter__(self) -> Context:
+        self._depth += 1
+        push_context(self)
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        tb: TracebackType | None,
+    ) -> bool | None:
+        self._depth -= 1
+        exit_result: bool | None = None
+        if self._depth == 0:
+            exit_result = self._close_with_exception_info(exc_type, exc_value, tb)
+        pop_context()
+
+        return exit_result
+
+    @contextmanager
+    def scope(self, cleanup: bool = True) -> cabc.Iterator[Context]:
+        """This helper method can be used with the context object to promote
+        it to the current thread local (see :func:`get_current_context`).
+        The default behavior of this is to invoke the cleanup functions which
+        can be disabled by setting `cleanup` to `False`.  The cleanup
+        functions are typically used for things such as closing file handles.
+
+        If the cleanup is intended the context object can also be directly
+        used as a context manager.
+
+        Example usage::
+
+            with ctx.scope():
+                assert get_current_context() is ctx
+
+        This is equivalent::
+
+            with ctx:
+                assert get_current_context() is ctx
+
+        .. versionadded:: 5.0
+
+        :param cleanup: controls if the cleanup functions should be run or
+                        not.  The default is to run these functions.  In
+                        some situations the context only wants to be
+                        temporarily pushed in which case this can be disabled.
+                        Nested pushes automatically defer the cleanup.
+        """
+        if not cleanup:
+            self._depth += 1
+        try:
+            with self as rv:
+                yield rv
+        finally:
+            if not cleanup:
+                self._depth -= 1
+
+    @property
+    def meta(self) -> dict[str, t.Any]:
+        """This is a dictionary which is shared with all the contexts
+        that are nested.  It exists so that click utilities can store some
+        state here if they need to.  It is however the responsibility of
+        that code to manage this dictionary well.
+
+        The keys are supposed to be unique dotted strings.  For instance
+        module paths are a good choice for it.  What is stored in there is
+        irrelevant for the operation of click.  However what is important is
+        that code that places data here adheres to the general semantics of
+        the system.
+
+        Example usage::
+
+            LANG_KEY = f'{__name__}.lang'
+
+            def set_language(value):
+                ctx = get_current_context()
+                ctx.meta[LANG_KEY] = value
+
+            def get_language():
+                return get_current_context().meta.get(LANG_KEY, 'en_US')
+
+        .. versionadded:: 5.0
+        """
+        return self._meta
+
+    def make_formatter(self) -> HelpFormatter:
+        """Creates the :class:`~click.HelpFormatter` for the help and
+        usage output.
+
+        To quickly customize the formatter class used without overriding
+        this method, set the :attr:`formatter_class` attribute.
+
+        .. versionchanged:: 8.0
+            Added the :attr:`formatter_class` attribute.
+        """
+        return self.formatter_class(
+            width=self.terminal_width, max_width=self.max_content_width
+        )
+
+    def with_resource(self, context_manager: AbstractContextManager[V]) -> V:
+        """Register a resource as if it were used in a ``with``
+        statement. The resource will be cleaned up when the context is
+        popped.
+
+        Uses :meth:`contextlib.ExitStack.enter_context`. It calls the
+        resource's ``__enter__()`` method and returns the result. When
+        the context is popped, it closes the stack, which calls the
+        resource's ``__exit__()`` method.
+
+        To register a cleanup function for something that isn't a
+        context manager, use :meth:`call_on_close`. Or use something
+        from :mod:`contextlib` to turn it into a context manager first.
+
+        .. code-block:: python
+
+            @click.group()
+            @click.option("--name")
+            @click.pass_context
+            def cli(ctx):
+                ctx.obj = ctx.with_resource(connect_db(name))
+
+        :param context_manager: The context manager to enter.
+        :return: Whatever ``context_manager.__enter__()`` returns.
+
+        .. versionadded:: 8.0
+        """
+        return self._exit_stack.enter_context(context_manager)
+
+    def call_on_close(self, f: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]:
+        """Register a function to be called when the context tears down.
+
+        This can be used to close resources opened during the script
+        execution. Resources that support Python's context manager
+        protocol which would be used in a ``with`` statement should be
+        registered with :meth:`with_resource` instead.
+
+        :param f: The function to execute on teardown.
+        """
+        return self._exit_stack.callback(f)
+
+    def close(self) -> None:
+        """Invoke all close callbacks registered with
+        :meth:`call_on_close`, and exit all context managers entered
+        with :meth:`with_resource`.
+        """
+        self._close_with_exception_info(None, None, None)
+
+    def _close_with_exception_info(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        tb: TracebackType | None,
+    ) -> bool | None:
+        """Unwind the exit stack by calling its :meth:`__exit__` providing the exception
+        information to allow for exception handling by the various resources registered
+        using :meth;`with_resource`
+
+        :return: Whatever ``exit_stack.__exit__()`` returns.
+        """
+        exit_result = self._exit_stack.__exit__(exc_type, exc_value, tb)
+        # In case the context is reused, create a new exit stack.
+        self._exit_stack = ExitStack()
+
+        return exit_result
+
+    @property
+    def command_path(self) -> str:
+        """The computed command path.  This is used for the ``usage``
+        information on the help page.  It's automatically created by
+        combining the info names of the chain of contexts to the root.
+        """
+        rv = ""
+        if self.info_name is not None:
+            rv = self.info_name
+        if self.parent is not None:
+            parent_command_path = [self.parent.command_path]
+
+            if isinstance(self.parent.command, Command):
+                for param in self.parent.command.get_params(self):
+                    parent_command_path.extend(param.get_usage_pieces(self))
+
+            rv = f"{' '.join(parent_command_path)} {rv}"
+        return rv.lstrip()
+
+    def find_root(self) -> Context:
+        """Finds the outermost context."""
+        node = self
+        while node.parent is not None:
+            node = node.parent
+        return node
+
+    def find_object(self, object_type: type[V]) -> V | None:
+        """Finds the closest object of a given type."""
+        node: Context | None = self
+
+        while node is not None:
+            if isinstance(node.obj, object_type):
+                return node.obj
+
+            node = node.parent
+
+        return None
+
+    def ensure_object(self, object_type: type[V]) -> V:
+        """Like :meth:`find_object` but sets the innermost object to a
+        new instance of `object_type` if it does not exist.
+        """
+        rv = self.find_object(object_type)
+        if rv is None:
+            self.obj = rv = object_type()
+        return rv
+
+    @t.overload
+    def lookup_default(
+        self, name: str, call: t.Literal[True] = True
+    ) -> t.Any | None: ...
+
+    @t.overload
+    def lookup_default(
+        self, name: str, call: t.Literal[False] = ...
+    ) -> t.Any | t.Callable[[], t.Any] | None: ...
+
+    def lookup_default(self, name: str, call: bool = True) -> t.Any | None:
+        """Get the default for a parameter from :attr:`default_map`.
+
+        :param name: Name of the parameter.
+        :param call: If the default is a callable, call it. Disable to
+            return the callable instead.
+
+        .. versionchanged:: 8.0
+            Added the ``call`` parameter.
+        """
+        if self.default_map is not None:
+            value = self.default_map.get(name)
+
+            if call and callable(value):
+                return value()
+
+            return value
+
+        return None
+
+    def fail(self, message: str) -> t.NoReturn:
+        """Aborts the execution of the program with a specific error
+        message.
+
+        :param message: the error message to fail with.
+        """
+        raise UsageError(message, self)
+
+    def abort(self) -> t.NoReturn:
+        """Aborts the script."""
+        raise Abort()
+
+    def exit(self, code: int = 0) -> t.NoReturn:
+        """Exits the application with a given exit code.
+
+        .. versionchanged:: 8.2
+            Callbacks and context managers registered with :meth:`call_on_close`
+            and :meth:`with_resource` are closed before exiting.
+        """
+        self.close()
+        raise Exit(code)
+
+    def get_usage(self) -> str:
+        """Helper method to get formatted usage string for the current
+        context and command.
+        """
+        return self.command.get_usage(self)
+
+    def get_help(self) -> str:
+        """Helper method to get formatted help page for the current
+        context and command.
+        """
+        return self.command.get_help(self)
+
+    def _make_sub_context(self, command: Command) -> Context:
+        """Create a new context of the same type as this context, but
+        for a new command.
+
+        :meta private:
+        """
+        return type(self)(command, info_name=command.name, parent=self)
+
+    @t.overload
+    def invoke(
+        self, callback: t.Callable[..., V], /, *args: t.Any, **kwargs: t.Any
+    ) -> V: ...
+
+    @t.overload
+    def invoke(self, callback: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any: ...
+
+    def invoke(
+        self, callback: Command | t.Callable[..., V], /, *args: t.Any, **kwargs: t.Any
+    ) -> t.Any | V:
+        """Invokes a command callback in exactly the way it expects.  There
+        are two ways to invoke this method:
+
+        1.  the first argument can be a callback and all other arguments and
+            keyword arguments are forwarded directly to the function.
+        2.  the first argument is a click command object.  In that case all
+            arguments are forwarded as well but proper click parameters
+            (options and click arguments) must be keyword arguments and Click
+            will fill in defaults.
+
+        .. versionchanged:: 8.0
+            All ``kwargs`` are tracked in :attr:`params` so they will be
+            passed if :meth:`forward` is called at multiple levels.
+
+        .. versionchanged:: 3.2
+            A new context is created, and missing arguments use default values.
+        """
+        if isinstance(callback, Command):
+            other_cmd = callback
+
+            if other_cmd.callback is None:
+                raise TypeError(
+                    "The given command does not have a callback that can be invoked."
+                )
+            else:
+                callback = t.cast("t.Callable[..., V]", other_cmd.callback)
+
+            ctx = self._make_sub_context(other_cmd)
+
+            for param in other_cmd.params:
+                if param.name not in kwargs and param.expose_value:
+                    default_value = param.get_default(ctx)
+                    # We explicitly hide the :attr:`UNSET` value to the user, as we
+                    # choose to make it an implementation detail. And because ``invoke``
+                    # has been designed as part of Click public API, we return ``None``
+                    # instead. Refs:
+                    # https://github.com/pallets/click/issues/3066
+                    # https://github.com/pallets/click/issues/3065
+                    # https://github.com/pallets/click/pull/3068
+                    if default_value is UNSET:
+                        default_value = None
+                    kwargs[param.name] = param.type_cast_value(  # type: ignore
+                        ctx, default_value
+                    )
+
+            # Track all kwargs as params, so that forward() will pass
+            # them on in subsequent calls.
+            ctx.params.update(kwargs)
+        else:
+            ctx = self
+
+        with augment_usage_errors(self):
+            with ctx:
+                return callback(*args, **kwargs)
+
+    def forward(self, cmd: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any:
+        """Similar to :meth:`invoke` but fills in default keyword
+        arguments from the current context if the other command expects
+        it.  This cannot invoke callbacks directly, only other commands.
+
+        .. versionchanged:: 8.0
+            All ``kwargs`` are tracked in :attr:`params` so they will be
+            passed if ``forward`` is called at multiple levels.
+        """
+        # Can only forward to other commands, not direct callbacks.
+        if not isinstance(cmd, Command):
+            raise TypeError("Callback is not a command.")
+
+        for param in self.params:
+            if param not in kwargs:
+                kwargs[param] = self.params[param]
+
+        return self.invoke(cmd, *args, **kwargs)
+
+    def set_parameter_source(self, name: str, source: ParameterSource) -> None:
+        """Set the source of a parameter. This indicates the location
+        from which the value of the parameter was obtained.
+
+        :param name: The name of the parameter.
+        :param source: A member of :class:`~click.core.ParameterSource`.
+        """
+        self._parameter_source[name] = source
+
+    def get_parameter_source(self, name: str) -> ParameterSource | None:
+        """Get the source of a parameter. This indicates the location
+        from which the value of the parameter was obtained.
+
+        This can be useful for determining when a user specified a value
+        on the command line that is the same as the default value. It
+        will be :attr:`~click.core.ParameterSource.DEFAULT` only if the
+        value was actually taken from the default.
+
+        :param name: The name of the parameter.
+        :rtype: ParameterSource
+
+        .. versionchanged:: 8.0
+            Returns ``None`` if the parameter was not provided from any
+            source.
+        """
+        return self._parameter_source.get(name)
+
+
+class Command:
+    """Commands are the basic building block of command line interfaces in
+    Click.  A basic command handles command line parsing and might dispatch
+    more parsing to commands nested below it.
+
+    :param name: the name of the command to use unless a group overrides it.
+    :param context_settings: an optional dictionary with defaults that are
+                             passed to the context object.
+    :param callback: the callback to invoke.  This is optional.
+    :param params: the parameters to register with this command.  This can
+                   be either :class:`Option` or :class:`Argument` objects.
+    :param help: the help string to use for this command.
+    :param epilog: like the help string but it's printed at the end of the
+                   help page after everything else.
+    :param short_help: the short help to use for this command.  This is
+                       shown on the command listing of the parent command.
+    :param add_help_option: by default each command registers a ``--help``
+                            option.  This can be disabled by this parameter.
+    :param no_args_is_help: this controls what happens if no arguments are
+                            provided.  This option is disabled by default.
+                            If enabled this will add ``--help`` as argument
+                            if no arguments are passed
+    :param hidden: hide this command from help outputs.
+    :param deprecated: If ``True`` or non-empty string, issues a message
+                        indicating that the command is deprecated and highlights
+                        its deprecation in --help. The message can be customized
+                        by using a string as the value.
+
+    .. versionchanged:: 8.2
+        This is the base class for all commands, not ``BaseCommand``.
+        ``deprecated`` can be set to a string as well to customize the
+        deprecation message.
+
+    .. versionchanged:: 8.1
+        ``help``, ``epilog``, and ``short_help`` are stored unprocessed,
+        all formatting is done when outputting help text, not at init,
+        and is done even if not using the ``@command`` decorator.
+
+    .. versionchanged:: 8.0
+        Added a ``repr`` showing the command name.
+
+    .. versionchanged:: 7.1
+        Added the ``no_args_is_help`` parameter.
+
+    .. versionchanged:: 2.0
+        Added the ``context_settings`` parameter.
+    """
+
+    #: The context class to create with :meth:`make_context`.
+    #:
+    #: .. versionadded:: 8.0
+    context_class: type[Context] = Context
+
+    #: the default for the :attr:`Context.allow_extra_args` flag.
+    allow_extra_args = False
+
+    #: the default for the :attr:`Context.allow_interspersed_args` flag.
+    allow_interspersed_args = True
+
+    #: the default for the :attr:`Context.ignore_unknown_options` flag.
+    ignore_unknown_options = False
+
+    def __init__(
+        self,
+        name: str | None,
+        context_settings: cabc.MutableMapping[str, t.Any] | None = None,
+        callback: t.Callable[..., t.Any] | None = None,
+        params: list[Parameter] | None = None,
+        help: str | None = None,
+        epilog: str | None = None,
+        short_help: str | None = None,
+        options_metavar: str | None = "[OPTIONS]",
+        add_help_option: bool = True,
+        no_args_is_help: bool = False,
+        hidden: bool = False,
+        deprecated: bool | str = False,
+    ) -> None:
+        #: the name the command thinks it has.  Upon registering a command
+        #: on a :class:`Group` the group will default the command name
+        #: with this information.  You should instead use the
+        #: :class:`Context`\'s :attr:`~Context.info_name` attribute.
+        self.name = name
+
+        if context_settings is None:
+            context_settings = {}
+
+        #: an optional dictionary with defaults passed to the context.
+        self.context_settings: cabc.MutableMapping[str, t.Any] = context_settings
+
+        #: the callback to execute when the command fires.  This might be
+        #: `None` in which case nothing happens.
+        self.callback = callback
+        #: the list of parameters for this command in the order they
+        #: should show up in the help page and execute.  Eager parameters
+        #: will automatically be handled before non eager ones.
+        self.params: list[Parameter] = params or []
+        self.help = help
+        self.epilog = epilog
+        self.options_metavar = options_metavar
+        self.short_help = short_help
+        self.add_help_option = add_help_option
+        self._help_option = None
+        self.no_args_is_help = no_args_is_help
+        self.hidden = hidden
+        self.deprecated = deprecated
+
+    def to_info_dict(self, ctx: Context) -> dict[str, t.Any]:
+        return {
+            "name": self.name,
+            "params": [param.to_info_dict() for param in self.get_params(ctx)],
+            "help": self.help,
+            "epilog": self.epilog,
+            "short_help": self.short_help,
+            "hidden": self.hidden,
+            "deprecated": self.deprecated,
+        }
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.name}>"
+
+    def get_usage(self, ctx: Context) -> str:
+        """Formats the usage line into a string and returns it.
+
+        Calls :meth:`format_usage` internally.
+        """
+        formatter = ctx.make_formatter()
+        self.format_usage(ctx, formatter)
+        return formatter.getvalue().rstrip("\n")
+
+    def get_params(self, ctx: Context) -> list[Parameter]:
+        params = self.params
+        help_option = self.get_help_option(ctx)
+
+        if help_option is not None:
+            params = [*params, help_option]
+
+        if __debug__:
+            import warnings
+
+            opts = [opt for param in params for opt in param.opts]
+            opts_counter = Counter(opts)
+            duplicate_opts = (opt for opt, count in opts_counter.items() if count > 1)
+
+            for duplicate_opt in duplicate_opts:
+                warnings.warn(
+                    (
+                        f"The parameter {duplicate_opt} is used more than once. "
+                        "Remove its duplicate as parameters should be unique."
+                    ),
+                    stacklevel=3,
+                )
+
+        return params
+
+    def format_usage(self, ctx: Context, formatter: HelpFormatter) -> None:
+        """Writes the usage line into the formatter.
+
+        This is a low-level method called by :meth:`get_usage`.
+        """
+        pieces = self.collect_usage_pieces(ctx)
+        formatter.write_usage(ctx.command_path, " ".join(pieces))
+
+    def collect_usage_pieces(self, ctx: Context) -> list[str]:
+        """Returns all the pieces that go into the usage line and returns
+        it as a list of strings.
+        """
+        rv = [self.options_metavar] if self.options_metavar else []
+
+        for param in self.get_params(ctx):
+            rv.extend(param.get_usage_pieces(ctx))
+
+        return rv
+
+    def get_help_option_names(self, ctx: Context) -> list[str]:
+        """Returns the names for the help option."""
+        all_names = set(ctx.help_option_names)
+        for param in self.params:
+            all_names.difference_update(param.opts)
+            all_names.difference_update(param.secondary_opts)
+        return list(all_names)
+
+    def get_help_option(self, ctx: Context) -> Option | None:
+        """Returns the help option object.
+
+        Skipped if :attr:`add_help_option` is ``False``.
+
+        .. versionchanged:: 8.1.8
+            The help option is now cached to avoid creating it multiple times.
+        """
+        help_option_names = self.get_help_option_names(ctx)
+
+        if not help_option_names or not self.add_help_option:
+            return None
+
+        # Cache the help option object in private _help_option attribute to
+        # avoid creating it multiple times. Not doing this will break the
+        # callback odering by iter_params_for_processing(), which relies on
+        # object comparison.
+        if self._help_option is None:
+            # Avoid circular import.
+            from .decorators import help_option
+
+            # Apply help_option decorator and pop resulting option
+            help_option(*help_option_names)(self)
+            self._help_option = self.params.pop()  # type: ignore[assignment]
+
+        return self._help_option
+
+    def make_parser(self, ctx: Context) -> _OptionParser:
+        """Creates the underlying option parser for this command."""
+        parser = _OptionParser(ctx)
+        for param in self.get_params(ctx):
+            param.add_to_parser(parser, ctx)
+        return parser
+
+    def get_help(self, ctx: Context) -> str:
+        """Formats the help into a string and returns it.
+
+        Calls :meth:`format_help` internally.
+        """
+        formatter = ctx.make_formatter()
+        self.format_help(ctx, formatter)
+        return formatter.getvalue().rstrip("\n")
+
+    def get_short_help_str(self, limit: int = 45) -> str:
+        """Gets short help for the command or makes it by shortening the
+        long help string.
+        """
+        if self.short_help:
+            text = inspect.cleandoc(self.short_help)
+        elif self.help:
+            text = make_default_short_help(self.help, limit)
+        else:
+            text = ""
+
+        if self.deprecated:
+            deprecated_message = (
+                f"(DEPRECATED: {self.deprecated})"
+                if isinstance(self.deprecated, str)
+                else "(DEPRECATED)"
+            )
+            text = _("{text} {deprecated_message}").format(
+                text=text, deprecated_message=deprecated_message
+            )
+
+        return text.strip()
+
+    def format_help(self, ctx: Context, formatter: HelpFormatter) -> None:
+        """Writes the help into the formatter if it exists.
+
+        This is a low-level method called by :meth:`get_help`.
+
+        This calls the following methods:
+
+        -   :meth:`format_usage`
+        -   :meth:`format_help_text`
+        -   :meth:`format_options`
+        -   :meth:`format_epilog`
+        """
+        self.format_usage(ctx, formatter)
+        self.format_help_text(ctx, formatter)
+        self.format_options(ctx, formatter)
+        self.format_epilog(ctx, formatter)
+
+    def format_help_text(self, ctx: Context, formatter: HelpFormatter) -> None:
+        """Writes the help text to the formatter if it exists."""
+        if self.help is not None:
+            # truncate the help text to the first form feed
+            text = inspect.cleandoc(self.help).partition("\f")[0]
+        else:
+            text = ""
+
+        if self.deprecated:
+            deprecated_message = (
+                f"(DEPRECATED: {self.deprecated})"
+                if isinstance(self.deprecated, str)
+                else "(DEPRECATED)"
+            )
+            text = _("{text} {deprecated_message}").format(
+                text=text, deprecated_message=deprecated_message
+            )
+
+        if text:
+            formatter.write_paragraph()
+
+            with formatter.indentation():
+                formatter.write_text(text)
+
+    def format_options(self, ctx: Context, formatter: HelpFormatter) -> None:
+        """Writes all the options into the formatter if they exist."""
+        opts = []
+        for param in self.get_params(ctx):
+            rv = param.get_help_record(ctx)
+            if rv is not None:
+                opts.append(rv)
+
+        if opts:
+            with formatter.section(_("Options")):
+                formatter.write_dl(opts)
+
+    def format_epilog(self, ctx: Context, formatter: HelpFormatter) -> None:
+        """Writes the epilog into the formatter if it exists."""
+        if self.epilog:
+            epilog = inspect.cleandoc(self.epilog)
+            formatter.write_paragraph()
+
+            with formatter.indentation():
+                formatter.write_text(epilog)
+
+    def make_context(
+        self,
+        info_name: str | None,
+        args: list[str],
+        parent: Context | None = None,
+        **extra: t.Any,
+    ) -> Context:
+        """This function when given an info name and arguments will kick
+        off the parsing and create a new :class:`Context`.  It does not
+        invoke the actual command callback though.
+
+        To quickly customize the context class used without overriding
+        this method, set the :attr:`context_class` attribute.
+
+        :param info_name: the info name for this invocation.  Generally this
+                          is the most descriptive name for the script or
+                          command.  For the toplevel script it's usually
+                          the name of the script, for commands below it's
+                          the name of the command.
+        :param args: the arguments to parse as list of strings.
+        :param parent: the parent context if available.
+        :param extra: extra keyword arguments forwarded to the context
+                      constructor.
+
+        .. versionchanged:: 8.0
+            Added the :attr:`context_class` attribute.
+        """
+        for key, value in self.context_settings.items():
+            if key not in extra:
+                extra[key] = value
+
+        ctx = self.context_class(self, info_name=info_name, parent=parent, **extra)
+
+        with ctx.scope(cleanup=False):
+            self.parse_args(ctx, args)
+        return ctx
+
+    def parse_args(self, ctx: Context, args: list[str]) -> list[str]:
+        if not args and self.no_args_is_help and not ctx.resilient_parsing:
+            raise NoArgsIsHelpError(ctx)
+
+        parser = self.make_parser(ctx)
+        opts, args, param_order = parser.parse_args(args=args)
+
+        for param in iter_params_for_processing(param_order, self.get_params(ctx)):
+            _, args = param.handle_parse_result(ctx, opts, args)
+
+        # We now have all parameters' values into `ctx.params`, but the data may contain
+        # the `UNSET` sentinel.
+        # Convert `UNSET` to `None` to ensure that the user doesn't see `UNSET`.
+        #
+        # Waiting until after the initial parse to convert allows us to treat `UNSET`
+        # more like a missing value when multiple params use the same name.
+        # Refs:
+        # https://github.com/pallets/click/issues/3071
+        # https://github.com/pallets/click/pull/3079
+        for name, value in ctx.params.items():
+            if value is UNSET:
+                ctx.params[name] = None
+
+        if args and not ctx.allow_extra_args and not ctx.resilient_parsing:
+            ctx.fail(
+                ngettext(
+                    "Got unexpected extra argument ({args})",
+                    "Got unexpected extra arguments ({args})",
+                    len(args),
+                ).format(args=" ".join(map(str, args)))
+            )
+
+        ctx.args = args
+        ctx._opt_prefixes.update(parser._opt_prefixes)
+        return args
+
+    def invoke(self, ctx: Context) -> t.Any:
+        """Given a context, this invokes the attached callback (if it exists)
+        in the right way.
+        """
+        if self.deprecated:
+            extra_message = (
+                f" {self.deprecated}" if isinstance(self.deprecated, str) else ""
+            )
+            message = _(
+                "DeprecationWarning: The command {name!r} is deprecated.{extra_message}"
+            ).format(name=self.name, extra_message=extra_message)
+            echo(style(message, fg="red"), err=True)
+
+        if self.callback is not None:
+            return ctx.invoke(self.callback, **ctx.params)
+
+    def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]:
+        """Return a list of completions for the incomplete value. Looks
+        at the names of options and chained multi-commands.
+
+        Any command could be part of a chained multi-command, so sibling
+        commands are valid at any point during command completion.
+
+        :param ctx: Invocation context for this command.
+        :param incomplete: Value being completed. May be empty.
+
+        .. versionadded:: 8.0
+        """
+        from click.shell_completion import CompletionItem
+
+        results: list[CompletionItem] = []
+
+        if incomplete and not incomplete[0].isalnum():
+            for param in self.get_params(ctx):
+                if (
+                    not isinstance(param, Option)
+                    or param.hidden
+                    or (
+                        not param.multiple
+                        and ctx.get_parameter_source(param.name)  # type: ignore
+                        is ParameterSource.COMMANDLINE
+                    )
+                ):
+                    continue
+
+                results.extend(
+                    CompletionItem(name, help=param.help)
+                    for name in [*param.opts, *param.secondary_opts]
+                    if name.startswith(incomplete)
+                )
+
+        while ctx.parent is not None:
+            ctx = ctx.parent
+
+            if isinstance(ctx.command, Group) and ctx.command.chain:
+                results.extend(
+                    CompletionItem(name, help=command.get_short_help_str())
+                    for name, command in _complete_visible_commands(ctx, incomplete)
+                    if name not in ctx._protected_args
+                )
+
+        return results
+
+    @t.overload
+    def main(
+        self,
+        args: cabc.Sequence[str] | None = None,
+        prog_name: str | None = None,
+        complete_var: str | None = None,
+        standalone_mode: t.Literal[True] = True,
+        **extra: t.Any,
+    ) -> t.NoReturn: ...
+
+    @t.overload
+    def main(
+        self,
+        args: cabc.Sequence[str] | None = None,
+        prog_name: str | None = None,
+        complete_var: str | None = None,
+        standalone_mode: bool = ...,
+        **extra: t.Any,
+    ) -> t.Any: ...
+
+    def main(
+        self,
+        args: cabc.Sequence[str] | None = None,
+        prog_name: str | None = None,
+        complete_var: str | None = None,
+        standalone_mode: bool = True,
+        windows_expand_args: bool = True,
+        **extra: t.Any,
+    ) -> t.Any:
+        """This is the way to invoke a script with all the bells and
+        whistles as a command line application.  This will always terminate
+        the application after a call.  If this is not wanted, ``SystemExit``
+        needs to be caught.
+
+        This method is also available by directly calling the instance of
+        a :class:`Command`.
+
+        :param args: the arguments that should be used for parsing.  If not
+                     provided, ``sys.argv[1:]`` is used.
+        :param prog_name: the program name that should be used.  By default
+                          the program name is constructed by taking the file
+                          name from ``sys.argv[0]``.
+        :param complete_var: the environment variable that controls the
+                             bash completion support.  The default is
+                             ``"_<prog_name>_COMPLETE"`` with prog_name in
+                             uppercase.
+        :param standalone_mode: the default behavior is to invoke the script
+                                in standalone mode.  Click will then
+                                handle exceptions and convert them into
+                                error messages and the function will never
+                                return but shut down the interpreter.  If
+                                this is set to `False` they will be
+                                propagated to the caller and the return
+                                value of this function is the return value
+                                of :meth:`invoke`.
+        :param windows_expand_args: Expand glob patterns, user dir, and
+            env vars in command line args on Windows.
+        :param extra: extra keyword arguments are forwarded to the context
+                      constructor.  See :class:`Context` for more information.
+
+        .. versionchanged:: 8.0.1
+            Added the ``windows_expand_args`` parameter to allow
+            disabling command line arg expansion on Windows.
+
+        .. versionchanged:: 8.0
+            When taking arguments from ``sys.argv`` on Windows, glob
+            patterns, user dir, and env vars are expanded.
+
+        .. versionchanged:: 3.0
+           Added the ``standalone_mode`` parameter.
+        """
+        if args is None:
+            args = sys.argv[1:]
+
+            if os.name == "nt" and windows_expand_args:
+                args = _expand_args(args)
+        else:
+            args = list(args)
+
+        if prog_name is None:
+            prog_name = _detect_program_name()
+
+        # Process shell completion requests and exit early.
+        self._main_shell_completion(extra, prog_name, complete_var)
+
+        try:
+            try:
+                with self.make_context(prog_name, args, **extra) as ctx:
+                    rv = self.invoke(ctx)
+                    if not standalone_mode:
+                        return rv
+                    # it's not safe to `ctx.exit(rv)` here!
+                    # note that `rv` may actually contain data like "1" which
+                    # has obvious effects
+                    # more subtle case: `rv=[None, None]` can come out of
+                    # chained commands which all returned `None` -- so it's not
+                    # even always obvious that `rv` indicates success/failure
+                    # by its truthiness/falsiness
+                    ctx.exit()
+            except (EOFError, KeyboardInterrupt) as e:
+                echo(file=sys.stderr)
+                raise Abort() from e
+            except ClickException as e:
+                if not standalone_mode:
+                    raise
+                e.show()
+                sys.exit(e.exit_code)
+            except OSError as e:
+                if e.errno == errno.EPIPE:
+                    sys.stdout = t.cast(t.TextIO, PacifyFlushWrapper(sys.stdout))
+                    sys.stderr = t.cast(t.TextIO, PacifyFlushWrapper(sys.stderr))
+                    sys.exit(1)
+                else:
+                    raise
+        except Exit as e:
+            if standalone_mode:
+                sys.exit(e.exit_code)
+            else:
+                # in non-standalone mode, return the exit code
+                # note that this is only reached if `self.invoke` above raises
+                # an Exit explicitly -- thus bypassing the check there which
+                # would return its result
+                # the results of non-standalone execution may therefore be
+                # somewhat ambiguous: if there are codepaths which lead to
+                # `ctx.exit(1)` and to `return 1`, the caller won't be able to
+                # tell the difference between the two
+                return e.exit_code
+        except Abort:
+            if not standalone_mode:
+                raise
+            echo(_("Aborted!"), file=sys.stderr)
+            sys.exit(1)
+
+    def _main_shell_completion(
+        self,
+        ctx_args: cabc.MutableMapping[str, t.Any],
+        prog_name: str,
+        complete_var: str | None = None,
+    ) -> None:
+        """Check if the shell is asking for tab completion, process
+        that, then exit early. Called from :meth:`main` before the
+        program is invoked.
+
+        :param prog_name: Name of the executable in the shell.
+        :param complete_var: Name of the environment variable that holds
+            the completion instruction. Defaults to
+            ``_{PROG_NAME}_COMPLETE``.
+
+        .. versionchanged:: 8.2.0
+            Dots (``.``) in ``prog_name`` are replaced with underscores (``_``).
+        """
+        if complete_var is None:
+            complete_name = prog_name.replace("-", "_").replace(".", "_")
+            complete_var = f"_{complete_name}_COMPLETE".upper()
+
+        instruction = os.environ.get(complete_var)
+
+        if not instruction:
+            return
+
+        from .shell_completion import shell_complete
+
+        rv = shell_complete(self, ctx_args, prog_name, complete_var, instruction)
+        sys.exit(rv)
+
+    def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:
+        """Alias for :meth:`main`."""
+        return self.main(*args, **kwargs)
+
+
+class _FakeSubclassCheck(type):
+    def __subclasscheck__(cls, subclass: type) -> bool:
+        return issubclass(subclass, cls.__bases__[0])
+
+    def __instancecheck__(cls, instance: t.Any) -> bool:
+        return isinstance(instance, cls.__bases__[0])
+
+
+class _BaseCommand(Command, metaclass=_FakeSubclassCheck):
+    """
+    .. deprecated:: 8.2
+        Will be removed in Click 9.0. Use ``Command`` instead.
+    """
+
+
+class Group(Command):
+    """A group is a command that nests other commands (or more groups).
+
+    :param name: The name of the group command.
+    :param commands: Map names to :class:`Command` objects. Can be a list, which
+        will use :attr:`Command.name` as the keys.
+    :param invoke_without_command: Invoke the group's callback even if a
+        subcommand is not given.
+    :param no_args_is_help: If no arguments are given, show the group's help and
+        exit. Defaults to the opposite of ``invoke_without_command``.
+    :param subcommand_metavar: How to represent the subcommand argument in help.
+        The default will represent whether ``chain`` is set or not.
+    :param chain: Allow passing more than one subcommand argument. After parsing
+        a command's arguments, if any arguments remain another command will be
+        matched, and so on.
+    :param result_callback: A function to call after the group's and
+        subcommand's callbacks. The value returned by the subcommand is passed.
+        If ``chain`` is enabled, the value will be a list of values returned by
+        all the commands. If ``invoke_without_command`` is enabled, the value
+        will be the value returned by the group's callback, or an empty list if
+        ``chain`` is enabled.
+    :param kwargs: Other arguments passed to :class:`Command`.
+
+    .. versionchanged:: 8.0
+        The ``commands`` argument can be a list of command objects.
+
+    .. versionchanged:: 8.2
+        Merged with and replaces the ``MultiCommand`` base class.
+    """
+
+    allow_extra_args = True
+    allow_interspersed_args = False
+
+    #: If set, this is used by the group's :meth:`command` decorator
+    #: as the default :class:`Command` class. This is useful to make all
+    #: subcommands use a custom command class.
+    #:
+    #: .. versionadded:: 8.0
+    command_class: type[Command] | None = None
+
+    #: If set, this is used by the group's :meth:`group` decorator
+    #: as the default :class:`Group` class. This is useful to make all
+    #: subgroups use a custom group class.
+    #:
+    #: If set to the special value :class:`type` (literally
+    #: ``group_class = type``), this group's class will be used as the
+    #: default class. This makes a custom group class continue to make
+    #: custom groups.
+    #:
+    #: .. versionadded:: 8.0
+    group_class: type[Group] | type[type] | None = None
+    # Literal[type] isn't valid, so use Type[type]
+
+    def __init__(
+        self,
+        name: str | None = None,
+        commands: cabc.MutableMapping[str, Command]
+        | cabc.Sequence[Command]
+        | None = None,
+        invoke_without_command: bool = False,
+        no_args_is_help: bool | None = None,
+        subcommand_metavar: str | None = None,
+        chain: bool = False,
+        result_callback: t.Callable[..., t.Any] | None = None,
+        **kwargs: t.Any,
+    ) -> None:
+        super().__init__(name, **kwargs)
+
+        if commands is None:
+            commands = {}
+        elif isinstance(commands, abc.Sequence):
+            commands = {c.name: c for c in commands if c.name is not None}
+
+        #: The registered subcommands by their exported names.
+        self.commands: cabc.MutableMapping[str, Command] = commands
+
+        if no_args_is_help is None:
+            no_args_is_help = not invoke_without_command
+
+        self.no_args_is_help = no_args_is_help
+        self.invoke_without_command = invoke_without_command
+
+        if subcommand_metavar is None:
+            if chain:
+                subcommand_metavar = "COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]..."
+            else:
+                subcommand_metavar = "COMMAND [ARGS]..."
+
+        self.subcommand_metavar = subcommand_metavar
+        self.chain = chain
+        # The result callback that is stored. This can be set or
+        # overridden with the :func:`result_callback` decorator.
+        self._result_callback = result_callback
+
+        if self.chain:
+            for param in self.params:
+                if isinstance(param, Argument) and not param.required:
+                    raise RuntimeError(
+                        "A group in chain mode cannot have optional arguments."
+                    )
+
+    def to_info_dict(self, ctx: Context) -> dict[str, t.Any]:
+        info_dict = super().to_info_dict(ctx)
+        commands = {}
+
+        for name in self.list_commands(ctx):
+            command = self.get_command(ctx, name)
+
+            if command is None:
+                continue
+
+            sub_ctx = ctx._make_sub_context(command)
+
+            with sub_ctx.scope(cleanup=False):
+                commands[name] = command.to_info_dict(sub_ctx)
+
+        info_dict.update(commands=commands, chain=self.chain)
+        return info_dict
+
+    def add_command(self, cmd: Command, name: str | None = None) -> None:
+        """Registers another :class:`Command` with this group.  If the name
+        is not provided, the name of the command is used.
+        """
+        name = name or cmd.name
+        if name is None:
+            raise TypeError("Command has no name.")
+        _check_nested_chain(self, name, cmd, register=True)
+        self.commands[name] = cmd
+
+    @t.overload
+    def command(self, __func: t.Callable[..., t.Any]) -> Command: ...
+
+    @t.overload
+    def command(
+        self, *args: t.Any, **kwargs: t.Any
+    ) -> t.Callable[[t.Callable[..., t.Any]], Command]: ...
+
+    def command(
+        self, *args: t.Any, **kwargs: t.Any
+    ) -> t.Callable[[t.Callable[..., t.Any]], Command] | Command:
+        """A shortcut decorator for declaring and attaching a command to
+        the group. This takes the same arguments as :func:`command` and
+        immediately registers the created command with this group by
+        calling :meth:`add_command`.
+
+        To customize the command class used, set the
+        :attr:`command_class` attribute.
+
+        .. versionchanged:: 8.1
+            This decorator can be applied without parentheses.
+
+        .. versionchanged:: 8.0
+            Added the :attr:`command_class` attribute.
+        """
+        from .decorators import command
+
+        func: t.Callable[..., t.Any] | None = None
+
+        if args and callable(args[0]):
+            assert len(args) == 1 and not kwargs, (
+                "Use 'command(**kwargs)(callable)' to provide arguments."
+            )
+            (func,) = args
+            args = ()
+
+        if self.command_class and kwargs.get("cls") is None:
+            kwargs["cls"] = self.command_class
+
+        def decorator(f: t.Callable[..., t.Any]) -> Command:
+            cmd: Command = command(*args, **kwargs)(f)
+            self.add_command(cmd)
+            return cmd
+
+        if func is not None:
+            return decorator(func)
+
+        return decorator
+
+    @t.overload
+    def group(self, __func: t.Callable[..., t.Any]) -> Group: ...
+
+    @t.overload
+    def group(
+        self, *args: t.Any, **kwargs: t.Any
+    ) -> t.Callable[[t.Callable[..., t.Any]], Group]: ...
+
+    def group(
+        self, *args: t.Any, **kwargs: t.Any
+    ) -> t.Callable[[t.Callable[..., t.Any]], Group] | Group:
+        """A shortcut decorator for declaring and attaching a group to
+        the group. This takes the same arguments as :func:`group` and
+        immediately registers the created group with this group by
+        calling :meth:`add_command`.
+
+        To customize the group class used, set the :attr:`group_class`
+        attribute.
+
+        .. versionchanged:: 8.1
+            This decorator can be applied without parentheses.
+
+        .. versionchanged:: 8.0
+            Added the :attr:`group_class` attribute.
+        """
+        from .decorators import group
+
+        func: t.Callable[..., t.Any] | None = None
+
+        if args and callable(args[0]):
+            assert len(args) == 1 and not kwargs, (
+                "Use 'group(**kwargs)(callable)' to provide arguments."
+            )
+            (func,) = args
+            args = ()
+
+        if self.group_class is not None and kwargs.get("cls") is None:
+            if self.group_class is type:
+                kwargs["cls"] = type(self)
+            else:
+                kwargs["cls"] = self.group_class
+
+        def decorator(f: t.Callable[..., t.Any]) -> Group:
+            cmd: Group = group(*args, **kwargs)(f)
+            self.add_command(cmd)
+            return cmd
+
+        if func is not None:
+            return decorator(func)
+
+        return decorator
+
+    def result_callback(self, replace: bool = False) -> t.Callable[[F], F]:
+        """Adds a result callback to the command.  By default if a
+        result callback is already registered this will chain them but
+        this can be disabled with the `replace` parameter.  The result
+        callback is invoked with the return value of the subcommand
+        (or the list of return values from all subcommands if chaining
+        is enabled) as well as the parameters as they would be passed
+        to the main callback.
+
+        Example::
+
+            @click.group()
+            @click.option('-i', '--input', default=23)
+            def cli(input):
+                return 42
+
+            @cli.result_callback()
+            def process_result(result, input):
+                return result + input
+
+        :param replace: if set to `True` an already existing result
+                        callback will be removed.
+
+        .. versionchanged:: 8.0
+            Renamed from ``resultcallback``.
+
+        .. versionadded:: 3.0
+        """
+
+        def decorator(f: F) -> F:
+            old_callback = self._result_callback
+
+            if old_callback is None or replace:
+                self._result_callback = f
+                return f
+
+            def function(value: t.Any, /, *args: t.Any, **kwargs: t.Any) -> t.Any:
+                inner = old_callback(value, *args, **kwargs)
+                return f(inner, *args, **kwargs)
+
+            self._result_callback = rv = update_wrapper(t.cast(F, function), f)
+            return rv  # type: ignore[return-value]
+
+        return decorator
+
+    def get_command(self, ctx: Context, cmd_name: str) -> Command | None:
+        """Given a context and a command name, this returns a :class:`Command`
+        object if it exists or returns ``None``.
+        """
+        return self.commands.get(cmd_name)
+
+    def list_commands(self, ctx: Context) -> list[str]:
+        """Returns a list of subcommand names in the order they should appear."""
+        return sorted(self.commands)
+
+    def collect_usage_pieces(self, ctx: Context) -> list[str]:
+        rv = super().collect_usage_pieces(ctx)
+        rv.append(self.subcommand_metavar)
+        return rv
+
+    def format_options(self, ctx: Context, formatter: HelpFormatter) -> None:
+        super().format_options(ctx, formatter)
+        self.format_commands(ctx, formatter)
+
+    def format_commands(self, ctx: Context, formatter: HelpFormatter) -> None:
+        """Extra format methods for multi methods that adds all the commands
+        after the options.
+        """
+        commands = []
+        for subcommand in self.list_commands(ctx):
+            cmd = self.get_command(ctx, subcommand)
+            # What is this, the tool lied about a command.  Ignore it
+            if cmd is None:
+                continue
+            if cmd.hidden:
+                continue
+
+            commands.append((subcommand, cmd))
+
+        # allow for 3 times the default spacing
+        if len(commands):
+            limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands)
+
+            rows = []
+            for subcommand, cmd in commands:
+                help = cmd.get_short_help_str(limit)
+                rows.append((subcommand, help))
+
+            if rows:
+                with formatter.section(_("Commands")):
+                    formatter.write_dl(rows)
+
+    def parse_args(self, ctx: Context, args: list[str]) -> list[str]:
+        if not args and self.no_args_is_help and not ctx.resilient_parsing:
+            raise NoArgsIsHelpError(ctx)
+
+        rest = super().parse_args(ctx, args)
+
+        if self.chain:
+            ctx._protected_args = rest
+            ctx.args = []
+        elif rest:
+            ctx._protected_args, ctx.args = rest[:1], rest[1:]
+
+        return ctx.args
+
+    def invoke(self, ctx: Context) -> t.Any:
+        def _process_result(value: t.Any) -> t.Any:
+            if self._result_callback is not None:
+                value = ctx.invoke(self._result_callback, value, **ctx.params)
+            return value
+
+        if not ctx._protected_args:
+            if self.invoke_without_command:
+                # No subcommand was invoked, so the result callback is
+                # invoked with the group return value for regular
+                # groups, or an empty list for chained groups.
+                with ctx:
+                    rv = super().invoke(ctx)
+                    return _process_result([] if self.chain else rv)
+            ctx.fail(_("Missing command."))
+
+        # Fetch args back out
+        args = [*ctx._protected_args, *ctx.args]
+        ctx.args = []
+        ctx._protected_args = []
+
+        # If we're not in chain mode, we only allow the invocation of a
+        # single command but we also inform the current context about the
+        # name of the command to invoke.
+        if not self.chain:
+            # Make sure the context is entered so we do not clean up
+            # resources until the result processor has worked.
+            with ctx:
+                cmd_name, cmd, args = self.resolve_command(ctx, args)
+                assert cmd is not None
+                ctx.invoked_subcommand = cmd_name
+                super().invoke(ctx)
+                sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)
+                with sub_ctx:
+                    return _process_result(sub_ctx.command.invoke(sub_ctx))
+
+        # In chain mode we create the contexts step by step, but after the
+        # base command has been invoked.  Because at that point we do not
+        # know the subcommands yet, the invoked subcommand attribute is
+        # set to ``*`` to inform the command that subcommands are executed
+        # but nothing else.
+        with ctx:
+            ctx.invoked_subcommand = "*" if args else None
+            super().invoke(ctx)
+
+            # Otherwise we make every single context and invoke them in a
+            # chain.  In that case the return value to the result processor
+            # is the list of all invoked subcommand's results.
+            contexts = []
+            while args:
+                cmd_name, cmd, args = self.resolve_command(ctx, args)
+                assert cmd is not None
+                sub_ctx = cmd.make_context(
+                    cmd_name,
+                    args,
+                    parent=ctx,
+                    allow_extra_args=True,
+                    allow_interspersed_args=False,
+                )
+                contexts.append(sub_ctx)
+                args, sub_ctx.args = sub_ctx.args, []
+
+            rv = []
+            for sub_ctx in contexts:
+                with sub_ctx:
+                    rv.append(sub_ctx.command.invoke(sub_ctx))
+            return _process_result(rv)
+
+    def resolve_command(
+        self, ctx: Context, args: list[str]
+    ) -> tuple[str | None, Command | None, list[str]]:
+        cmd_name = make_str(args[0])
+        original_cmd_name = cmd_name
+
+        # Get the command
+        cmd = self.get_command(ctx, cmd_name)
+
+        # If we can't find the command but there is a normalization
+        # function available, we try with that one.
+        if cmd is None and ctx.token_normalize_func is not None:
+            cmd_name = ctx.token_normalize_func(cmd_name)
+            cmd = self.get_command(ctx, cmd_name)
+
+        # If we don't find the command we want to show an error message
+        # to the user that it was not provided.  However, there is
+        # something else we should do: if the first argument looks like
+        # an option we want to kick off parsing again for arguments to
+        # resolve things like --help which now should go to the main
+        # place.
+        if cmd is None and not ctx.resilient_parsing:
+            if _split_opt(cmd_name)[0]:
+                self.parse_args(ctx, args)
+            ctx.fail(_("No such command {name!r}.").format(name=original_cmd_name))
+        return cmd_name if cmd else None, cmd, args[1:]
+
+    def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]:
+        """Return a list of completions for the incomplete value. Looks
+        at the names of options, subcommands, and chained
+        multi-commands.
+
+        :param ctx: Invocation context for this command.
+        :param incomplete: Value being completed. May be empty.
+
+        .. versionadded:: 8.0
+        """
+        from click.shell_completion import CompletionItem
+
+        results = [
+            CompletionItem(name, help=command.get_short_help_str())
+            for name, command in _complete_visible_commands(ctx, incomplete)
+        ]
+        results.extend(super().shell_complete(ctx, incomplete))
+        return results
+
+
+class _MultiCommand(Group, metaclass=_FakeSubclassCheck):
+    """
+    .. deprecated:: 8.2
+        Will be removed in Click 9.0. Use ``Group`` instead.
+    """
+
+
+class CommandCollection(Group):
+    """A :class:`Group` that looks up subcommands on other groups. If a command
+    is not found on this group, each registered source is checked in order.
+    Parameters on a source are not added to this group, and a source's callback
+    is not invoked when invoking its commands. In other words, this "flattens"
+    commands in many groups into this one group.
+
+    :param name: The name of the group command.
+    :param sources: A list of :class:`Group` objects to look up commands from.
+    :param kwargs: Other arguments passed to :class:`Group`.
+
+    .. versionchanged:: 8.2
+        This is a subclass of ``Group``. Commands are looked up first on this
+        group, then each of its sources.
+    """
+
+    def __init__(
+        self,
+        name: str | None = None,
+        sources: list[Group] | None = None,
+        **kwargs: t.Any,
+    ) -> None:
+        super().__init__(name, **kwargs)
+        #: The list of registered groups.
+        self.sources: list[Group] = sources or []
+
+    def add_source(self, group: Group) -> None:
+        """Add a group as a source of commands."""
+        self.sources.append(group)
+
+    def get_command(self, ctx: Context, cmd_name: str) -> Command | None:
+        rv = super().get_command(ctx, cmd_name)
+
+        if rv is not None:
+            return rv
+
+        for source in self.sources:
+            rv = source.get_command(ctx, cmd_name)
+
+            if rv is not None:
+                if self.chain:
+                    _check_nested_chain(self, cmd_name, rv)
+
+                return rv
+
+        return None
+
+    def list_commands(self, ctx: Context) -> list[str]:
+        rv: set[str] = set(super().list_commands(ctx))
+
+        for source in self.sources:
+            rv.update(source.list_commands(ctx))
+
+        return sorted(rv)
+
+
+def _check_iter(value: t.Any) -> cabc.Iterator[t.Any]:
+    """Check if the value is iterable but not a string. Raises a type
+    error, or return an iterator over the value.
+    """
+    if isinstance(value, str):
+        raise TypeError
+
+    return iter(value)
+
+
+class Parameter:
+    r"""A parameter to a command comes in two versions: they are either
+    :class:`Option`\s or :class:`Argument`\s.  Other subclasses are currently
+    not supported by design as some of the internals for parsing are
+    intentionally not finalized.
+
+    Some settings are supported by both options and arguments.
+
+    :param param_decls: the parameter declarations for this option or
+                        argument.  This is a list of flags or argument
+                        names.
+    :param type: the type that should be used.  Either a :class:`ParamType`
+                 or a Python type.  The latter is converted into the former
+                 automatically if supported.
+    :param required: controls if this is optional or not.
+    :param default: the default value if omitted.  This can also be a callable,
+                    in which case it's invoked when the default is needed
+                    without any arguments.
+    :param callback: A function to further process or validate the value
+        after type conversion. It is called as ``f(ctx, param, value)``
+        and must return the value. It is called for all sources,
+        including prompts.
+    :param nargs: the number of arguments to match.  If not ``1`` the return
+                  value is a tuple instead of single value.  The default for
+                  nargs is ``1`` (except if the type is a tuple, then it's
+                  the arity of the tuple). If ``nargs=-1``, all remaining
+                  parameters are collected.
+    :param metavar: how the value is represented in the help page.
+    :param expose_value: if this is `True` then the value is passed onwards
+                         to the command callback and stored on the context,
+                         otherwise it's skipped.
+    :param is_eager: eager values are processed before non eager ones.  This
+                     should not be set for arguments or it will inverse the
+                     order of processing.
+    :param envvar: environment variable(s) that are used to provide a default value for
+        this parameter. This can be a string or a sequence of strings. If a sequence is
+        given, only the first non-empty environment variable is used for the parameter.
+    :param shell_complete: A function that returns custom shell
+        completions. Used instead of the param's type completion if
+        given. Takes ``ctx, param, incomplete`` and must return a list
+        of :class:`~click.shell_completion.CompletionItem` or a list of
+        strings.
+    :param deprecated: If ``True`` or non-empty string, issues a message
+                        indicating that the argument is deprecated and highlights
+                        its deprecation in --help. The message can be customized
+                        by using a string as the value. A deprecated parameter
+                        cannot be required, a ValueError will be raised otherwise.
+
+    .. versionchanged:: 8.2.0
+        Introduction of ``deprecated``.
+
+    .. versionchanged:: 8.2
+        Adding duplicate parameter names to a :class:`~click.core.Command` will
+        result in a ``UserWarning`` being shown.
+
+    .. versionchanged:: 8.2
+        Adding duplicate parameter names to a :class:`~click.core.Command` will
+        result in a ``UserWarning`` being shown.
+
+    .. versionchanged:: 8.0
+        ``process_value`` validates required parameters and bounded
+        ``nargs``, and invokes the parameter callback before returning
+        the value. This allows the callback to validate prompts.
+        ``full_process_value`` is removed.
+
+    .. versionchanged:: 8.0
+        ``autocompletion`` is renamed to ``shell_complete`` and has new
+        semantics described above. The old name is deprecated and will
+        be removed in 8.1, until then it will be wrapped to match the
+        new requirements.
+
+    .. versionchanged:: 8.0
+        For ``multiple=True, nargs>1``, the default must be a list of
+        tuples.
+
+    .. versionchanged:: 8.0
+        Setting a default is no longer required for ``nargs>1``, it will
+        default to ``None``. ``multiple=True`` or ``nargs=-1`` will
+        default to ``()``.
+
+    .. versionchanged:: 7.1
+        Empty environment variables are ignored rather than taking the
+        empty string value. This makes it possible for scripts to clear
+        variables if they can't unset them.
+
+    .. versionchanged:: 2.0
+        Changed signature for parameter callback to also be passed the
+        parameter. The old callback format will still work, but it will
+        raise a warning to give you a chance to migrate the code easier.
+    """
+
+    param_type_name = "parameter"
+
+    def __init__(
+        self,
+        param_decls: cabc.Sequence[str] | None = None,
+        type: types.ParamType | t.Any | None = None,
+        required: bool = False,
+        # XXX The default historically embed two concepts:
+        # - the declaration of a Parameter object carrying the default (handy to
+        #   arbitrage the default value of coupled Parameters sharing the same
+        #   self.name, like flag options),
+        # - and the actual value of the default.
+        # It is confusing and is the source of many issues discussed in:
+        # https://github.com/pallets/click/pull/3030
+        # In the future, we might think of splitting it in two, not unlike
+        # Option.is_flag and Option.flag_value: we could have something like
+        # Parameter.is_default and Parameter.default_value.
+        default: t.Any | t.Callable[[], t.Any] | None = UNSET,
+        callback: t.Callable[[Context, Parameter, t.Any], t.Any] | None = None,
+        nargs: int | None = None,
+        multiple: bool = False,
+        metavar: str | None = None,
+        expose_value: bool = True,
+        is_eager: bool = False,
+        envvar: str | cabc.Sequence[str] | None = None,
+        shell_complete: t.Callable[
+            [Context, Parameter, str], list[CompletionItem] | list[str]
+        ]
+        | None = None,
+        deprecated: bool | str = False,
+    ) -> None:
+        self.name: str | None
+        self.opts: list[str]
+        self.secondary_opts: list[str]
+        self.name, self.opts, self.secondary_opts = self._parse_decls(
+            param_decls or (), expose_value
+        )
+        self.type: types.ParamType = types.convert_type(type, default)
+
+        # Default nargs to what the type tells us if we have that
+        # information available.
+        if nargs is None:
+            if self.type.is_composite:
+                nargs = self.type.arity
+            else:
+                nargs = 1
+
+        self.required = required
+        self.callback = callback
+        self.nargs = nargs
+        self.multiple = multiple
+        self.expose_value = expose_value
+        self.default: t.Any | t.Callable[[], t.Any] | None = default
+        self.is_eager = is_eager
+        self.metavar = metavar
+        self.envvar = envvar
+        self._custom_shell_complete = shell_complete
+        self.deprecated = deprecated
+
+        if __debug__:
+            if self.type.is_composite and nargs != self.type.arity:
+                raise ValueError(
+                    f"'nargs' must be {self.type.arity} (or None) for"
+                    f" type {self.type!r}, but it was {nargs}."
+                )
+
+            if required and deprecated:
+                raise ValueError(
+                    f"The {self.param_type_name} '{self.human_readable_name}' "
+                    "is deprecated and still required. A deprecated "
+                    f"{self.param_type_name} cannot be required."
+                )
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        """Gather information that could be useful for a tool generating
+        user-facing documentation.
+
+        Use :meth:`click.Context.to_info_dict` to traverse the entire
+        CLI structure.
+
+        .. versionchanged:: 8.3.0
+            Returns ``None`` for the :attr:`default` if it was not set.
+
+        .. versionadded:: 8.0
+        """
+        return {
+            "name": self.name,
+            "param_type_name": self.param_type_name,
+            "opts": self.opts,
+            "secondary_opts": self.secondary_opts,
+            "type": self.type.to_info_dict(),
+            "required": self.required,
+            "nargs": self.nargs,
+            "multiple": self.multiple,
+            # We explicitly hide the :attr:`UNSET` value to the user, as we choose to
+            # make it an implementation detail. And because ``to_info_dict`` has been
+            # designed for documentation purposes, we return ``None`` instead.
+            "default": self.default if self.default is not UNSET else None,
+            "envvar": self.envvar,
+        }
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.name}>"
+
+    def _parse_decls(
+        self, decls: cabc.Sequence[str], expose_value: bool
+    ) -> tuple[str | None, list[str], list[str]]:
+        raise NotImplementedError()
+
+    @property
+    def human_readable_name(self) -> str:
+        """Returns the human readable name of this parameter.  This is the
+        same as the name for options, but the metavar for arguments.
+        """
+        return self.name  # type: ignore
+
+    def make_metavar(self, ctx: Context) -> str:
+        if self.metavar is not None:
+            return self.metavar
+
+        metavar = self.type.get_metavar(param=self, ctx=ctx)
+
+        if metavar is None:
+            metavar = self.type.name.upper()
+
+        if self.nargs != 1:
+            metavar += "..."
+
+        return metavar
+
+    @t.overload
+    def get_default(
+        self, ctx: Context, call: t.Literal[True] = True
+    ) -> t.Any | None: ...
+
+    @t.overload
+    def get_default(
+        self, ctx: Context, call: bool = ...
+    ) -> t.Any | t.Callable[[], t.Any] | None: ...
+
+    def get_default(
+        self, ctx: Context, call: bool = True
+    ) -> t.Any | t.Callable[[], t.Any] | None:
+        """Get the default for the parameter. Tries
+        :meth:`Context.lookup_default` first, then the local default.
+
+        :param ctx: Current context.
+        :param call: If the default is a callable, call it. Disable to
+            return the callable instead.
+
+        .. versionchanged:: 8.0.2
+            Type casting is no longer performed when getting a default.
+
+        .. versionchanged:: 8.0.1
+            Type casting can fail in resilient parsing mode. Invalid
+            defaults will not prevent showing help text.
+
+        .. versionchanged:: 8.0
+            Looks at ``ctx.default_map`` first.
+
+        .. versionchanged:: 8.0
+            Added the ``call`` parameter.
+        """
+        name = self.name
+        value = ctx.lookup_default(name, call=False) if name is not None else None
+
+        if value is None and not (
+            ctx.default_map is not None and name is not None and name in ctx.default_map
+        ):
+            value = self.default
+
+        if call and callable(value):
+            value = value()
+
+        return value
+
+    def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None:
+        raise NotImplementedError()
+
+    def consume_value(
+        self, ctx: Context, opts: cabc.Mapping[str, t.Any]
+    ) -> tuple[t.Any, ParameterSource]:
+        """Returns the parameter value produced by the parser.
+
+        If the parser did not produce a value from user input, the value is either
+        sourced from the environment variable, the default map, or the parameter's
+        default value. In that order of precedence.
+
+        If no value is found, an internal sentinel value is returned.
+
+        :meta private:
+        """
+        # Collect from the parse the value passed by the user to the CLI.
+        value = opts.get(self.name, UNSET)  # type: ignore
+        # If the value is set, it means it was sourced from the command line by the
+        # parser, otherwise it left unset by default.
+        source = (
+            ParameterSource.COMMANDLINE
+            if value is not UNSET
+            else ParameterSource.DEFAULT
+        )
+
+        if value is UNSET:
+            envvar_value = self.value_from_envvar(ctx)
+            if envvar_value is not None:
+                value = envvar_value
+                source = ParameterSource.ENVIRONMENT
+
+        if value is UNSET:
+            default_map_value = ctx.lookup_default(self.name)  # type: ignore[arg-type]
+            if default_map_value is not None or (
+                ctx.default_map is not None and self.name in ctx.default_map
+            ):
+                value = default_map_value
+                source = ParameterSource.DEFAULT_MAP
+
+        if value is UNSET:
+            default_value = self.get_default(ctx)
+            if default_value is not UNSET:
+                value = default_value
+                source = ParameterSource.DEFAULT
+
+        return value, source
+
+    def type_cast_value(self, ctx: Context, value: t.Any) -> t.Any:
+        """Convert and validate a value against the parameter's
+        :attr:`type`, :attr:`multiple`, and :attr:`nargs`.
+        """
+        if value is None:
+            if self.multiple or self.nargs == -1:
+                return ()
+            else:
+                return value
+
+        def check_iter(value: t.Any) -> cabc.Iterator[t.Any]:
+            try:
+                return _check_iter(value)
+            except TypeError:
+                # This should only happen when passing in args manually,
+                # the parser should construct an iterable when parsing
+                # the command line.
+                raise BadParameter(
+                    _("Value must be an iterable."), ctx=ctx, param=self
+                ) from None
+
+        # Define the conversion function based on nargs and type.
+
+        if self.nargs == 1 or self.type.is_composite:
+
+            def convert(value: t.Any) -> t.Any:
+                return self.type(value, param=self, ctx=ctx)
+
+        elif self.nargs == -1:
+
+            def convert(value: t.Any) -> t.Any:  # tuple[t.Any, ...]
+                return tuple(self.type(x, self, ctx) for x in check_iter(value))
+
+        else:  # nargs > 1
+
+            def convert(value: t.Any) -> t.Any:  # tuple[t.Any, ...]
+                value = tuple(check_iter(value))
+
+                if len(value) != self.nargs:
+                    raise BadParameter(
+                        ngettext(
+                            "Takes {nargs} values but 1 was given.",
+                            "Takes {nargs} values but {len} were given.",
+                            len(value),
+                        ).format(nargs=self.nargs, len=len(value)),
+                        ctx=ctx,
+                        param=self,
+                    )
+
+                return tuple(self.type(x, self, ctx) for x in value)
+
+        if self.multiple:
+            return tuple(convert(x) for x in check_iter(value))
+
+        return convert(value)
+
+    def value_is_missing(self, value: t.Any) -> bool:
+        """A value is considered missing if:
+
+        - it is :attr:`UNSET`,
+        - or if it is an empty sequence while the parameter is suppose to have
+          non-single value (i.e. :attr:`nargs` is not ``1`` or :attr:`multiple` is
+          set).
+
+        :meta private:
+        """
+        if value is UNSET:
+            return True
+
+        if (self.nargs != 1 or self.multiple) and value == ():
+            return True
+
+        return False
+
+    def process_value(self, ctx: Context, value: t.Any) -> t.Any:
+        """Process the value of this parameter:
+
+        1. Type cast the value using :meth:`type_cast_value`.
+        2. Check if the value is missing (see: :meth:`value_is_missing`), and raise
+           :exc:`MissingParameter` if it is required.
+        3. If a :attr:`callback` is set, call it to have the value replaced by the
+           result of the callback. If the value was not set, the callback receive
+           ``None``. This keep the legacy behavior as it was before the introduction of
+           the :attr:`UNSET` sentinel.
+
+        :meta private:
+        """
+        # shelter `type_cast_value` from ever seeing an `UNSET` value by handling the
+        # cases in which `UNSET` gets special treatment explicitly at this layer
+        #
+        # Refs:
+        # https://github.com/pallets/click/issues/3069
+        if value is UNSET:
+            if self.multiple or self.nargs == -1:
+                value = ()
+        else:
+            value = self.type_cast_value(ctx, value)
+
+        if self.required and self.value_is_missing(value):
+            raise MissingParameter(ctx=ctx, param=self)
+
+        if self.callback is not None:
+            # Legacy case: UNSET is not exposed directly to the callback, but converted
+            # to None.
+            if value is UNSET:
+                value = None
+
+            # Search for parameters with UNSET values in the context.
+            unset_keys = {k: None for k, v in ctx.params.items() if v is UNSET}
+            # No UNSET values, call the callback as usual.
+            if not unset_keys:
+                value = self.callback(ctx, self, value)
+
+            # Legacy case: provide a temporarily manipulated context to the callback
+            # to hide UNSET values as None.
+            #
+            # Refs:
+            # https://github.com/pallets/click/issues/3136
+            # https://github.com/pallets/click/pull/3137
+            else:
+                # Add another layer to the context stack to clearly hint that the
+                # context is temporarily modified.
+                with ctx:
+                    # Update the context parameters to replace UNSET with None.
+                    ctx.params.update(unset_keys)
+                    # Feed these fake context parameters to the callback.
+                    value = self.callback(ctx, self, value)
+                    # Restore the UNSET values in the context parameters.
+                    ctx.params.update(
+                        {
+                            k: UNSET
+                            for k in unset_keys
+                            # Only restore keys that are present and still None, in case
+                            # the callback modified other parameters.
+                            if k in ctx.params and ctx.params[k] is None
+                        }
+                    )
+
+        return value
+
+    def resolve_envvar_value(self, ctx: Context) -> str | None:
+        """Returns the value found in the environment variable(s) attached to this
+        parameter.
+
+        Environment variables values are `always returned as strings
+        <https://docs.python.org/3/library/os.html#os.environ>`_.
+
+        This method returns ``None`` if:
+
+        - the :attr:`envvar` property is not set on the :class:`Parameter`,
+        - the environment variable is not found in the environment,
+        - the variable is found in the environment but its value is empty (i.e. the
+          environment variable is present but has an empty string).
+
+        If :attr:`envvar` is setup with multiple environment variables,
+        then only the first non-empty value is returned.
+
+        .. caution::
+
+            The raw value extracted from the environment is not normalized and is
+            returned as-is. Any normalization or reconciliation is performed later by
+            the :class:`Parameter`'s :attr:`type`.
+
+        :meta private:
+        """
+        if not self.envvar:
+            return None
+
+        if isinstance(self.envvar, str):
+            rv = os.environ.get(self.envvar)
+
+            if rv:
+                return rv
+        else:
+            for envvar in self.envvar:
+                rv = os.environ.get(envvar)
+
+                # Return the first non-empty value of the list of environment variables.
+                if rv:
+                    return rv
+                # Else, absence of value is interpreted as an environment variable that
+                # is not set, so proceed to the next one.
+
+        return None
+
+    def value_from_envvar(self, ctx: Context) -> str | cabc.Sequence[str] | None:
+        """Process the raw environment variable string for this parameter.
+
+        Returns the string as-is or splits it into a sequence of strings if the
+        parameter is expecting multiple values (i.e. its :attr:`nargs` property is set
+        to a value other than ``1``).
+
+        :meta private:
+        """
+        rv = self.resolve_envvar_value(ctx)
+
+        if rv is not None and self.nargs != 1:
+            return self.type.split_envvar_value(rv)
+
+        return rv
+
+    def handle_parse_result(
+        self, ctx: Context, opts: cabc.Mapping[str, t.Any], args: list[str]
+    ) -> tuple[t.Any, list[str]]:
+        """Process the value produced by the parser from user input.
+
+        Always process the value through the Parameter's :attr:`type`, wherever it
+        comes from.
+
+        If the parameter is deprecated, this method warn the user about it. But only if
+        the value has been explicitly set by the user (and as such, is not coming from
+        a default).
+
+        :meta private:
+        """
+        with augment_usage_errors(ctx, param=self):
+            value, source = self.consume_value(ctx, opts)
+
+            ctx.set_parameter_source(self.name, source)  # type: ignore
+
+            # Display a deprecation warning if necessary.
+            if (
+                self.deprecated
+                and value is not UNSET
+                and source not in (ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP)
+            ):
+                extra_message = (
+                    f" {self.deprecated}" if isinstance(self.deprecated, str) else ""
+                )
+                message = _(
+                    "DeprecationWarning: The {param_type} {name!r} is deprecated."
+                    "{extra_message}"
+                ).format(
+                    param_type=self.param_type_name,
+                    name=self.human_readable_name,
+                    extra_message=extra_message,
+                )
+                echo(style(message, fg="red"), err=True)
+
+            # Process the value through the parameter's type.
+            try:
+                value = self.process_value(ctx, value)
+            except Exception:
+                if not ctx.resilient_parsing:
+                    raise
+                # In resilient parsing mode, we do not want to fail the command if the
+                # value is incompatible with the parameter type, so we reset the value
+                # to UNSET, which will be interpreted as a missing value.
+                value = UNSET
+
+        # Add parameter's value to the context.
+        if (
+            self.expose_value
+            # We skip adding the value if it was previously set by another parameter
+            # targeting the same variable name. This prevents parameters competing for
+            # the same name to override each other.
+            and (self.name not in ctx.params or ctx.params[self.name] is UNSET)
+        ):
+            # Click is logically enforcing that the name is None if the parameter is
+            # not to be exposed. We still assert it here to please the type checker.
+            assert self.name is not None, (
+                f"{self!r} parameter's name should not be None when exposing value."
+            )
+            ctx.params[self.name] = value
+
+        return value, args
+
+    def get_help_record(self, ctx: Context) -> tuple[str, str] | None:
+        pass
+
+    def get_usage_pieces(self, ctx: Context) -> list[str]:
+        return []
+
+    def get_error_hint(self, ctx: Context) -> str:
+        """Get a stringified version of the param for use in error messages to
+        indicate which param caused the error.
+        """
+        hint_list = self.opts or [self.human_readable_name]
+        return " / ".join(f"'{x}'" for x in hint_list)
+
+    def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]:
+        """Return a list of completions for the incomplete value. If a
+        ``shell_complete`` function was given during init, it is used.
+        Otherwise, the :attr:`type`
+        :meth:`~click.types.ParamType.shell_complete` function is used.
+
+        :param ctx: Invocation context for this command.
+        :param incomplete: Value being completed. May be empty.
+
+        .. versionadded:: 8.0
+        """
+        if self._custom_shell_complete is not None:
+            results = self._custom_shell_complete(ctx, self, incomplete)
+
+            if results and isinstance(results[0], str):
+                from click.shell_completion import CompletionItem
+
+                results = [CompletionItem(c) for c in results]
+
+            return t.cast("list[CompletionItem]", results)
+
+        return self.type.shell_complete(ctx, self, incomplete)
+
+
+class Option(Parameter):
+    """Options are usually optional values on the command line and
+    have some extra features that arguments don't have.
+
+    All other parameters are passed onwards to the parameter constructor.
+
+    :param show_default: Show the default value for this option in its
+        help text. Values are not shown by default, unless
+        :attr:`Context.show_default` is ``True``. If this value is a
+        string, it shows that string in parentheses instead of the
+        actual value. This is particularly useful for dynamic options.
+        For single option boolean flags, the default remains hidden if
+        its value is ``False``.
+    :param show_envvar: Controls if an environment variable should be
+        shown on the help page and error messages.
+        Normally, environment variables are not shown.
+    :param prompt: If set to ``True`` or a non empty string then the
+        user will be prompted for input. If set to ``True`` the prompt
+        will be the option name capitalized. A deprecated option cannot be
+        prompted.
+    :param confirmation_prompt: Prompt a second time to confirm the
+        value if it was prompted for. Can be set to a string instead of
+        ``True`` to customize the message.
+    :param prompt_required: If set to ``False``, the user will be
+        prompted for input only when the option was specified as a flag
+        without a value.
+    :param hide_input: If this is ``True`` then the input on the prompt
+        will be hidden from the user. This is useful for password input.
+    :param is_flag: forces this option to act as a flag.  The default is
+                    auto detection.
+    :param flag_value: which value should be used for this flag if it's
+                       enabled.  This is set to a boolean automatically if
+                       the option string contains a slash to mark two options.
+    :param multiple: if this is set to `True` then the argument is accepted
+                     multiple times and recorded.  This is similar to ``nargs``
+                     in how it works but supports arbitrary number of
+                     arguments.
+    :param count: this flag makes an option increment an integer.
+    :param allow_from_autoenv: if this is enabled then the value of this
+                               parameter will be pulled from an environment
+                               variable in case a prefix is defined on the
+                               context.
+    :param help: the help string.
+    :param hidden: hide this option from help outputs.
+    :param attrs: Other command arguments described in :class:`Parameter`.
+
+    .. versionchanged:: 8.2
+        ``envvar`` used with ``flag_value`` will always use the ``flag_value``,
+        previously it would use the value of the environment variable.
+
+    .. versionchanged:: 8.1
+        Help text indentation is cleaned here instead of only in the
+        ``@option`` decorator.
+
+    .. versionchanged:: 8.1
+        The ``show_default`` parameter overrides
+        ``Context.show_default``.
+
+    .. versionchanged:: 8.1
+        The default of a single option boolean flag is not shown if the
+        default value is ``False``.
+
+    .. versionchanged:: 8.0.1
+        ``type`` is detected from ``flag_value`` if given.
+    """
+
+    param_type_name = "option"
+
+    def __init__(
+        self,
+        param_decls: cabc.Sequence[str] | None = None,
+        show_default: bool | str | None = None,
+        prompt: bool | str = False,
+        confirmation_prompt: bool | str = False,
+        prompt_required: bool = True,
+        hide_input: bool = False,
+        is_flag: bool | None = None,
+        flag_value: t.Any = UNSET,
+        multiple: bool = False,
+        count: bool = False,
+        allow_from_autoenv: bool = True,
+        type: types.ParamType | t.Any | None = None,
+        help: str | None = None,
+        hidden: bool = False,
+        show_choices: bool = True,
+        show_envvar: bool = False,
+        deprecated: bool | str = False,
+        **attrs: t.Any,
+    ) -> None:
+        if help:
+            help = inspect.cleandoc(help)
+
+        super().__init__(
+            param_decls, type=type, multiple=multiple, deprecated=deprecated, **attrs
+        )
+
+        if prompt is True:
+            if self.name is None:
+                raise TypeError("'name' is required with 'prompt=True'.")
+
+            prompt_text: str | None = self.name.replace("_", " ").capitalize()
+        elif prompt is False:
+            prompt_text = None
+        else:
+            prompt_text = prompt
+
+        if deprecated:
+            deprecated_message = (
+                f"(DEPRECATED: {deprecated})"
+                if isinstance(deprecated, str)
+                else "(DEPRECATED)"
+            )
+            help = help + deprecated_message if help is not None else deprecated_message
+
+        self.prompt = prompt_text
+        self.confirmation_prompt = confirmation_prompt
+        self.prompt_required = prompt_required
+        self.hide_input = hide_input
+        self.hidden = hidden
+
+        # The _flag_needs_value property tells the parser that this option is a flag
+        # that cannot be used standalone and needs a value. With this information, the
+        # parser can determine whether to consider the next user-provided argument in
+        # the CLI as a value for this flag or as a new option.
+        # If prompt is enabled but not required, then it opens the possibility for the
+        # option to gets its value from the user.
+        self._flag_needs_value = self.prompt is not None and not self.prompt_required
+
+        # Auto-detect if this is a flag or not.
+        if is_flag is None:
+            # Implicitly a flag because flag_value was set.
+            if flag_value is not UNSET:
+                is_flag = True
+            # Not a flag, but when used as a flag it shows a prompt.
+            elif self._flag_needs_value:
+                is_flag = False
+            # Implicitly a flag because secondary options names were given.
+            elif self.secondary_opts:
+                is_flag = True
+
+        # The option is explicitly not a flag, but to determine whether or not it needs
+        # value, we need to check if `flag_value` or `default` was set. Either one is
+        # sufficient.
+        # Ref: https://github.com/pallets/click/issues/3084
+        elif is_flag is False and not self._flag_needs_value:
+            self._flag_needs_value = flag_value is not UNSET or self.default is UNSET
+
+        if is_flag:
+            # Set missing default for flags if not explicitly required or prompted.
+            if self.default is UNSET and not self.required and not self.prompt:
+                if multiple:
+                    self.default = ()
+
+            # Auto-detect the type of the flag based on the flag_value.
+            if type is None:
+                # A flag without a flag_value is a boolean flag.
+                if flag_value is UNSET:
+                    self.type: types.ParamType = types.BoolParamType()
+                # If the flag value is a boolean, use BoolParamType.
+                elif isinstance(flag_value, bool):
+                    self.type = types.BoolParamType()
+                # Otherwise, guess the type from the flag value.
+                else:
+                    self.type = types.convert_type(None, flag_value)
+
+        self.is_flag: bool = bool(is_flag)
+        self.is_bool_flag: bool = bool(
+            is_flag and isinstance(self.type, types.BoolParamType)
+        )
+        self.flag_value: t.Any = flag_value
+
+        # Set boolean flag default to False if unset and not required.
+        if self.is_bool_flag:
+            if self.default is UNSET and not self.required:
+                self.default = False
+
+        # The alignement of default to the flag_value is resolved lazily in
+        # get_default() to prevent callable flag_values (like classes) from
+        # being instantiated. Refs:
+        # https://github.com/pallets/click/issues/3121
+        # https://github.com/pallets/click/issues/3024#issuecomment-3146199461
+        # https://github.com/pallets/click/pull/3030/commits/06847da
+
+        # Set the default flag_value if it is not set.
+        if self.flag_value is UNSET:
+            if self.is_flag:
+                self.flag_value = True
+            else:
+                self.flag_value = None
+
+        # Counting.
+        self.count = count
+        if count:
+            if type is None:
+                self.type = types.IntRange(min=0)
+            if self.default is UNSET:
+                self.default = 0
+
+        self.allow_from_autoenv = allow_from_autoenv
+        self.help = help
+        self.show_default = show_default
+        self.show_choices = show_choices
+        self.show_envvar = show_envvar
+
+        if __debug__:
+            if deprecated and prompt:
+                raise ValueError("`deprecated` options cannot use `prompt`.")
+
+            if self.nargs == -1:
+                raise TypeError("nargs=-1 is not supported for options.")
+
+            if not self.is_bool_flag and self.secondary_opts:
+                raise TypeError("Secondary flag is not valid for non-boolean flag.")
+
+            if self.is_bool_flag and self.hide_input and self.prompt is not None:
+                raise TypeError(
+                    "'prompt' with 'hide_input' is not valid for boolean flag."
+                )
+
+            if self.count:
+                if self.multiple:
+                    raise TypeError("'count' is not valid with 'multiple'.")
+
+                if self.is_flag:
+                    raise TypeError("'count' is not valid with 'is_flag'.")
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        """
+        .. versionchanged:: 8.3.0
+            Returns ``None`` for the :attr:`flag_value` if it was not set.
+        """
+        info_dict = super().to_info_dict()
+        info_dict.update(
+            help=self.help,
+            prompt=self.prompt,
+            is_flag=self.is_flag,
+            # We explicitly hide the :attr:`UNSET` value to the user, as we choose to
+            # make it an implementation detail. And because ``to_info_dict`` has been
+            # designed for documentation purposes, we return ``None`` instead.
+            flag_value=self.flag_value if self.flag_value is not UNSET else None,
+            count=self.count,
+            hidden=self.hidden,
+        )
+        return info_dict
+
+    def get_default(
+        self, ctx: Context, call: bool = True
+    ) -> t.Any | t.Callable[[], t.Any] | None:
+        value = super().get_default(ctx, call=False)
+
+        # Lazily resolve default=True to flag_value. Doing this here
+        # (instead of eagerly in __init__) prevents callable flag_values
+        # (like classes) from being instantiated by the callable check below.
+        # https://github.com/pallets/click/issues/3121
+        if value is True and self.is_flag:
+            value = self.flag_value
+        elif call and callable(value):
+            value = value()
+
+        return value
+
+    def get_error_hint(self, ctx: Context) -> str:
+        result = super().get_error_hint(ctx)
+        if self.show_envvar and self.envvar is not None:
+            result += f" (env var: '{self.envvar}')"
+        return result
+
+    def _parse_decls(
+        self, decls: cabc.Sequence[str], expose_value: bool
+    ) -> tuple[str | None, list[str], list[str]]:
+        opts = []
+        secondary_opts = []
+        name = None
+        possible_names = []
+
+        for decl in decls:
+            if decl.isidentifier():
+                if name is not None:
+                    raise TypeError(f"Name '{name}' defined twice")
+                name = decl
+            else:
+                split_char = ";" if decl[:1] == "/" else "/"
+                if split_char in decl:
+                    first, second = decl.split(split_char, 1)
+                    first = first.rstrip()
+                    if first:
+                        possible_names.append(_split_opt(first))
+                        opts.append(first)
+                    second = second.lstrip()
+                    if second:
+                        secondary_opts.append(second.lstrip())
+                    if first == second:
+                        raise ValueError(
+                            f"Boolean option {decl!r} cannot use the"
+                            " same flag for true/false."
+                        )
+                else:
+                    possible_names.append(_split_opt(decl))
+                    opts.append(decl)
+
+        if name is None and possible_names:
+            possible_names.sort(key=lambda x: -len(x[0]))  # group long options first
+            name = possible_names[0][1].replace("-", "_").lower()
+            if not name.isidentifier():
+                name = None
+
+        if name is None:
+            if not expose_value:
+                return None, opts, secondary_opts
+            raise TypeError(
+                f"Could not determine name for option with declarations {decls!r}"
+            )
+
+        if not opts and not secondary_opts:
+            raise TypeError(
+                f"No options defined but a name was passed ({name})."
+                " Did you mean to declare an argument instead? Did"
+                f" you mean to pass '--{name}'?"
+            )
+
+        return name, opts, secondary_opts
+
+    def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None:
+        if self.multiple:
+            action = "append"
+        elif self.count:
+            action = "count"
+        else:
+            action = "store"
+
+        if self.is_flag:
+            action = f"{action}_const"
+
+            if self.is_bool_flag and self.secondary_opts:
+                parser.add_option(
+                    obj=self, opts=self.opts, dest=self.name, action=action, const=True
+                )
+                parser.add_option(
+                    obj=self,
+                    opts=self.secondary_opts,
+                    dest=self.name,
+                    action=action,
+                    const=False,
+                )
+            else:
+                parser.add_option(
+                    obj=self,
+                    opts=self.opts,
+                    dest=self.name,
+                    action=action,
+                    const=self.flag_value,
+                )
+        else:
+            parser.add_option(
+                obj=self,
+                opts=self.opts,
+                dest=self.name,
+                action=action,
+                nargs=self.nargs,
+            )
+
+    def get_help_record(self, ctx: Context) -> tuple[str, str] | None:
+        if self.hidden:
+            return None
+
+        any_prefix_is_slash = False
+
+        def _write_opts(opts: cabc.Sequence[str]) -> str:
+            nonlocal any_prefix_is_slash
+
+            rv, any_slashes = join_options(opts)
+
+            if any_slashes:
+                any_prefix_is_slash = True
+
+            if not self.is_flag and not self.count:
+                rv += f" {self.make_metavar(ctx=ctx)}"
+
+            return rv
+
+        rv = [_write_opts(self.opts)]
+
+        if self.secondary_opts:
+            rv.append(_write_opts(self.secondary_opts))
+
+        help = self.help or ""
+
+        extra = self.get_help_extra(ctx)
+        extra_items = []
+        if "envvars" in extra:
+            extra_items.append(
+                _("env var: {var}").format(var=", ".join(extra["envvars"]))
+            )
+        if "default" in extra:
+            extra_items.append(_("default: {default}").format(default=extra["default"]))
+        if "range" in extra:
+            extra_items.append(extra["range"])
+        if "required" in extra:
+            extra_items.append(_(extra["required"]))
+
+        if extra_items:
+            extra_str = "; ".join(extra_items)
+            help = f"{help}  [{extra_str}]" if help else f"[{extra_str}]"
+
+        return ("; " if any_prefix_is_slash else " / ").join(rv), help
+
+    def get_help_extra(self, ctx: Context) -> types.OptionHelpExtra:
+        extra: types.OptionHelpExtra = {}
+
+        if self.show_envvar:
+            envvar = self.envvar
+
+            if envvar is None:
+                if (
+                    self.allow_from_autoenv
+                    and ctx.auto_envvar_prefix is not None
+                    and self.name is not None
+                ):
+                    envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}"
+
+            if envvar is not None:
+                if isinstance(envvar, str):
+                    extra["envvars"] = (envvar,)
+                else:
+                    extra["envvars"] = tuple(str(d) for d in envvar)
+
+        # Temporarily enable resilient parsing to avoid type casting
+        # failing for the default. Might be possible to extend this to
+        # help formatting in general.
+        resilient = ctx.resilient_parsing
+        ctx.resilient_parsing = True
+
+        try:
+            default_value = self.get_default(ctx, call=False)
+        finally:
+            ctx.resilient_parsing = resilient
+
+        show_default = False
+        show_default_is_str = False
+
+        if self.show_default is not None:
+            if isinstance(self.show_default, str):
+                show_default_is_str = show_default = True
+            else:
+                show_default = self.show_default
+        elif ctx.show_default is not None:
+            show_default = ctx.show_default
+
+        if show_default_is_str or (
+            show_default and (default_value not in (None, UNSET))
+        ):
+            if show_default_is_str:
+                default_string = f"({self.show_default})"
+            elif isinstance(default_value, (list, tuple)):
+                default_string = ", ".join(str(d) for d in default_value)
+            elif isinstance(default_value, enum.Enum):
+                default_string = default_value.name
+            elif inspect.isfunction(default_value):
+                default_string = _("(dynamic)")
+            elif self.is_bool_flag and self.secondary_opts:
+                # For boolean flags that have distinct True/False opts,
+                # use the opt without prefix instead of the value.
+                default_string = _split_opt(
+                    (self.opts if default_value else self.secondary_opts)[0]
+                )[1]
+            elif self.is_bool_flag and not self.secondary_opts and not default_value:
+                default_string = ""
+            elif default_value == "":
+                default_string = '""'
+            else:
+                default_string = str(default_value)
+
+            if default_string:
+                extra["default"] = default_string
+
+        if (
+            isinstance(self.type, types._NumberRangeBase)
+            # skip count with default range type
+            and not (self.count and self.type.min == 0 and self.type.max is None)
+        ):
+            range_str = self.type._describe_range()
+
+            if range_str:
+                extra["range"] = range_str
+
+        if self.required:
+            extra["required"] = "required"
+
+        return extra
+
+    def prompt_for_value(self, ctx: Context) -> t.Any:
+        """This is an alternative flow that can be activated in the full
+        value processing if a value does not exist.  It will prompt the
+        user until a valid value exists and then returns the processed
+        value as result.
+        """
+        assert self.prompt is not None
+
+        # Calculate the default before prompting anything to lock in the value before
+        # attempting any user interaction.
+        default = self.get_default(ctx)
+
+        # A boolean flag can use a simplified [y/n] confirmation prompt.
+        if self.is_bool_flag:
+            # If we have no boolean default, we force the user to explicitly provide
+            # one.
+            if default in (UNSET, None):
+                default = None
+            # Nothing prevent you to declare an option that is simultaneously:
+            # 1) auto-detected as a boolean flag,
+            # 2) allowed to prompt, and
+            # 3) still declare a non-boolean default.
+            # This forced casting into a boolean is necessary to align any non-boolean
+            # default to the prompt, which is going to be a [y/n]-style confirmation
+            # because the option is still a boolean flag. That way, instead of [y/n],
+            # we get [Y/n] or [y/N] depending on the truthy value of the default.
+            # Refs: https://github.com/pallets/click/pull/3030#discussion_r2289180249
+            else:
+                default = bool(default)
+            return confirm(self.prompt, default)
+
+        # If show_default is set to True/False, provide this to `prompt` as well. For
+        # non-bool values of `show_default`, we use `prompt`'s default behavior
+        prompt_kwargs: t.Any = {}
+        if isinstance(self.show_default, bool):
+            prompt_kwargs["show_default"] = self.show_default
+
+        return prompt(
+            self.prompt,
+            # Use ``None`` to inform the prompt() function to reiterate until a valid
+            # value is provided by the user if we have no default.
+            default=None if default is UNSET else default,
+            type=self.type,
+            hide_input=self.hide_input,
+            show_choices=self.show_choices,
+            confirmation_prompt=self.confirmation_prompt,
+            value_proc=lambda x: self.process_value(ctx, x),
+            **prompt_kwargs,
+        )
+
+    def resolve_envvar_value(self, ctx: Context) -> str | None:
+        """:class:`Option` resolves its environment variable the same way as
+        :func:`Parameter.resolve_envvar_value`, but it also supports
+        :attr:`Context.auto_envvar_prefix`. If we could not find an environment from
+        the :attr:`envvar` property, we fallback on :attr:`Context.auto_envvar_prefix`
+        to build dynamiccaly the environment variable name using the
+        :python:`{ctx.auto_envvar_prefix}_{self.name.upper()}` template.
+
+        :meta private:
+        """
+        rv = super().resolve_envvar_value(ctx)
+
+        if rv is not None:
+            return rv
+
+        if (
+            self.allow_from_autoenv
+            and ctx.auto_envvar_prefix is not None
+            and self.name is not None
+        ):
+            envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}"
+            rv = os.environ.get(envvar)
+
+            if rv:
+                return rv
+
+        return None
+
+    def value_from_envvar(self, ctx: Context) -> t.Any:
+        """For :class:`Option`, this method processes the raw environment variable
+        string the same way as :func:`Parameter.value_from_envvar` does.
+
+        But in the case of non-boolean flags, the value is analyzed to determine if the
+        flag is activated or not, and returns a boolean of its activation, or the
+        :attr:`flag_value` if the latter is set.
+
+        This method also takes care of repeated options (i.e. options with
+        :attr:`multiple` set to ``True``).
+
+        :meta private:
+        """
+        rv = self.resolve_envvar_value(ctx)
+
+        # Absent environment variable or an empty string is interpreted as unset.
+        if rv is None:
+            return None
+
+        # Non-boolean flags are more liberal in what they accept. But a flag being a
+        # flag, its envvar value still needs to be analyzed to determine if the flag is
+        # activated or not.
+        if self.is_flag and not self.is_bool_flag:
+            # If the flag_value is set and match the envvar value, return it
+            # directly.
+            if self.flag_value is not UNSET and rv == self.flag_value:
+                return self.flag_value
+            # Analyze the envvar value as a boolean to know if the flag is
+            # activated or not.
+            return types.BoolParamType.str_to_bool(rv)
+
+        # Split the envvar value if it is allowed to be repeated.
+        value_depth = (self.nargs != 1) + bool(self.multiple)
+        if value_depth > 0:
+            multi_rv = self.type.split_envvar_value(rv)
+            if self.multiple and self.nargs != 1:
+                multi_rv = batch(multi_rv, self.nargs)  # type: ignore[assignment]
+
+            return multi_rv
+
+        return rv
+
+    def consume_value(
+        self, ctx: Context, opts: cabc.Mapping[str, Parameter]
+    ) -> tuple[t.Any, ParameterSource]:
+        """For :class:`Option`, the value can be collected from an interactive prompt
+        if the option is a flag that needs a value (and the :attr:`prompt` property is
+        set).
+
+        Additionally, this method handles flag option that are activated without a
+        value, in which case the :attr:`flag_value` is returned.
+
+        :meta private:
+        """
+        value, source = super().consume_value(ctx, opts)
+
+        # The parser will emit a sentinel value if the option is allowed to as a flag
+        # without a value.
+        if value is FLAG_NEEDS_VALUE:
+            # If the option allows for a prompt, we start an interaction with the user.
+            if self.prompt is not None and not ctx.resilient_parsing:
+                value = self.prompt_for_value(ctx)
+                source = ParameterSource.PROMPT
+            # Else the flag takes its flag_value as value.
+            else:
+                value = self.flag_value
+                source = ParameterSource.COMMANDLINE
+
+        # A flag which is activated always returns the flag value, unless the value
+        # comes from the explicitly sets default.
+        elif (
+            self.is_flag
+            and value is True
+            and not self.is_bool_flag
+            and source not in (ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP)
+        ):
+            value = self.flag_value
+
+        # Re-interpret a multiple option which has been sent as-is by the parser.
+        # Here we replace each occurrence of value-less flags (marked by the
+        # FLAG_NEEDS_VALUE sentinel) with the flag_value.
+        elif (
+            self.multiple
+            and value is not UNSET
+            and source not in (ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP)
+            and any(v is FLAG_NEEDS_VALUE for v in value)
+        ):
+            value = [self.flag_value if v is FLAG_NEEDS_VALUE else v for v in value]
+            source = ParameterSource.COMMANDLINE
+
+        # The value wasn't set, or used the param's default, prompt for one to the user
+        # if prompting is enabled.
+        elif (
+            (
+                value is UNSET
+                or source in (ParameterSource.DEFAULT, ParameterSource.DEFAULT_MAP)
+            )
+            and self.prompt is not None
+            and (self.required or self.prompt_required)
+            and not ctx.resilient_parsing
+        ):
+            value = self.prompt_for_value(ctx)
+            source = ParameterSource.PROMPT
+
+        return value, source
+
+    def process_value(self, ctx: Context, value: t.Any) -> t.Any:
+        # process_value has to be overridden on Options in order to capture
+        # `value == UNSET` cases before `type_cast_value()` gets called.
+        #
+        # Refs:
+        # https://github.com/pallets/click/issues/3069
+        if self.is_flag and not self.required and self.is_bool_flag and value is UNSET:
+            value = False
+
+            if self.callback is not None:
+                value = self.callback(ctx, self, value)
+
+            return value
+
+        # in the normal case, rely on Parameter.process_value
+        return super().process_value(ctx, value)
+
+
+class Argument(Parameter):
+    """Arguments are positional parameters to a command.  They generally
+    provide fewer features than options but can have infinite ``nargs``
+    and are required by default.
+
+    All parameters are passed onwards to the constructor of :class:`Parameter`.
+    """
+
+    param_type_name = "argument"
+
+    def __init__(
+        self,
+        param_decls: cabc.Sequence[str],
+        required: bool | None = None,
+        **attrs: t.Any,
+    ) -> None:
+        # Auto-detect the requirement status of the argument if not explicitly set.
+        if required is None:
+            # The argument gets automatically required if it has no explicit default
+            # value set and is setup to match at least one value.
+            if attrs.get("default", UNSET) is UNSET:
+                required = attrs.get("nargs", 1) > 0
+            # If the argument has a default value, it is not required.
+            else:
+                required = False
+
+        if "multiple" in attrs:
+            raise TypeError("__init__() got an unexpected keyword argument 'multiple'.")
+
+        super().__init__(param_decls, required=required, **attrs)
+
+    @property
+    def human_readable_name(self) -> str:
+        if self.metavar is not None:
+            return self.metavar
+        return self.name.upper()  # type: ignore
+
+    def make_metavar(self, ctx: Context) -> str:
+        if self.metavar is not None:
+            return self.metavar
+        var = self.type.get_metavar(param=self, ctx=ctx)
+        if not var:
+            var = self.name.upper()  # type: ignore
+        if self.deprecated:
+            var += "!"
+        if not self.required:
+            var = f"[{var}]"
+        if self.nargs != 1:
+            var += "..."
+        return var
+
+    def _parse_decls(
+        self, decls: cabc.Sequence[str], expose_value: bool
+    ) -> tuple[str | None, list[str], list[str]]:
+        if not decls:
+            if not expose_value:
+                return None, [], []
+            raise TypeError("Argument is marked as exposed, but does not have a name.")
+        if len(decls) == 1:
+            name = arg = decls[0]
+            name = name.replace("-", "_").lower()
+        else:
+            raise TypeError(
+                "Arguments take exactly one parameter declaration, got"
+                f" {len(decls)}: {decls}."
+            )
+        return name, [arg], []
+
+    def get_usage_pieces(self, ctx: Context) -> list[str]:
+        return [self.make_metavar(ctx)]
+
+    def get_error_hint(self, ctx: Context) -> str:
+        return f"'{self.make_metavar(ctx)}'"
+
+    def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None:
+        parser.add_argument(dest=self.name, nargs=self.nargs, obj=self)
+
+
+def __getattr__(name: str) -> object:
+    import warnings
+
+    if name == "BaseCommand":
+        warnings.warn(
+            "'BaseCommand' is deprecated and will be removed in Click 9.0. Use"
+            " 'Command' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return _BaseCommand
+
+    if name == "MultiCommand":
+        warnings.warn(
+            "'MultiCommand' is deprecated and will be removed in Click 9.0. Use"
+            " 'Group' instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return _MultiCommand
+
+    raise AttributeError(name)

python/py/Lib/site-packages/click/decorators.py

@@ -0,0 +1,551 @@
+from __future__ import annotations
+
+import inspect
+import typing as t
+from functools import update_wrapper
+from gettext import gettext as _
+
+from .core import Argument
+from .core import Command
+from .core import Context
+from .core import Group
+from .core import Option
+from .core import Parameter
+from .globals import get_current_context
+from .utils import echo
+
+if t.TYPE_CHECKING:
+    import typing_extensions as te
+
+    P = te.ParamSpec("P")
+
+R = t.TypeVar("R")
+T = t.TypeVar("T")
+_AnyCallable = t.Callable[..., t.Any]
+FC = t.TypeVar("FC", bound="_AnyCallable | Command")
+
+
+def pass_context(f: t.Callable[te.Concatenate[Context, P], R]) -> t.Callable[P, R]:
+    """Marks a callback as wanting to receive the current context
+    object as first argument.
+    """
+
+    def new_func(*args: P.args, **kwargs: P.kwargs) -> R:
+        return f(get_current_context(), *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+def pass_obj(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]:
+    """Similar to :func:`pass_context`, but only pass the object on the
+    context onwards (:attr:`Context.obj`).  This is useful if that object
+    represents the state of a nested system.
+    """
+
+    def new_func(*args: P.args, **kwargs: P.kwargs) -> R:
+        return f(get_current_context().obj, *args, **kwargs)
+
+    return update_wrapper(new_func, f)
+
+
+def make_pass_decorator(
+    object_type: type[T], ensure: bool = False
+) -> t.Callable[[t.Callable[te.Concatenate[T, P], R]], t.Callable[P, R]]:
+    """Given an object type this creates a decorator that will work
+    similar to :func:`pass_obj` but instead of passing the object of the
+    current context, it will find the innermost context of type
+    :func:`object_type`.
+
+    This generates a decorator that works roughly like this::
+
+        from functools import update_wrapper
+
+        def decorator(f):
+            @pass_context
+            def new_func(ctx, *args, **kwargs):
+                obj = ctx.find_object(object_type)
+                return ctx.invoke(f, obj, *args, **kwargs)
+            return update_wrapper(new_func, f)
+        return decorator
+
+    :param object_type: the type of the object to pass.
+    :param ensure: if set to `True`, a new object will be created and
+                   remembered on the context if it's not there yet.
+    """
+
+    def decorator(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]:
+        def new_func(*args: P.args, **kwargs: P.kwargs) -> R:
+            ctx = get_current_context()
+
+            obj: T | None
+            if ensure:
+                obj = ctx.ensure_object(object_type)
+            else:
+                obj = ctx.find_object(object_type)
+
+            if obj is None:
+                raise RuntimeError(
+                    "Managed to invoke callback without a context"
+                    f" object of type {object_type.__name__!r}"
+                    " existing."
+                )
+
+            return ctx.invoke(f, obj, *args, **kwargs)
+
+        return update_wrapper(new_func, f)
+
+    return decorator
+
+
+def pass_meta_key(
+    key: str, *, doc_description: str | None = None
+) -> t.Callable[[t.Callable[te.Concatenate[T, P], R]], t.Callable[P, R]]:
+    """Create a decorator that passes a key from
+    :attr:`click.Context.meta` as the first argument to the decorated
+    function.
+
+    :param key: Key in ``Context.meta`` to pass.
+    :param doc_description: Description of the object being passed,
+        inserted into the decorator's docstring. Defaults to "the 'key'
+        key from Context.meta".
+
+    .. versionadded:: 8.0
+    """
+
+    def decorator(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]:
+        def new_func(*args: P.args, **kwargs: P.kwargs) -> R:
+            ctx = get_current_context()
+            obj = ctx.meta[key]
+            return ctx.invoke(f, obj, *args, **kwargs)
+
+        return update_wrapper(new_func, f)
+
+    if doc_description is None:
+        doc_description = f"the {key!r} key from :attr:`click.Context.meta`"
+
+    decorator.__doc__ = (
+        f"Decorator that passes {doc_description} as the first argument"
+        " to the decorated function."
+    )
+    return decorator
+
+
+CmdType = t.TypeVar("CmdType", bound=Command)
+
+
+# variant: no call, directly as decorator for a function.
+@t.overload
+def command(name: _AnyCallable) -> Command: ...
+
+
+# variant: with positional name and with positional or keyword cls argument:
+# @command(namearg, CommandCls, ...) or @command(namearg, cls=CommandCls, ...)
+@t.overload
+def command(
+    name: str | None,
+    cls: type[CmdType],
+    **attrs: t.Any,
+) -> t.Callable[[_AnyCallable], CmdType]: ...
+
+
+# variant: name omitted, cls _must_ be a keyword argument, @command(cls=CommandCls, ...)
+@t.overload
+def command(
+    name: None = None,
+    *,
+    cls: type[CmdType],
+    **attrs: t.Any,
+) -> t.Callable[[_AnyCallable], CmdType]: ...
+
+
+# variant: with optional string name, no cls argument provided.
+@t.overload
+def command(
+    name: str | None = ..., cls: None = None, **attrs: t.Any
+) -> t.Callable[[_AnyCallable], Command]: ...
+
+
+def command(
+    name: str | _AnyCallable | None = None,
+    cls: type[CmdType] | None = None,
+    **attrs: t.Any,
+) -> Command | t.Callable[[_AnyCallable], Command | CmdType]:
+    r"""Creates a new :class:`Command` and uses the decorated function as
+    callback.  This will also automatically attach all decorated
+    :func:`option`\s and :func:`argument`\s as parameters to the command.
+
+    The name of the command defaults to the name of the function, converted to
+    lowercase, with underscores ``_`` replaced by dashes ``-``, and the suffixes
+    ``_command``, ``_cmd``, ``_group``, and ``_grp`` are removed. For example,
+    ``init_data_command`` becomes ``init-data``.
+
+    All keyword arguments are forwarded to the underlying command class.
+    For the ``params`` argument, any decorated params are appended to
+    the end of the list.
+
+    Once decorated the function turns into a :class:`Command` instance
+    that can be invoked as a command line utility or be attached to a
+    command :class:`Group`.
+
+    :param name: The name of the command. Defaults to modifying the function's
+        name as described above.
+    :param cls: The command class to create. Defaults to :class:`Command`.
+
+    .. versionchanged:: 8.2
+        The suffixes ``_command``, ``_cmd``, ``_group``, and ``_grp`` are
+        removed when generating the name.
+
+    .. versionchanged:: 8.1
+        This decorator can be applied without parentheses.
+
+    .. versionchanged:: 8.1
+        The ``params`` argument can be used. Decorated params are
+        appended to the end of the list.
+    """
+
+    func: t.Callable[[_AnyCallable], t.Any] | None = None
+
+    if callable(name):
+        func = name
+        name = None
+        assert cls is None, "Use 'command(cls=cls)(callable)' to specify a class."
+        assert not attrs, "Use 'command(**kwargs)(callable)' to provide arguments."
+
+    if cls is None:
+        cls = t.cast("type[CmdType]", Command)
+
+    def decorator(f: _AnyCallable) -> CmdType:
+        if isinstance(f, Command):
+            raise TypeError("Attempted to convert a callback into a command twice.")
+
+        attr_params = attrs.pop("params", None)
+        params = attr_params if attr_params is not None else []
+
+        try:
+            decorator_params = f.__click_params__  # type: ignore
+        except AttributeError:
+            pass
+        else:
+            del f.__click_params__  # type: ignore
+            params.extend(reversed(decorator_params))
+
+        if attrs.get("help") is None:
+            attrs["help"] = f.__doc__
+
+        if t.TYPE_CHECKING:
+            assert cls is not None
+            assert not callable(name)
+
+        if name is not None:
+            cmd_name = name
+        else:
+            cmd_name = f.__name__.lower().replace("_", "-")
+            cmd_left, sep, suffix = cmd_name.rpartition("-")
+
+            if sep and suffix in {"command", "cmd", "group", "grp"}:
+                cmd_name = cmd_left
+
+        cmd = cls(name=cmd_name, callback=f, params=params, **attrs)
+        cmd.__doc__ = f.__doc__
+        return cmd
+
+    if func is not None:
+        return decorator(func)
+
+    return decorator
+
+
+GrpType = t.TypeVar("GrpType", bound=Group)
+
+
+# variant: no call, directly as decorator for a function.
+@t.overload
+def group(name: _AnyCallable) -> Group: ...
+
+
+# variant: with positional name and with positional or keyword cls argument:
+# @group(namearg, GroupCls, ...) or @group(namearg, cls=GroupCls, ...)
+@t.overload
+def group(
+    name: str | None,
+    cls: type[GrpType],
+    **attrs: t.Any,
+) -> t.Callable[[_AnyCallable], GrpType]: ...
+
+
+# variant: name omitted, cls _must_ be a keyword argument, @group(cmd=GroupCls, ...)
+@t.overload
+def group(
+    name: None = None,
+    *,
+    cls: type[GrpType],
+    **attrs: t.Any,
+) -> t.Callable[[_AnyCallable], GrpType]: ...
+
+
+# variant: with optional string name, no cls argument provided.
+@t.overload
+def group(
+    name: str | None = ..., cls: None = None, **attrs: t.Any
+) -> t.Callable[[_AnyCallable], Group]: ...
+
+
+def group(
+    name: str | _AnyCallable | None = None,
+    cls: type[GrpType] | None = None,
+    **attrs: t.Any,
+) -> Group | t.Callable[[_AnyCallable], Group | GrpType]:
+    """Creates a new :class:`Group` with a function as callback.  This
+    works otherwise the same as :func:`command` just that the `cls`
+    parameter is set to :class:`Group`.
+
+    .. versionchanged:: 8.1
+        This decorator can be applied without parentheses.
+    """
+    if cls is None:
+        cls = t.cast("type[GrpType]", Group)
+
+    if callable(name):
+        return command(cls=cls, **attrs)(name)
+
+    return command(name, cls, **attrs)
+
+
+def _param_memo(f: t.Callable[..., t.Any], param: Parameter) -> None:
+    if isinstance(f, Command):
+        f.params.append(param)
+    else:
+        if not hasattr(f, "__click_params__"):
+            f.__click_params__ = []  # type: ignore
+
+        f.__click_params__.append(param)  # type: ignore
+
+
+def argument(
+    *param_decls: str, cls: type[Argument] | None = None, **attrs: t.Any
+) -> t.Callable[[FC], FC]:
+    """Attaches an argument to the command.  All positional arguments are
+    passed as parameter declarations to :class:`Argument`; all keyword
+    arguments are forwarded unchanged (except ``cls``).
+    This is equivalent to creating an :class:`Argument` instance manually
+    and attaching it to the :attr:`Command.params` list.
+
+    For the default argument class, refer to :class:`Argument` and
+    :class:`Parameter` for descriptions of parameters.
+
+    :param cls: the argument class to instantiate.  This defaults to
+                :class:`Argument`.
+    :param param_decls: Passed as positional arguments to the constructor of
+        ``cls``.
+    :param attrs: Passed as keyword arguments to the constructor of ``cls``.
+    """
+    if cls is None:
+        cls = Argument
+
+    def decorator(f: FC) -> FC:
+        _param_memo(f, cls(param_decls, **attrs))
+        return f
+
+    return decorator
+
+
+def option(
+    *param_decls: str, cls: type[Option] | None = None, **attrs: t.Any
+) -> t.Callable[[FC], FC]:
+    """Attaches an option to the command.  All positional arguments are
+    passed as parameter declarations to :class:`Option`; all keyword
+    arguments are forwarded unchanged (except ``cls``).
+    This is equivalent to creating an :class:`Option` instance manually
+    and attaching it to the :attr:`Command.params` list.
+
+    For the default option class, refer to :class:`Option` and
+    :class:`Parameter` for descriptions of parameters.
+
+    :param cls: the option class to instantiate.  This defaults to
+                :class:`Option`.
+    :param param_decls: Passed as positional arguments to the constructor of
+        ``cls``.
+    :param attrs: Passed as keyword arguments to the constructor of ``cls``.
+    """
+    if cls is None:
+        cls = Option
+
+    def decorator(f: FC) -> FC:
+        _param_memo(f, cls(param_decls, **attrs))
+        return f
+
+    return decorator
+
+
+def confirmation_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]:
+    """Add a ``--yes`` option which shows a prompt before continuing if
+    not passed. If the prompt is declined, the program will exit.
+
+    :param param_decls: One or more option names. Defaults to the single
+        value ``"--yes"``.
+    :param kwargs: Extra arguments are passed to :func:`option`.
+    """
+
+    def callback(ctx: Context, param: Parameter, value: bool) -> None:
+        if not value:
+            ctx.abort()
+
+    if not param_decls:
+        param_decls = ("--yes",)
+
+    kwargs.setdefault("is_flag", True)
+    kwargs.setdefault("callback", callback)
+    kwargs.setdefault("expose_value", False)
+    kwargs.setdefault("prompt", "Do you want to continue?")
+    kwargs.setdefault("help", "Confirm the action without prompting.")
+    return option(*param_decls, **kwargs)
+
+
+def password_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]:
+    """Add a ``--password`` option which prompts for a password, hiding
+    input and asking to enter the value again for confirmation.
+
+    :param param_decls: One or more option names. Defaults to the single
+        value ``"--password"``.
+    :param kwargs: Extra arguments are passed to :func:`option`.
+    """
+    if not param_decls:
+        param_decls = ("--password",)
+
+    kwargs.setdefault("prompt", True)
+    kwargs.setdefault("confirmation_prompt", True)
+    kwargs.setdefault("hide_input", True)
+    return option(*param_decls, **kwargs)
+
+
+def version_option(
+    version: str | None = None,
+    *param_decls: str,
+    package_name: str | None = None,
+    prog_name: str | None = None,
+    message: str | None = None,
+    **kwargs: t.Any,
+) -> t.Callable[[FC], FC]:
+    """Add a ``--version`` option which immediately prints the version
+    number and exits the program.
+
+    If ``version`` is not provided, Click will try to detect it using
+    :func:`importlib.metadata.version` to get the version for the
+    ``package_name``.
+
+    If ``package_name`` is not provided, Click will try to detect it by
+    inspecting the stack frames. This will be used to detect the
+    version, so it must match the name of the installed package.
+
+    :param version: The version number to show. If not provided, Click
+        will try to detect it.
+    :param param_decls: One or more option names. Defaults to the single
+        value ``"--version"``.
+    :param package_name: The package name to detect the version from. If
+        not provided, Click will try to detect it.
+    :param prog_name: The name of the CLI to show in the message. If not
+        provided, it will be detected from the command.
+    :param message: The message to show. The values ``%(prog)s``,
+        ``%(package)s``, and ``%(version)s`` are available. Defaults to
+        ``"%(prog)s, version %(version)s"``.
+    :param kwargs: Extra arguments are passed to :func:`option`.
+    :raise RuntimeError: ``version`` could not be detected.
+
+    .. versionchanged:: 8.0
+        Add the ``package_name`` parameter, and the ``%(package)s``
+        value for messages.
+
+    .. versionchanged:: 8.0
+        Use :mod:`importlib.metadata` instead of ``pkg_resources``. The
+        version is detected based on the package name, not the entry
+        point name. The Python package name must match the installed
+        package name, or be passed with ``package_name=``.
+    """
+    if message is None:
+        message = _("%(prog)s, version %(version)s")
+
+    if version is None and package_name is None:
+        frame = inspect.currentframe()
+        f_back = frame.f_back if frame is not None else None
+        f_globals = f_back.f_globals if f_back is not None else None
+        # break reference cycle
+        # https://docs.python.org/3/library/inspect.html#the-interpreter-stack
+        del frame
+
+        if f_globals is not None:
+            package_name = f_globals.get("__name__")
+
+            if package_name == "__main__":
+                package_name = f_globals.get("__package__")
+
+            if package_name:
+                package_name = package_name.partition(".")[0]
+
+    def callback(ctx: Context, param: Parameter, value: bool) -> None:
+        if not value or ctx.resilient_parsing:
+            return
+
+        nonlocal prog_name
+        nonlocal version
+
+        if prog_name is None:
+            prog_name = ctx.find_root().info_name
+
+        if version is None and package_name is not None:
+            import importlib.metadata
+
+            try:
+                version = importlib.metadata.version(package_name)
+            except importlib.metadata.PackageNotFoundError:
+                raise RuntimeError(
+                    f"{package_name!r} is not installed. Try passing"
+                    " 'package_name' instead."
+                ) from None
+
+        if version is None:
+            raise RuntimeError(
+                f"Could not determine the version for {package_name!r} automatically."
+            )
+
+        echo(
+            message % {"prog": prog_name, "package": package_name, "version": version},
+            color=ctx.color,
+        )
+        ctx.exit()
+
+    if not param_decls:
+        param_decls = ("--version",)
+
+    kwargs.setdefault("is_flag", True)
+    kwargs.setdefault("expose_value", False)
+    kwargs.setdefault("is_eager", True)
+    kwargs.setdefault("help", _("Show the version and exit."))
+    kwargs["callback"] = callback
+    return option(*param_decls, **kwargs)
+
+
+def help_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]:
+    """Pre-configured ``--help`` option which immediately prints the help page
+    and exits the program.
+
+    :param param_decls: One or more option names. Defaults to the single
+        value ``"--help"``.
+    :param kwargs: Extra arguments are passed to :func:`option`.
+    """
+
+    def show_help(ctx: Context, param: Parameter, value: bool) -> None:
+        """Callback that print the help page on ``<stdout>`` and exits."""
+        if value and not ctx.resilient_parsing:
+            echo(ctx.get_help(), color=ctx.color)
+            ctx.exit()
+
+    if not param_decls:
+        param_decls = ("--help",)
+
+    kwargs.setdefault("is_flag", True)
+    kwargs.setdefault("expose_value", False)
+    kwargs.setdefault("is_eager", True)
+    kwargs.setdefault("help", _("Show this message and exit."))
+    kwargs.setdefault("callback", show_help)
+
+    return option(*param_decls, **kwargs)

python/py/Lib/site-packages/click/exceptions.py

@@ -0,0 +1,308 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+import typing as t
+from gettext import gettext as _
+from gettext import ngettext
+
+from ._compat import get_text_stderr
+from .globals import resolve_color_default
+from .utils import echo
+from .utils import format_filename
+
+if t.TYPE_CHECKING:
+    from .core import Command
+    from .core import Context
+    from .core import Parameter
+
+
+def _join_param_hints(param_hint: cabc.Sequence[str] | str | None) -> str | None:
+    if param_hint is not None and not isinstance(param_hint, str):
+        return " / ".join(repr(x) for x in param_hint)
+
+    return param_hint
+
+
+class ClickException(Exception):
+    """An exception that Click can handle and show to the user."""
+
+    #: The exit code for this exception.
+    exit_code = 1
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        # The context will be removed by the time we print the message, so cache
+        # the color settings here to be used later on (in `show`)
+        self.show_color: bool | None = resolve_color_default()
+        self.message = message
+
+    def format_message(self) -> str:
+        return self.message
+
+    def __str__(self) -> str:
+        return self.message
+
+    def show(self, file: t.IO[t.Any] | None = None) -> None:
+        if file is None:
+            file = get_text_stderr()
+
+        echo(
+            _("Error: {message}").format(message=self.format_message()),
+            file=file,
+            color=self.show_color,
+        )
+
+
+class UsageError(ClickException):
+    """An internal exception that signals a usage error.  This typically
+    aborts any further handling.
+
+    :param message: the error message to display.
+    :param ctx: optionally the context that caused this error.  Click will
+                fill in the context automatically in some situations.
+    """
+
+    exit_code = 2
+
+    def __init__(self, message: str, ctx: Context | None = None) -> None:
+        super().__init__(message)
+        self.ctx = ctx
+        self.cmd: Command | None = self.ctx.command if self.ctx else None
+
+    def show(self, file: t.IO[t.Any] | None = None) -> None:
+        if file is None:
+            file = get_text_stderr()
+        color = None
+        hint = ""
+        if (
+            self.ctx is not None
+            and self.ctx.command.get_help_option(self.ctx) is not None
+        ):
+            hint = _("Try '{command} {option}' for help.").format(
+                command=self.ctx.command_path, option=self.ctx.help_option_names[0]
+            )
+            hint = f"{hint}\n"
+        if self.ctx is not None:
+            color = self.ctx.color
+            echo(f"{self.ctx.get_usage()}\n{hint}", file=file, color=color)
+        echo(
+            _("Error: {message}").format(message=self.format_message()),
+            file=file,
+            color=color,
+        )
+
+
+class BadParameter(UsageError):
+    """An exception that formats out a standardized error message for a
+    bad parameter.  This is useful when thrown from a callback or type as
+    Click will attach contextual information to it (for instance, which
+    parameter it is).
+
+    .. versionadded:: 2.0
+
+    :param param: the parameter object that caused this error.  This can
+                  be left out, and Click will attach this info itself
+                  if possible.
+    :param param_hint: a string that shows up as parameter name.  This
+                       can be used as alternative to `param` in cases
+                       where custom validation should happen.  If it is
+                       a string it's used as such, if it's a list then
+                       each item is quoted and separated.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        ctx: Context | None = None,
+        param: Parameter | None = None,
+        param_hint: cabc.Sequence[str] | str | None = None,
+    ) -> None:
+        super().__init__(message, ctx)
+        self.param = param
+        self.param_hint = param_hint
+
+    def format_message(self) -> str:
+        if self.param_hint is not None:
+            param_hint = self.param_hint
+        elif self.param is not None:
+            param_hint = self.param.get_error_hint(self.ctx)  # type: ignore
+        else:
+            return _("Invalid value: {message}").format(message=self.message)
+
+        return _("Invalid value for {param_hint}: {message}").format(
+            param_hint=_join_param_hints(param_hint), message=self.message
+        )
+
+
+class MissingParameter(BadParameter):
+    """Raised if click required an option or argument but it was not
+    provided when invoking the script.
+
+    .. versionadded:: 4.0
+
+    :param param_type: a string that indicates the type of the parameter.
+                       The default is to inherit the parameter type from
+                       the given `param`.  Valid values are ``'parameter'``,
+                       ``'option'`` or ``'argument'``.
+    """
+
+    def __init__(
+        self,
+        message: str | None = None,
+        ctx: Context | None = None,
+        param: Parameter | None = None,
+        param_hint: cabc.Sequence[str] | str | None = None,
+        param_type: str | None = None,
+    ) -> None:
+        super().__init__(message or "", ctx, param, param_hint)
+        self.param_type = param_type
+
+    def format_message(self) -> str:
+        if self.param_hint is not None:
+            param_hint: cabc.Sequence[str] | str | None = self.param_hint
+        elif self.param is not None:
+            param_hint = self.param.get_error_hint(self.ctx)  # type: ignore
+        else:
+            param_hint = None
+
+        param_hint = _join_param_hints(param_hint)
+        param_hint = f" {param_hint}" if param_hint else ""
+
+        param_type = self.param_type
+        if param_type is None and self.param is not None:
+            param_type = self.param.param_type_name
+
+        msg = self.message
+        if self.param is not None:
+            msg_extra = self.param.type.get_missing_message(
+                param=self.param, ctx=self.ctx
+            )
+            if msg_extra:
+                if msg:
+                    msg += f". {msg_extra}"
+                else:
+                    msg = msg_extra
+
+        msg = f" {msg}" if msg else ""
+
+        # Translate param_type for known types.
+        if param_type == "argument":
+            missing = _("Missing argument")
+        elif param_type == "option":
+            missing = _("Missing option")
+        elif param_type == "parameter":
+            missing = _("Missing parameter")
+        else:
+            missing = _("Missing {param_type}").format(param_type=param_type)
+
+        return f"{missing}{param_hint}.{msg}"
+
+    def __str__(self) -> str:
+        if not self.message:
+            param_name = self.param.name if self.param else None
+            return _("Missing parameter: {param_name}").format(param_name=param_name)
+        else:
+            return self.message
+
+
+class NoSuchOption(UsageError):
+    """Raised if click attempted to handle an option that does not
+    exist.
+
+    .. versionadded:: 4.0
+    """
+
+    def __init__(
+        self,
+        option_name: str,
+        message: str | None = None,
+        possibilities: cabc.Sequence[str] | None = None,
+        ctx: Context | None = None,
+    ) -> None:
+        if message is None:
+            message = _("No such option: {name}").format(name=option_name)
+
+        super().__init__(message, ctx)
+        self.option_name = option_name
+        self.possibilities = possibilities
+
+    def format_message(self) -> str:
+        if not self.possibilities:
+            return self.message
+
+        possibility_str = ", ".join(sorted(self.possibilities))
+        suggest = ngettext(
+            "Did you mean {possibility}?",
+            "(Possible options: {possibilities})",
+            len(self.possibilities),
+        ).format(possibility=possibility_str, possibilities=possibility_str)
+        return f"{self.message} {suggest}"
+
+
+class BadOptionUsage(UsageError):
+    """Raised if an option is generally supplied but the use of the option
+    was incorrect.  This is for instance raised if the number of arguments
+    for an option is not correct.
+
+    .. versionadded:: 4.0
+
+    :param option_name: the name of the option being used incorrectly.
+    """
+
+    def __init__(
+        self, option_name: str, message: str, ctx: Context | None = None
+    ) -> None:
+        super().__init__(message, ctx)
+        self.option_name = option_name
+
+
+class BadArgumentUsage(UsageError):
+    """Raised if an argument is generally supplied but the use of the argument
+    was incorrect.  This is for instance raised if the number of values
+    for an argument is not correct.
+
+    .. versionadded:: 6.0
+    """
+
+
+class NoArgsIsHelpError(UsageError):
+    def __init__(self, ctx: Context) -> None:
+        self.ctx: Context
+        super().__init__(ctx.get_help(), ctx=ctx)
+
+    def show(self, file: t.IO[t.Any] | None = None) -> None:
+        echo(self.format_message(), file=file, err=True, color=self.ctx.color)
+
+
+class FileError(ClickException):
+    """Raised if a file cannot be opened."""
+
+    def __init__(self, filename: str, hint: str | None = None) -> None:
+        if hint is None:
+            hint = _("unknown error")
+
+        super().__init__(hint)
+        self.ui_filename: str = format_filename(filename)
+        self.filename = filename
+
+    def format_message(self) -> str:
+        return _("Could not open file {filename!r}: {message}").format(
+            filename=self.ui_filename, message=self.message
+        )
+
+
+class Abort(RuntimeError):
+    """An internal signalling exception that signals Click to abort."""
+
+
+class Exit(RuntimeError):
+    """An exception that indicates that the application should exit with some
+    status code.
+
+    :param code: the status code to exit with.
+    """
+
+    __slots__ = ("exit_code",)
+
+    def __init__(self, code: int = 0) -> None:
+        self.exit_code: int = code

python/py/Lib/site-packages/click/formatting.py

@@ -0,0 +1,301 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+from contextlib import contextmanager
+from gettext import gettext as _
+
+from ._compat import term_len
+from .parser import _split_opt
+
+# Can force a width.  This is used by the test system
+FORCED_WIDTH: int | None = None
+
+
+def measure_table(rows: cabc.Iterable[tuple[str, str]]) -> tuple[int, ...]:
+    widths: dict[int, int] = {}
+
+    for row in rows:
+        for idx, col in enumerate(row):
+            widths[idx] = max(widths.get(idx, 0), term_len(col))
+
+    return tuple(y for x, y in sorted(widths.items()))
+
+
+def iter_rows(
+    rows: cabc.Iterable[tuple[str, str]], col_count: int
+) -> cabc.Iterator[tuple[str, ...]]:
+    for row in rows:
+        yield row + ("",) * (col_count - len(row))
+
+
+def wrap_text(
+    text: str,
+    width: int = 78,
+    initial_indent: str = "",
+    subsequent_indent: str = "",
+    preserve_paragraphs: bool = False,
+) -> str:
+    """A helper function that intelligently wraps text.  By default, it
+    assumes that it operates on a single paragraph of text but if the
+    `preserve_paragraphs` parameter is provided it will intelligently
+    handle paragraphs (defined by two empty lines).
+
+    If paragraphs are handled, a paragraph can be prefixed with an empty
+    line containing the ``\\b`` character (``\\x08``) to indicate that
+    no rewrapping should happen in that block.
+
+    :param text: the text that should be rewrapped.
+    :param width: the maximum width for the text.
+    :param initial_indent: the initial indent that should be placed on the
+                           first line as a string.
+    :param subsequent_indent: the indent string that should be placed on
+                              each consecutive line.
+    :param preserve_paragraphs: if this flag is set then the wrapping will
+                                intelligently handle paragraphs.
+    """
+    from ._textwrap import TextWrapper
+
+    text = text.expandtabs()
+    wrapper = TextWrapper(
+        width,
+        initial_indent=initial_indent,
+        subsequent_indent=subsequent_indent,
+        replace_whitespace=False,
+    )
+    if not preserve_paragraphs:
+        return wrapper.fill(text)
+
+    p: list[tuple[int, bool, str]] = []
+    buf: list[str] = []
+    indent = None
+
+    def _flush_par() -> None:
+        if not buf:
+            return
+        if buf[0].strip() == "\b":
+            p.append((indent or 0, True, "\n".join(buf[1:])))
+        else:
+            p.append((indent or 0, False, " ".join(buf)))
+        del buf[:]
+
+    for line in text.splitlines():
+        if not line:
+            _flush_par()
+            indent = None
+        else:
+            if indent is None:
+                orig_len = term_len(line)
+                line = line.lstrip()
+                indent = orig_len - term_len(line)
+            buf.append(line)
+    _flush_par()
+
+    rv = []
+    for indent, raw, text in p:
+        with wrapper.extra_indent(" " * indent):
+            if raw:
+                rv.append(wrapper.indent_only(text))
+            else:
+                rv.append(wrapper.fill(text))
+
+    return "\n\n".join(rv)
+
+
+class HelpFormatter:
+    """This class helps with formatting text-based help pages.  It's
+    usually just needed for very special internal cases, but it's also
+    exposed so that developers can write their own fancy outputs.
+
+    At present, it always writes into memory.
+
+    :param indent_increment: the additional increment for each level.
+    :param width: the width for the text.  This defaults to the terminal
+                  width clamped to a maximum of 78.
+    """
+
+    def __init__(
+        self,
+        indent_increment: int = 2,
+        width: int | None = None,
+        max_width: int | None = None,
+    ) -> None:
+        self.indent_increment = indent_increment
+        if max_width is None:
+            max_width = 80
+        if width is None:
+            import shutil
+
+            width = FORCED_WIDTH
+            if width is None:
+                width = max(min(shutil.get_terminal_size().columns, max_width) - 2, 50)
+        self.width = width
+        self.current_indent: int = 0
+        self.buffer: list[str] = []
+
+    def write(self, string: str) -> None:
+        """Writes a unicode string into the internal buffer."""
+        self.buffer.append(string)
+
+    def indent(self) -> None:
+        """Increases the indentation."""
+        self.current_indent += self.indent_increment
+
+    def dedent(self) -> None:
+        """Decreases the indentation."""
+        self.current_indent -= self.indent_increment
+
+    def write_usage(self, prog: str, args: str = "", prefix: str | None = None) -> None:
+        """Writes a usage line into the buffer.
+
+        :param prog: the program name.
+        :param args: whitespace separated list of arguments.
+        :param prefix: The prefix for the first line. Defaults to
+            ``"Usage: "``.
+        """
+        if prefix is None:
+            prefix = f"{_('Usage:')} "
+
+        usage_prefix = f"{prefix:>{self.current_indent}}{prog} "
+        text_width = self.width - self.current_indent
+
+        if text_width >= (term_len(usage_prefix) + 20):
+            # The arguments will fit to the right of the prefix.
+            indent = " " * term_len(usage_prefix)
+            self.write(
+                wrap_text(
+                    args,
+                    text_width,
+                    initial_indent=usage_prefix,
+                    subsequent_indent=indent,
+                )
+            )
+        else:
+            # The prefix is too long, put the arguments on the next line.
+            self.write(usage_prefix)
+            self.write("\n")
+            indent = " " * (max(self.current_indent, term_len(prefix)) + 4)
+            self.write(
+                wrap_text(
+                    args, text_width, initial_indent=indent, subsequent_indent=indent
+                )
+            )
+
+        self.write("\n")
+
+    def write_heading(self, heading: str) -> None:
+        """Writes a heading into the buffer."""
+        self.write(f"{'':>{self.current_indent}}{heading}:\n")
+
+    def write_paragraph(self) -> None:
+        """Writes a paragraph into the buffer."""
+        if self.buffer:
+            self.write("\n")
+
+    def write_text(self, text: str) -> None:
+        """Writes re-indented text into the buffer.  This rewraps and
+        preserves paragraphs.
+        """
+        indent = " " * self.current_indent
+        self.write(
+            wrap_text(
+                text,
+                self.width,
+                initial_indent=indent,
+                subsequent_indent=indent,
+                preserve_paragraphs=True,
+            )
+        )
+        self.write("\n")
+
+    def write_dl(
+        self,
+        rows: cabc.Sequence[tuple[str, str]],
+        col_max: int = 30,
+        col_spacing: int = 2,
+    ) -> None:
+        """Writes a definition list into the buffer.  This is how options
+        and commands are usually formatted.
+
+        :param rows: a list of two item tuples for the terms and values.
+        :param col_max: the maximum width of the first column.
+        :param col_spacing: the number of spaces between the first and
+                            second column.
+        """
+        rows = list(rows)
+        widths = measure_table(rows)
+        if len(widths) != 2:
+            raise TypeError("Expected two columns for definition list")
+
+        first_col = min(widths[0], col_max) + col_spacing
+
+        for first, second in iter_rows(rows, len(widths)):
+            self.write(f"{'':>{self.current_indent}}{first}")
+            if not second:
+                self.write("\n")
+                continue
+            if term_len(first) <= first_col - col_spacing:
+                self.write(" " * (first_col - term_len(first)))
+            else:
+                self.write("\n")
+                self.write(" " * (first_col + self.current_indent))
+
+            text_width = max(self.width - first_col - 2, 10)
+            wrapped_text = wrap_text(second, text_width, preserve_paragraphs=True)
+            lines = wrapped_text.splitlines()
+
+            if lines:
+                self.write(f"{lines[0]}\n")
+
+                for line in lines[1:]:
+                    self.write(f"{'':>{first_col + self.current_indent}}{line}\n")
+            else:
+                self.write("\n")
+
+    @contextmanager
+    def section(self, name: str) -> cabc.Iterator[None]:
+        """Helpful context manager that writes a paragraph, a heading,
+        and the indents.
+
+        :param name: the section name that is written as heading.
+        """
+        self.write_paragraph()
+        self.write_heading(name)
+        self.indent()
+        try:
+            yield
+        finally:
+            self.dedent()
+
+    @contextmanager
+    def indentation(self) -> cabc.Iterator[None]:
+        """A context manager that increases the indentation."""
+        self.indent()
+        try:
+            yield
+        finally:
+            self.dedent()
+
+    def getvalue(self) -> str:
+        """Returns the buffer contents."""
+        return "".join(self.buffer)
+
+
+def join_options(options: cabc.Sequence[str]) -> tuple[str, bool]:
+    """Given a list of option strings this joins them in the most appropriate
+    way and returns them in the form ``(formatted_string,
+    any_prefix_is_slash)`` where the second item in the tuple is a flag that
+    indicates if any of the option prefixes was a slash.
+    """
+    rv = []
+    any_prefix_is_slash = False
+
+    for opt in options:
+        prefix = _split_opt(opt)[0]
+
+        if prefix == "/":
+            any_prefix_is_slash = True
+
+        rv.append((len(prefix), opt))
+
+    rv.sort(key=lambda x: x[0])
+    return ", ".join(x[1] for x in rv), any_prefix_is_slash

python/py/Lib/site-packages/click/globals.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import typing as t
+from threading import local
+
+if t.TYPE_CHECKING:
+    from .core import Context
+
+_local = local()
+
+
+@t.overload
+def get_current_context(silent: t.Literal[False] = False) -> Context: ...
+
+
+@t.overload
+def get_current_context(silent: bool = ...) -> Context | None: ...
+
+
+def get_current_context(silent: bool = False) -> Context | None:
+    """Returns the current click context.  This can be used as a way to
+    access the current context object from anywhere.  This is a more implicit
+    alternative to the :func:`pass_context` decorator.  This function is
+    primarily useful for helpers such as :func:`echo` which might be
+    interested in changing its behavior based on the current context.
+
+    To push the current context, :meth:`Context.scope` can be used.
+
+    .. versionadded:: 5.0
+
+    :param silent: if set to `True` the return value is `None` if no context
+                   is available.  The default behavior is to raise a
+                   :exc:`RuntimeError`.
+    """
+    try:
+        return t.cast("Context", _local.stack[-1])
+    except (AttributeError, IndexError) as e:
+        if not silent:
+            raise RuntimeError("There is no active click context.") from e
+
+    return None
+
+
+def push_context(ctx: Context) -> None:
+    """Pushes a new context to the current stack."""
+    _local.__dict__.setdefault("stack", []).append(ctx)
+
+
+def pop_context() -> None:
+    """Removes the top level from the stack."""
+    _local.stack.pop()
+
+
+def resolve_color_default(color: bool | None = None) -> bool | None:
+    """Internal helper to get the default value of the color flag.  If a
+    value is passed it's returned unchanged, otherwise it's looked up from
+    the current context.
+    """
+    if color is not None:
+        return color
+
+    ctx = get_current_context(silent=True)
+
+    if ctx is not None:
+        return ctx.color
+
+    return None

python/py/Lib/site-packages/click/parser.py

@@ -0,0 +1,532 @@
+"""
+This module started out as largely a copy paste from the stdlib's
+optparse module with the features removed that we do not need from
+optparse because we implement them in Click on a higher level (for
+instance type handling, help formatting and a lot more).
+
+The plan is to remove more and more from here over time.
+
+The reason this is a different module and not optparse from the stdlib
+is that there are differences in 2.x and 3.x about the error messages
+generated and optparse in the stdlib uses gettext for no good reason
+and might cause us issues.
+
+Click uses parts of optparse written by Gregory P. Ward and maintained
+by the Python Software Foundation. This is limited to code in parser.py.
+
+Copyright 2001-2006 Gregory P. Ward. All rights reserved.
+Copyright 2002-2006 Python Software Foundation. All rights reserved.
+"""
+
+# This code uses parts of optparse written by Gregory P. Ward and
+# maintained by the Python Software Foundation.
+# Copyright 2001-2006 Gregory P. Ward
+# Copyright 2002-2006 Python Software Foundation
+from __future__ import annotations
+
+import collections.abc as cabc
+import typing as t
+from collections import deque
+from gettext import gettext as _
+from gettext import ngettext
+
+from ._utils import FLAG_NEEDS_VALUE
+from ._utils import UNSET
+from .exceptions import BadArgumentUsage
+from .exceptions import BadOptionUsage
+from .exceptions import NoSuchOption
+from .exceptions import UsageError
+
+if t.TYPE_CHECKING:
+    from ._utils import T_FLAG_NEEDS_VALUE
+    from ._utils import T_UNSET
+    from .core import Argument as CoreArgument
+    from .core import Context
+    from .core import Option as CoreOption
+    from .core import Parameter as CoreParameter
+
+V = t.TypeVar("V")
+
+
+def _unpack_args(
+    args: cabc.Sequence[str], nargs_spec: cabc.Sequence[int]
+) -> tuple[cabc.Sequence[str | cabc.Sequence[str | None] | None], list[str]]:
+    """Given an iterable of arguments and an iterable of nargs specifications,
+    it returns a tuple with all the unpacked arguments at the first index
+    and all remaining arguments as the second.
+
+    The nargs specification is the number of arguments that should be consumed
+    or `-1` to indicate that this position should eat up all the remainders.
+
+    Missing items are filled with ``UNSET``.
+    """
+    args = deque(args)
+    nargs_spec = deque(nargs_spec)
+    rv: list[str | tuple[str | T_UNSET, ...] | T_UNSET] = []
+    spos: int | None = None
+
+    def _fetch(c: deque[V]) -> V | T_UNSET:
+        try:
+            if spos is None:
+                return c.popleft()
+            else:
+                return c.pop()
+        except IndexError:
+            return UNSET
+
+    while nargs_spec:
+        nargs = _fetch(nargs_spec)
+
+        if nargs is None:
+            continue
+
+        if nargs == 1:
+            rv.append(_fetch(args))  # type: ignore[arg-type]
+        elif nargs > 1:
+            x = [_fetch(args) for _ in range(nargs)]
+
+            # If we're reversed, we're pulling in the arguments in reverse,
+            # so we need to turn them around.
+            if spos is not None:
+                x.reverse()
+
+            rv.append(tuple(x))
+        elif nargs < 0:
+            if spos is not None:
+                raise TypeError("Cannot have two nargs < 0")
+
+            spos = len(rv)
+            rv.append(UNSET)
+
+    # spos is the position of the wildcard (star).  If it's not `None`,
+    # we fill it with the remainder.
+    if spos is not None:
+        rv[spos] = tuple(args)
+        args = []
+        rv[spos + 1 :] = reversed(rv[spos + 1 :])
+
+    return tuple(rv), list(args)
+
+
+def _split_opt(opt: str) -> tuple[str, str]:
+    first = opt[:1]
+    if first.isalnum():
+        return "", opt
+    if opt[1:2] == first:
+        return opt[:2], opt[2:]
+    return first, opt[1:]
+
+
+def _normalize_opt(opt: str, ctx: Context | None) -> str:
+    if ctx is None or ctx.token_normalize_func is None:
+        return opt
+    prefix, opt = _split_opt(opt)
+    return f"{prefix}{ctx.token_normalize_func(opt)}"
+
+
+class _Option:
+    def __init__(
+        self,
+        obj: CoreOption,
+        opts: cabc.Sequence[str],
+        dest: str | None,
+        action: str | None = None,
+        nargs: int = 1,
+        const: t.Any | None = None,
+    ):
+        self._short_opts = []
+        self._long_opts = []
+        self.prefixes: set[str] = set()
+
+        for opt in opts:
+            prefix, value = _split_opt(opt)
+            if not prefix:
+                raise ValueError(f"Invalid start character for option ({opt})")
+            self.prefixes.add(prefix[0])
+            if len(prefix) == 1 and len(value) == 1:
+                self._short_opts.append(opt)
+            else:
+                self._long_opts.append(opt)
+                self.prefixes.add(prefix)
+
+        if action is None:
+            action = "store"
+
+        self.dest = dest
+        self.action = action
+        self.nargs = nargs
+        self.const = const
+        self.obj = obj
+
+    @property
+    def takes_value(self) -> bool:
+        return self.action in ("store", "append")
+
+    def process(self, value: t.Any, state: _ParsingState) -> None:
+        if self.action == "store":
+            state.opts[self.dest] = value  # type: ignore
+        elif self.action == "store_const":
+            state.opts[self.dest] = self.const  # type: ignore
+        elif self.action == "append":
+            state.opts.setdefault(self.dest, []).append(value)  # type: ignore
+        elif self.action == "append_const":
+            state.opts.setdefault(self.dest, []).append(self.const)  # type: ignore
+        elif self.action == "count":
+            state.opts[self.dest] = state.opts.get(self.dest, 0) + 1  # type: ignore
+        else:
+            raise ValueError(f"unknown action '{self.action}'")
+        state.order.append(self.obj)
+
+
+class _Argument:
+    def __init__(self, obj: CoreArgument, dest: str | None, nargs: int = 1):
+        self.dest = dest
+        self.nargs = nargs
+        self.obj = obj
+
+    def process(
+        self,
+        value: str | cabc.Sequence[str | None] | None | T_UNSET,
+        state: _ParsingState,
+    ) -> None:
+        if self.nargs > 1:
+            assert isinstance(value, cabc.Sequence)
+            holes = sum(1 for x in value if x is UNSET)
+            if holes == len(value):
+                value = UNSET
+            elif holes != 0:
+                raise BadArgumentUsage(
+                    _("Argument {name!r} takes {nargs} values.").format(
+                        name=self.dest, nargs=self.nargs
+                    )
+                )
+
+        # We failed to collect any argument value so we consider the argument as unset.
+        if value == ():
+            value = UNSET
+
+        state.opts[self.dest] = value  # type: ignore
+        state.order.append(self.obj)
+
+
+class _ParsingState:
+    def __init__(self, rargs: list[str]) -> None:
+        self.opts: dict[str, t.Any] = {}
+        self.largs: list[str] = []
+        self.rargs = rargs
+        self.order: list[CoreParameter] = []
+
+
+class _OptionParser:
+    """The option parser is an internal class that is ultimately used to
+    parse options and arguments.  It's modelled after optparse and brings
+    a similar but vastly simplified API.  It should generally not be used
+    directly as the high level Click classes wrap it for you.
+
+    It's not nearly as extensible as optparse or argparse as it does not
+    implement features that are implemented on a higher level (such as
+    types or defaults).
+
+    :param ctx: optionally the :class:`~click.Context` where this parser
+                should go with.
+
+    .. deprecated:: 8.2
+        Will be removed in Click 9.0.
+    """
+
+    def __init__(self, ctx: Context | None = None) -> None:
+        #: The :class:`~click.Context` for this parser.  This might be
+        #: `None` for some advanced use cases.
+        self.ctx = ctx
+        #: This controls how the parser deals with interspersed arguments.
+        #: If this is set to `False`, the parser will stop on the first
+        #: non-option.  Click uses this to implement nested subcommands
+        #: safely.
+        self.allow_interspersed_args: bool = True
+        #: This tells the parser how to deal with unknown options.  By
+        #: default it will error out (which is sensible), but there is a
+        #: second mode where it will ignore it and continue processing
+        #: after shifting all the unknown options into the resulting args.
+        self.ignore_unknown_options: bool = False
+
+        if ctx is not None:
+            self.allow_interspersed_args = ctx.allow_interspersed_args
+            self.ignore_unknown_options = ctx.ignore_unknown_options
+
+        self._short_opt: dict[str, _Option] = {}
+        self._long_opt: dict[str, _Option] = {}
+        self._opt_prefixes = {"-", "--"}
+        self._args: list[_Argument] = []
+
+    def add_option(
+        self,
+        obj: CoreOption,
+        opts: cabc.Sequence[str],
+        dest: str | None,
+        action: str | None = None,
+        nargs: int = 1,
+        const: t.Any | None = None,
+    ) -> None:
+        """Adds a new option named `dest` to the parser.  The destination
+        is not inferred (unlike with optparse) and needs to be explicitly
+        provided.  Action can be any of ``store``, ``store_const``,
+        ``append``, ``append_const`` or ``count``.
+
+        The `obj` can be used to identify the option in the order list
+        that is returned from the parser.
+        """
+        opts = [_normalize_opt(opt, self.ctx) for opt in opts]
+        option = _Option(obj, opts, dest, action=action, nargs=nargs, const=const)
+        self._opt_prefixes.update(option.prefixes)
+        for opt in option._short_opts:
+            self._short_opt[opt] = option
+        for opt in option._long_opts:
+            self._long_opt[opt] = option
+
+    def add_argument(self, obj: CoreArgument, dest: str | None, nargs: int = 1) -> None:
+        """Adds a positional argument named `dest` to the parser.
+
+        The `obj` can be used to identify the option in the order list
+        that is returned from the parser.
+        """
+        self._args.append(_Argument(obj, dest=dest, nargs=nargs))
+
+    def parse_args(
+        self, args: list[str]
+    ) -> tuple[dict[str, t.Any], list[str], list[CoreParameter]]:
+        """Parses positional arguments and returns ``(values, args, order)``
+        for the parsed options and arguments as well as the leftover
+        arguments if there are any.  The order is a list of objects as they
+        appear on the command line.  If arguments appear multiple times they
+        will be memorized multiple times as well.
+        """
+        state = _ParsingState(args)
+        try:
+            self._process_args_for_options(state)
+            self._process_args_for_args(state)
+        except UsageError:
+            if self.ctx is None or not self.ctx.resilient_parsing:
+                raise
+        return state.opts, state.largs, state.order
+
+    def _process_args_for_args(self, state: _ParsingState) -> None:
+        pargs, args = _unpack_args(
+            state.largs + state.rargs, [x.nargs for x in self._args]
+        )
+
+        for idx, arg in enumerate(self._args):
+            arg.process(pargs[idx], state)
+
+        state.largs = args
+        state.rargs = []
+
+    def _process_args_for_options(self, state: _ParsingState) -> None:
+        while state.rargs:
+            arg = state.rargs.pop(0)
+            arglen = len(arg)
+            # Double dashes always handled explicitly regardless of what
+            # prefixes are valid.
+            if arg == "--":
+                return
+            elif arg[:1] in self._opt_prefixes and arglen > 1:
+                self._process_opts(arg, state)
+            elif self.allow_interspersed_args:
+                state.largs.append(arg)
+            else:
+                state.rargs.insert(0, arg)
+                return
+
+        # Say this is the original argument list:
+        # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)]
+        #                            ^
+        # (we are about to process arg(i)).
+        #
+        # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of
+        # [arg0, ..., arg(i-1)] (any options and their arguments will have
+        # been removed from largs).
+        #
+        # The while loop will usually consume 1 or more arguments per pass.
+        # If it consumes 1 (eg. arg is an option that takes no arguments),
+        # then after _process_arg() is done the situation is:
+        #
+        #   largs = subset of [arg0, ..., arg(i)]
+        #   rargs = [arg(i+1), ..., arg(N-1)]
+        #
+        # If allow_interspersed_args is false, largs will always be
+        # *empty* -- still a subset of [arg0, ..., arg(i-1)], but
+        # not a very interesting subset!
+
+    def _match_long_opt(
+        self, opt: str, explicit_value: str | None, state: _ParsingState
+    ) -> None:
+        if opt not in self._long_opt:
+            from difflib import get_close_matches
+
+            possibilities = get_close_matches(opt, self._long_opt)
+            raise NoSuchOption(opt, possibilities=possibilities, ctx=self.ctx)
+
+        option = self._long_opt[opt]
+        if option.takes_value:
+            # At this point it's safe to modify rargs by injecting the
+            # explicit value, because no exception is raised in this
+            # branch.  This means that the inserted value will be fully
+            # consumed.
+            if explicit_value is not None:
+                state.rargs.insert(0, explicit_value)
+
+            value = self._get_value_from_state(opt, option, state)
+
+        elif explicit_value is not None:
+            raise BadOptionUsage(
+                opt, _("Option {name!r} does not take a value.").format(name=opt)
+            )
+
+        else:
+            value = UNSET
+
+        option.process(value, state)
+
+    def _match_short_opt(self, arg: str, state: _ParsingState) -> None:
+        stop = False
+        i = 1
+        prefix = arg[0]
+        unknown_options = []
+
+        for ch in arg[1:]:
+            opt = _normalize_opt(f"{prefix}{ch}", self.ctx)
+            option = self._short_opt.get(opt)
+            i += 1
+
+            if not option:
+                if self.ignore_unknown_options:
+                    unknown_options.append(ch)
+                    continue
+                raise NoSuchOption(opt, ctx=self.ctx)
+            if option.takes_value:
+                # Any characters left in arg?  Pretend they're the
+                # next arg, and stop consuming characters of arg.
+                if i < len(arg):
+                    state.rargs.insert(0, arg[i:])
+                    stop = True
+
+                value = self._get_value_from_state(opt, option, state)
+
+            else:
+                value = UNSET
+
+            option.process(value, state)
+
+            if stop:
+                break
+
+        # If we got any unknown options we recombine the string of the
+        # remaining options and re-attach the prefix, then report that
+        # to the state as new larg.  This way there is basic combinatorics
+        # that can be achieved while still ignoring unknown arguments.
+        if self.ignore_unknown_options and unknown_options:
+            state.largs.append(f"{prefix}{''.join(unknown_options)}")
+
+    def _get_value_from_state(
+        self, option_name: str, option: _Option, state: _ParsingState
+    ) -> str | cabc.Sequence[str] | T_FLAG_NEEDS_VALUE:
+        nargs = option.nargs
+
+        value: str | cabc.Sequence[str] | T_FLAG_NEEDS_VALUE
+
+        if len(state.rargs) < nargs:
+            if option.obj._flag_needs_value:
+                # Option allows omitting the value.
+                value = FLAG_NEEDS_VALUE
+            else:
+                raise BadOptionUsage(
+                    option_name,
+                    ngettext(
+                        "Option {name!r} requires an argument.",
+                        "Option {name!r} requires {nargs} arguments.",
+                        nargs,
+                    ).format(name=option_name, nargs=nargs),
+                )
+        elif nargs == 1:
+            next_rarg = state.rargs[0]
+
+            if (
+                option.obj._flag_needs_value
+                and isinstance(next_rarg, str)
+                and next_rarg[:1] in self._opt_prefixes
+                and len(next_rarg) > 1
+            ):
+                # The next arg looks like the start of an option, don't
+                # use it as the value if omitting the value is allowed.
+                value = FLAG_NEEDS_VALUE
+            else:
+                value = state.rargs.pop(0)
+        else:
+            value = tuple(state.rargs[:nargs])
+            del state.rargs[:nargs]
+
+        return value
+
+    def _process_opts(self, arg: str, state: _ParsingState) -> None:
+        explicit_value = None
+        # Long option handling happens in two parts.  The first part is
+        # supporting explicitly attached values.  In any case, we will try
+        # to long match the option first.
+        if "=" in arg:
+            long_opt, explicit_value = arg.split("=", 1)
+        else:
+            long_opt = arg
+        norm_long_opt = _normalize_opt(long_opt, self.ctx)
+
+        # At this point we will match the (assumed) long option through
+        # the long option matching code.  Note that this allows options
+        # like "-foo" to be matched as long options.
+        try:
+            self._match_long_opt(norm_long_opt, explicit_value, state)
+        except NoSuchOption:
+            # At this point the long option matching failed, and we need
+            # to try with short options.  However there is a special rule
+            # which says, that if we have a two character options prefix
+            # (applies to "--foo" for instance), we do not dispatch to the
+            # short option code and will instead raise the no option
+            # error.
+            if arg[:2] not in self._opt_prefixes:
+                self._match_short_opt(arg, state)
+                return
+
+            if not self.ignore_unknown_options:
+                raise
+
+            state.largs.append(arg)
+
+
+def __getattr__(name: str) -> object:
+    import warnings
+
+    if name in {
+        "OptionParser",
+        "Argument",
+        "Option",
+        "split_opt",
+        "normalize_opt",
+        "ParsingState",
+    }:
+        warnings.warn(
+            f"'parser.{name}' is deprecated and will be removed in Click 9.0."
+            " The old parser is available in 'optparse'.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return globals()[f"_{name}"]
+
+    if name == "split_arg_string":
+        from .shell_completion import split_arg_string
+
+        warnings.warn(
+            "Importing 'parser.split_arg_string' is deprecated, it will only be"
+            " available in 'shell_completion' in Click 9.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return split_arg_string
+
+    raise AttributeError(name)

python/py/Lib/site-packages/click/shell_completion.py

@@ -0,0 +1,667 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+import os
+import re
+import typing as t
+from gettext import gettext as _
+
+from .core import Argument
+from .core import Command
+from .core import Context
+from .core import Group
+from .core import Option
+from .core import Parameter
+from .core import ParameterSource
+from .utils import echo
+
+
+def shell_complete(
+    cli: Command,
+    ctx_args: cabc.MutableMapping[str, t.Any],
+    prog_name: str,
+    complete_var: str,
+    instruction: str,
+) -> int:
+    """Perform shell completion for the given CLI program.
+
+    :param cli: Command being called.
+    :param ctx_args: Extra arguments to pass to
+        ``cli.make_context``.
+    :param prog_name: Name of the executable in the shell.
+    :param complete_var: Name of the environment variable that holds
+        the completion instruction.
+    :param instruction: Value of ``complete_var`` with the completion
+        instruction and shell, in the form ``instruction_shell``.
+    :return: Status code to exit with.
+    """
+    shell, _, instruction = instruction.partition("_")
+    comp_cls = get_completion_class(shell)
+
+    if comp_cls is None:
+        return 1
+
+    comp = comp_cls(cli, ctx_args, prog_name, complete_var)
+
+    if instruction == "source":
+        echo(comp.source())
+        return 0
+
+    if instruction == "complete":
+        echo(comp.complete())
+        return 0
+
+    return 1
+
+
+class CompletionItem:
+    """Represents a completion value and metadata about the value. The
+    default metadata is ``type`` to indicate special shell handling,
+    and ``help`` if a shell supports showing a help string next to the
+    value.
+
+    Arbitrary parameters can be passed when creating the object, and
+    accessed using ``item.attr``. If an attribute wasn't passed,
+    accessing it returns ``None``.
+
+    :param value: The completion suggestion.
+    :param type: Tells the shell script to provide special completion
+        support for the type. Click uses ``"dir"`` and ``"file"``.
+    :param help: String shown next to the value if supported.
+    :param kwargs: Arbitrary metadata. The built-in implementations
+        don't use this, but custom type completions paired with custom
+        shell support could use it.
+    """
+
+    __slots__ = ("value", "type", "help", "_info")
+
+    def __init__(
+        self,
+        value: t.Any,
+        type: str = "plain",
+        help: str | None = None,
+        **kwargs: t.Any,
+    ) -> None:
+        self.value: t.Any = value
+        self.type: str = type
+        self.help: str | None = help
+        self._info = kwargs
+
+    def __getattr__(self, name: str) -> t.Any:
+        return self._info.get(name)
+
+
+# Only Bash >= 4.4 has the nosort option.
+_SOURCE_BASH = """\
+%(complete_func)s() {
+    local IFS=$'\\n'
+    local response
+
+    response=$(env COMP_WORDS="${COMP_WORDS[*]}" COMP_CWORD=$COMP_CWORD \
+%(complete_var)s=bash_complete $1)
+
+    for completion in $response; do
+        IFS=',' read type value <<< "$completion"
+
+        if [[ $type == 'dir' ]]; then
+            COMPREPLY=()
+            compopt -o dirnames
+        elif [[ $type == 'file' ]]; then
+            COMPREPLY=()
+            compopt -o default
+        elif [[ $type == 'plain' ]]; then
+            COMPREPLY+=($value)
+        fi
+    done
+
+    return 0
+}
+
+%(complete_func)s_setup() {
+    complete -o nosort -F %(complete_func)s %(prog_name)s
+}
+
+%(complete_func)s_setup;
+"""
+
+# See ZshComplete.format_completion below, and issue #2703, before
+# changing this script.
+#
+# (TL;DR: _describe is picky about the format, but this Zsh script snippet
+# is already widely deployed.  So freeze this script, and use clever-ish
+# handling of colons in ZshComplet.format_completion.)
+_SOURCE_ZSH = """\
+#compdef %(prog_name)s
+
+%(complete_func)s() {
+    local -a completions
+    local -a completions_with_descriptions
+    local -a response
+    (( ! $+commands[%(prog_name)s] )) && return 1
+
+    response=("${(@f)$(env COMP_WORDS="${words[*]}" COMP_CWORD=$((CURRENT-1)) \
+%(complete_var)s=zsh_complete %(prog_name)s)}")
+
+    for type key descr in ${response}; do
+        if [[ "$type" == "plain" ]]; then
+            if [[ "$descr" == "_" ]]; then
+                completions+=("$key")
+            else
+                completions_with_descriptions+=("$key":"$descr")
+            fi
+        elif [[ "$type" == "dir" ]]; then
+            _path_files -/
+        elif [[ "$type" == "file" ]]; then
+            _path_files -f
+        fi
+    done
+
+    if [ -n "$completions_with_descriptions" ]; then
+        _describe -V unsorted completions_with_descriptions -U
+    fi
+
+    if [ -n "$completions" ]; then
+        compadd -U -V unsorted -a completions
+    fi
+}
+
+if [[ $zsh_eval_context[-1] == loadautofunc ]]; then
+    # autoload from fpath, call function directly
+    %(complete_func)s "$@"
+else
+    # eval/source/. command, register function for later
+    compdef %(complete_func)s %(prog_name)s
+fi
+"""
+
+_SOURCE_FISH = """\
+function %(complete_func)s;
+    set -l response (env %(complete_var)s=fish_complete COMP_WORDS=(commandline -cp) \
+COMP_CWORD=(commandline -t) %(prog_name)s);
+
+    for completion in $response;
+        set -l metadata (string split "," $completion);
+
+        if test $metadata[1] = "dir";
+            __fish_complete_directories $metadata[2];
+        else if test $metadata[1] = "file";
+            __fish_complete_path $metadata[2];
+        else if test $metadata[1] = "plain";
+            echo $metadata[2];
+        end;
+    end;
+end;
+
+complete --no-files --command %(prog_name)s --arguments \
+"(%(complete_func)s)";
+"""
+
+
+class ShellComplete:
+    """Base class for providing shell completion support. A subclass for
+    a given shell will override attributes and methods to implement the
+    completion instructions (``source`` and ``complete``).
+
+    :param cli: Command being called.
+    :param prog_name: Name of the executable in the shell.
+    :param complete_var: Name of the environment variable that holds
+        the completion instruction.
+
+    .. versionadded:: 8.0
+    """
+
+    name: t.ClassVar[str]
+    """Name to register the shell as with :func:`add_completion_class`.
+    This is used in completion instructions (``{name}_source`` and
+    ``{name}_complete``).
+    """
+
+    source_template: t.ClassVar[str]
+    """Completion script template formatted by :meth:`source`. This must
+    be provided by subclasses.
+    """
+
+    def __init__(
+        self,
+        cli: Command,
+        ctx_args: cabc.MutableMapping[str, t.Any],
+        prog_name: str,
+        complete_var: str,
+    ) -> None:
+        self.cli = cli
+        self.ctx_args = ctx_args
+        self.prog_name = prog_name
+        self.complete_var = complete_var
+
+    @property
+    def func_name(self) -> str:
+        """The name of the shell function defined by the completion
+        script.
+        """
+        safe_name = re.sub(r"\W*", "", self.prog_name.replace("-", "_"), flags=re.ASCII)
+        return f"_{safe_name}_completion"
+
+    def source_vars(self) -> dict[str, t.Any]:
+        """Vars for formatting :attr:`source_template`.
+
+        By default this provides ``complete_func``, ``complete_var``,
+        and ``prog_name``.
+        """
+        return {
+            "complete_func": self.func_name,
+            "complete_var": self.complete_var,
+            "prog_name": self.prog_name,
+        }
+
+    def source(self) -> str:
+        """Produce the shell script that defines the completion
+        function. By default this ``%``-style formats
+        :attr:`source_template` with the dict returned by
+        :meth:`source_vars`.
+        """
+        return self.source_template % self.source_vars()
+
+    def get_completion_args(self) -> tuple[list[str], str]:
+        """Use the env vars defined by the shell script to return a
+        tuple of ``args, incomplete``. This must be implemented by
+        subclasses.
+        """
+        raise NotImplementedError
+
+    def get_completions(self, args: list[str], incomplete: str) -> list[CompletionItem]:
+        """Determine the context and last complete command or parameter
+        from the complete args. Call that object's ``shell_complete``
+        method to get the completions for the incomplete value.
+
+        :param args: List of complete args before the incomplete value.
+        :param incomplete: Value being completed. May be empty.
+        """
+        ctx = _resolve_context(self.cli, self.ctx_args, self.prog_name, args)
+        obj, incomplete = _resolve_incomplete(ctx, args, incomplete)
+        return obj.shell_complete(ctx, incomplete)
+
+    def format_completion(self, item: CompletionItem) -> str:
+        """Format a completion item into the form recognized by the
+        shell script. This must be implemented by subclasses.
+
+        :param item: Completion item to format.
+        """
+        raise NotImplementedError
+
+    def complete(self) -> str:
+        """Produce the completion data to send back to the shell.
+
+        By default this calls :meth:`get_completion_args`, gets the
+        completions, then calls :meth:`format_completion` for each
+        completion.
+        """
+        args, incomplete = self.get_completion_args()
+        completions = self.get_completions(args, incomplete)
+        out = [self.format_completion(item) for item in completions]
+        return "\n".join(out)
+
+
+class BashComplete(ShellComplete):
+    """Shell completion for Bash."""
+
+    name = "bash"
+    source_template = _SOURCE_BASH
+
+    @staticmethod
+    def _check_version() -> None:
+        import shutil
+        import subprocess
+
+        bash_exe = shutil.which("bash")
+
+        if bash_exe is None:
+            match = None
+        else:
+            output = subprocess.run(
+                [bash_exe, "--norc", "-c", 'echo "${BASH_VERSION}"'],
+                stdout=subprocess.PIPE,
+            )
+            match = re.search(r"^(\d+)\.(\d+)\.\d+", output.stdout.decode())
+
+        if match is not None:
+            major, minor = match.groups()
+
+            if major < "4" or major == "4" and minor < "4":
+                echo(
+                    _(
+                        "Shell completion is not supported for Bash"
+                        " versions older than 4.4."
+                    ),
+                    err=True,
+                )
+        else:
+            echo(
+                _("Couldn't detect Bash version, shell completion is not supported."),
+                err=True,
+            )
+
+    def source(self) -> str:
+        self._check_version()
+        return super().source()
+
+    def get_completion_args(self) -> tuple[list[str], str]:
+        cwords = split_arg_string(os.environ["COMP_WORDS"])
+        cword = int(os.environ["COMP_CWORD"])
+        args = cwords[1:cword]
+
+        try:
+            incomplete = cwords[cword]
+        except IndexError:
+            incomplete = ""
+
+        return args, incomplete
+
+    def format_completion(self, item: CompletionItem) -> str:
+        return f"{item.type},{item.value}"
+
+
+class ZshComplete(ShellComplete):
+    """Shell completion for Zsh."""
+
+    name = "zsh"
+    source_template = _SOURCE_ZSH
+
+    def get_completion_args(self) -> tuple[list[str], str]:
+        cwords = split_arg_string(os.environ["COMP_WORDS"])
+        cword = int(os.environ["COMP_CWORD"])
+        args = cwords[1:cword]
+
+        try:
+            incomplete = cwords[cword]
+        except IndexError:
+            incomplete = ""
+
+        return args, incomplete
+
+    def format_completion(self, item: CompletionItem) -> str:
+        help_ = item.help or "_"
+        # The zsh completion script uses `_describe` on items with help
+        # texts (which splits the item help from the item value at the
+        # first unescaped colon) and `compadd` on items without help
+        # text (which uses the item value as-is and does not support
+        # colon escaping).  So escape colons in the item value if and
+        # only if the item help is not the sentinel "_" value, as used
+        # by the completion script.
+        #
+        # (The zsh completion script is potentially widely deployed, and
+        # thus harder to fix than this method.)
+        #
+        # See issue #1812 and issue #2703 for further context.
+        value = item.value.replace(":", r"\:") if help_ != "_" else item.value
+        return f"{item.type}\n{value}\n{help_}"
+
+
+class FishComplete(ShellComplete):
+    """Shell completion for Fish."""
+
+    name = "fish"
+    source_template = _SOURCE_FISH
+
+    def get_completion_args(self) -> tuple[list[str], str]:
+        cwords = split_arg_string(os.environ["COMP_WORDS"])
+        incomplete = os.environ["COMP_CWORD"]
+        if incomplete:
+            incomplete = split_arg_string(incomplete)[0]
+        args = cwords[1:]
+
+        # Fish stores the partial word in both COMP_WORDS and
+        # COMP_CWORD, remove it from complete args.
+        if incomplete and args and args[-1] == incomplete:
+            args.pop()
+
+        return args, incomplete
+
+    def format_completion(self, item: CompletionItem) -> str:
+        if item.help:
+            return f"{item.type},{item.value}\t{item.help}"
+
+        return f"{item.type},{item.value}"
+
+
+ShellCompleteType = t.TypeVar("ShellCompleteType", bound="type[ShellComplete]")
+
+
+_available_shells: dict[str, type[ShellComplete]] = {
+    "bash": BashComplete,
+    "fish": FishComplete,
+    "zsh": ZshComplete,
+}
+
+
+def add_completion_class(
+    cls: ShellCompleteType, name: str | None = None
+) -> ShellCompleteType:
+    """Register a :class:`ShellComplete` subclass under the given name.
+    The name will be provided by the completion instruction environment
+    variable during completion.
+
+    :param cls: The completion class that will handle completion for the
+        shell.
+    :param name: Name to register the class under. Defaults to the
+        class's ``name`` attribute.
+    """
+    if name is None:
+        name = cls.name
+
+    _available_shells[name] = cls
+
+    return cls
+
+
+def get_completion_class(shell: str) -> type[ShellComplete] | None:
+    """Look up a registered :class:`ShellComplete` subclass by the name
+    provided by the completion instruction environment variable. If the
+    name isn't registered, returns ``None``.
+
+    :param shell: Name the class is registered under.
+    """
+    return _available_shells.get(shell)
+
+
+def split_arg_string(string: str) -> list[str]:
+    """Split an argument string as with :func:`shlex.split`, but don't
+    fail if the string is incomplete. Ignores a missing closing quote or
+    incomplete escape sequence and uses the partial token as-is.
+
+    .. code-block:: python
+
+        split_arg_string("example 'my file")
+        ["example", "my file"]
+
+        split_arg_string("example my\\")
+        ["example", "my"]
+
+    :param string: String to split.
+
+    .. versionchanged:: 8.2
+        Moved to ``shell_completion`` from ``parser``.
+    """
+    import shlex
+
+    lex = shlex.shlex(string, posix=True)
+    lex.whitespace_split = True
+    lex.commenters = ""
+    out = []
+
+    try:
+        for token in lex:
+            out.append(token)
+    except ValueError:
+        # Raised when end-of-string is reached in an invalid state. Use
+        # the partial token as-is. The quote or escape character is in
+        # lex.state, not lex.token.
+        out.append(lex.token)
+
+    return out
+
+
+def _is_incomplete_argument(ctx: Context, param: Parameter) -> bool:
+    """Determine if the given parameter is an argument that can still
+    accept values.
+
+    :param ctx: Invocation context for the command represented by the
+        parsed complete args.
+    :param param: Argument object being checked.
+    """
+    if not isinstance(param, Argument):
+        return False
+
+    assert param.name is not None
+    # Will be None if expose_value is False.
+    value = ctx.params.get(param.name)
+    return (
+        param.nargs == -1
+        or ctx.get_parameter_source(param.name) is not ParameterSource.COMMANDLINE
+        or (
+            param.nargs > 1
+            and isinstance(value, (tuple, list))
+            and len(value) < param.nargs
+        )
+    )
+
+
+def _start_of_option(ctx: Context, value: str) -> bool:
+    """Check if the value looks like the start of an option."""
+    if not value:
+        return False
+
+    c = value[0]
+    return c in ctx._opt_prefixes
+
+
+def _is_incomplete_option(ctx: Context, args: list[str], param: Parameter) -> bool:
+    """Determine if the given parameter is an option that needs a value.
+
+    :param args: List of complete args before the incomplete value.
+    :param param: Option object being checked.
+    """
+    if not isinstance(param, Option):
+        return False
+
+    if param.is_flag or param.count:
+        return False
+
+    last_option = None
+
+    for index, arg in enumerate(reversed(args)):
+        if index + 1 > param.nargs:
+            break
+
+        if _start_of_option(ctx, arg):
+            last_option = arg
+            break
+
+    return last_option is not None and last_option in param.opts
+
+
+def _resolve_context(
+    cli: Command,
+    ctx_args: cabc.MutableMapping[str, t.Any],
+    prog_name: str,
+    args: list[str],
+) -> Context:
+    """Produce the context hierarchy starting with the command and
+    traversing the complete arguments. This only follows the commands,
+    it doesn't trigger input prompts or callbacks.
+
+    :param cli: Command being called.
+    :param prog_name: Name of the executable in the shell.
+    :param args: List of complete args before the incomplete value.
+    """
+    ctx_args["resilient_parsing"] = True
+    with cli.make_context(prog_name, args.copy(), **ctx_args) as ctx:
+        args = ctx._protected_args + ctx.args
+
+        while args:
+            command = ctx.command
+
+            if isinstance(command, Group):
+                if not command.chain:
+                    name, cmd, args = command.resolve_command(ctx, args)
+
+                    if cmd is None:
+                        return ctx
+
+                    with cmd.make_context(
+                        name, args, parent=ctx, resilient_parsing=True
+                    ) as sub_ctx:
+                        ctx = sub_ctx
+                        args = ctx._protected_args + ctx.args
+                else:
+                    sub_ctx = ctx
+
+                    while args:
+                        name, cmd, args = command.resolve_command(ctx, args)
+
+                        if cmd is None:
+                            return ctx
+
+                        with cmd.make_context(
+                            name,
+                            args,
+                            parent=ctx,
+                            allow_extra_args=True,
+                            allow_interspersed_args=False,
+                            resilient_parsing=True,
+                        ) as sub_sub_ctx:
+                            sub_ctx = sub_sub_ctx
+                            args = sub_ctx.args
+
+                    ctx = sub_ctx
+                    args = [*sub_ctx._protected_args, *sub_ctx.args]
+            else:
+                break
+
+    return ctx
+
+
+def _resolve_incomplete(
+    ctx: Context, args: list[str], incomplete: str
+) -> tuple[Command | Parameter, str]:
+    """Find the Click object that will handle the completion of the
+    incomplete value. Return the object and the incomplete value.
+
+    :param ctx: Invocation context for the command represented by
+        the parsed complete args.
+    :param args: List of complete args before the incomplete value.
+    :param incomplete: Value being completed. May be empty.
+    """
+    # Different shells treat an "=" between a long option name and
+    # value differently. Might keep the value joined, return the "="
+    # as a separate item, or return the split name and value. Always
+    # split and discard the "=" to make completion easier.
+    if incomplete == "=":
+        incomplete = ""
+    elif "=" in incomplete and _start_of_option(ctx, incomplete):
+        name, _, incomplete = incomplete.partition("=")
+        args.append(name)
+
+    # The "--" marker tells Click to stop treating values as options
+    # even if they start with the option character. If it hasn't been
+    # given and the incomplete arg looks like an option, the current
+    # command will provide option name completions.
+    if "--" not in args and _start_of_option(ctx, incomplete):
+        return ctx.command, incomplete
+
+    params = ctx.command.get_params(ctx)
+
+    # If the last complete arg is an option name with an incomplete
+    # value, the option will provide value completions.
+    for param in params:
+        if _is_incomplete_option(ctx, args, param):
+            return param, incomplete
+
+    # It's not an option name or value. The first argument without a
+    # parsed value will provide value completions.
+    for param in params:
+        if _is_incomplete_argument(ctx, param):
+            return param, incomplete
+
+    # There were no unparsed arguments, the command may be a group that
+    # will provide command name completions.
+    return ctx.command, incomplete

python/py/Lib/site-packages/click/termui.py

@@ -0,0 +1,883 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+import inspect
+import io
+import itertools
+import sys
+import typing as t
+from contextlib import AbstractContextManager
+from gettext import gettext as _
+
+from ._compat import isatty
+from ._compat import strip_ansi
+from .exceptions import Abort
+from .exceptions import UsageError
+from .globals import resolve_color_default
+from .types import Choice
+from .types import convert_type
+from .types import ParamType
+from .utils import echo
+from .utils import LazyFile
+
+if t.TYPE_CHECKING:
+    from ._termui_impl import ProgressBar
+
+V = t.TypeVar("V")
+
+# The prompt functions to use.  The doc tools currently override these
+# functions to customize how they work.
+visible_prompt_func: t.Callable[[str], str] = input
+
+_ansi_colors = {
+    "black": 30,
+    "red": 31,
+    "green": 32,
+    "yellow": 33,
+    "blue": 34,
+    "magenta": 35,
+    "cyan": 36,
+    "white": 37,
+    "reset": 39,
+    "bright_black": 90,
+    "bright_red": 91,
+    "bright_green": 92,
+    "bright_yellow": 93,
+    "bright_blue": 94,
+    "bright_magenta": 95,
+    "bright_cyan": 96,
+    "bright_white": 97,
+}
+_ansi_reset_all = "\033[0m"
+
+
+def hidden_prompt_func(prompt: str) -> str:
+    import getpass
+
+    return getpass.getpass(prompt)
+
+
+def _build_prompt(
+    text: str,
+    suffix: str,
+    show_default: bool = False,
+    default: t.Any | None = None,
+    show_choices: bool = True,
+    type: ParamType | None = None,
+) -> str:
+    prompt = text
+    if type is not None and show_choices and isinstance(type, Choice):
+        prompt += f" ({', '.join(map(str, type.choices))})"
+    if default is not None and show_default:
+        prompt = f"{prompt} [{_format_default(default)}]"
+    return f"{prompt}{suffix}"
+
+
+def _format_default(default: t.Any) -> t.Any:
+    if isinstance(default, (io.IOBase, LazyFile)) and hasattr(default, "name"):
+        return default.name
+
+    return default
+
+
+def prompt(
+    text: str,
+    default: t.Any | None = None,
+    hide_input: bool = False,
+    confirmation_prompt: bool | str = False,
+    type: ParamType | t.Any | None = None,
+    value_proc: t.Callable[[str], t.Any] | None = None,
+    prompt_suffix: str = ": ",
+    show_default: bool = True,
+    err: bool = False,
+    show_choices: bool = True,
+) -> t.Any:
+    """Prompts a user for input.  This is a convenience function that can
+    be used to prompt a user for input later.
+
+    If the user aborts the input by sending an interrupt signal, this
+    function will catch it and raise a :exc:`Abort` exception.
+
+    :param text: the text to show for the prompt.
+    :param default: the default value to use if no input happens.  If this
+                    is not given it will prompt until it's aborted.
+    :param hide_input: if this is set to true then the input value will
+                       be hidden.
+    :param confirmation_prompt: Prompt a second time to confirm the
+        value. Can be set to a string instead of ``True`` to customize
+        the message.
+    :param type: the type to use to check the value against.
+    :param value_proc: if this parameter is provided it's a function that
+                       is invoked instead of the type conversion to
+                       convert a value.
+    :param prompt_suffix: a suffix that should be added to the prompt.
+    :param show_default: shows or hides the default value in the prompt.
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    :param show_choices: Show or hide choices if the passed type is a Choice.
+                         For example if type is a Choice of either day or week,
+                         show_choices is true and text is "Group by" then the
+                         prompt will be "Group by (day, week): ".
+
+    .. versionchanged:: 8.3.1
+        A space is no longer appended to the prompt.
+
+    .. versionadded:: 8.0
+        ``confirmation_prompt`` can be a custom string.
+
+    .. versionadded:: 7.0
+        Added the ``show_choices`` parameter.
+
+    .. versionadded:: 6.0
+        Added unicode support for cmd.exe on Windows.
+
+    .. versionadded:: 4.0
+        Added the `err` parameter.
+
+    """
+
+    def prompt_func(text: str) -> str:
+        f = hidden_prompt_func if hide_input else visible_prompt_func
+        try:
+            # Write the prompt separately so that we get nice
+            # coloring through colorama on Windows
+            echo(text[:-1], nl=False, err=err)
+            # Echo the last character to stdout to work around an issue where
+            # readline causes backspace to clear the whole line.
+            return f(text[-1:])
+        except (KeyboardInterrupt, EOFError):
+            # getpass doesn't print a newline if the user aborts input with ^C.
+            # Allegedly this behavior is inherited from getpass(3).
+            # A doc bug has been filed at https://bugs.python.org/issue24711
+            if hide_input:
+                echo(None, err=err)
+            raise Abort() from None
+
+    if value_proc is None:
+        value_proc = convert_type(type, default)
+
+    prompt = _build_prompt(
+        text, prompt_suffix, show_default, default, show_choices, type
+    )
+
+    if confirmation_prompt:
+        if confirmation_prompt is True:
+            confirmation_prompt = _("Repeat for confirmation")
+
+        confirmation_prompt = _build_prompt(confirmation_prompt, prompt_suffix)
+
+    while True:
+        while True:
+            value = prompt_func(prompt)
+            if value:
+                break
+            elif default is not None:
+                value = default
+                break
+        try:
+            result = value_proc(value)
+        except UsageError as e:
+            if hide_input:
+                echo(_("Error: The value you entered was invalid."), err=err)
+            else:
+                echo(_("Error: {e.message}").format(e=e), err=err)
+            continue
+        if not confirmation_prompt:
+            return result
+        while True:
+            value2 = prompt_func(confirmation_prompt)
+            is_empty = not value and not value2
+            if value2 or is_empty:
+                break
+        if value == value2:
+            return result
+        echo(_("Error: The two entered values do not match."), err=err)
+
+
+def confirm(
+    text: str,
+    default: bool | None = False,
+    abort: bool = False,
+    prompt_suffix: str = ": ",
+    show_default: bool = True,
+    err: bool = False,
+) -> bool:
+    """Prompts for confirmation (yes/no question).
+
+    If the user aborts the input by sending a interrupt signal this
+    function will catch it and raise a :exc:`Abort` exception.
+
+    :param text: the question to ask.
+    :param default: The default value to use when no input is given. If
+        ``None``, repeat until input is given.
+    :param abort: if this is set to `True` a negative answer aborts the
+                  exception by raising :exc:`Abort`.
+    :param prompt_suffix: a suffix that should be added to the prompt.
+    :param show_default: shows or hides the default value in the prompt.
+    :param err: if set to true the file defaults to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+
+    .. versionchanged:: 8.3.1
+        A space is no longer appended to the prompt.
+
+    .. versionchanged:: 8.0
+        Repeat until input is given if ``default`` is ``None``.
+
+    .. versionadded:: 4.0
+        Added the ``err`` parameter.
+    """
+    prompt = _build_prompt(
+        text,
+        prompt_suffix,
+        show_default,
+        "y/n" if default is None else ("Y/n" if default else "y/N"),
+    )
+
+    while True:
+        try:
+            # Write the prompt separately so that we get nice
+            # coloring through colorama on Windows
+            echo(prompt[:-1], nl=False, err=err)
+            # Echo the last character to stdout to work around an issue where
+            # readline causes backspace to clear the whole line.
+            value = visible_prompt_func(prompt[-1:]).lower().strip()
+        except (KeyboardInterrupt, EOFError):
+            raise Abort() from None
+        if value in ("y", "yes"):
+            rv = True
+        elif value in ("n", "no"):
+            rv = False
+        elif default is not None and value == "":
+            rv = default
+        else:
+            echo(_("Error: invalid input"), err=err)
+            continue
+        break
+    if abort and not rv:
+        raise Abort()
+    return rv
+
+
+def echo_via_pager(
+    text_or_generator: cabc.Iterable[str] | t.Callable[[], cabc.Iterable[str]] | str,
+    color: bool | None = None,
+) -> None:
+    """This function takes a text and shows it via an environment specific
+    pager on stdout.
+
+    .. versionchanged:: 3.0
+       Added the `color` flag.
+
+    :param text_or_generator: the text to page, or alternatively, a
+                              generator emitting the text to page.
+    :param color: controls if the pager supports ANSI colors or not.  The
+                  default is autodetection.
+    """
+    color = resolve_color_default(color)
+
+    if inspect.isgeneratorfunction(text_or_generator):
+        i = t.cast("t.Callable[[], cabc.Iterable[str]]", text_or_generator)()
+    elif isinstance(text_or_generator, str):
+        i = [text_or_generator]
+    else:
+        i = iter(t.cast("cabc.Iterable[str]", text_or_generator))
+
+    # convert every element of i to a text type if necessary
+    text_generator = (el if isinstance(el, str) else str(el) for el in i)
+
+    from ._termui_impl import pager
+
+    return pager(itertools.chain(text_generator, "\n"), color)
+
+
+@t.overload
+def progressbar(
+    *,
+    length: int,
+    label: str | None = None,
+    hidden: bool = False,
+    show_eta: bool = True,
+    show_percent: bool | None = None,
+    show_pos: bool = False,
+    fill_char: str = "#",
+    empty_char: str = "-",
+    bar_template: str = "%(label)s  [%(bar)s]  %(info)s",
+    info_sep: str = "  ",
+    width: int = 36,
+    file: t.TextIO | None = None,
+    color: bool | None = None,
+    update_min_steps: int = 1,
+) -> ProgressBar[int]: ...
+
+
+@t.overload
+def progressbar(
+    iterable: cabc.Iterable[V] | None = None,
+    length: int | None = None,
+    label: str | None = None,
+    hidden: bool = False,
+    show_eta: bool = True,
+    show_percent: bool | None = None,
+    show_pos: bool = False,
+    item_show_func: t.Callable[[V | None], str | None] | None = None,
+    fill_char: str = "#",
+    empty_char: str = "-",
+    bar_template: str = "%(label)s  [%(bar)s]  %(info)s",
+    info_sep: str = "  ",
+    width: int = 36,
+    file: t.TextIO | None = None,
+    color: bool | None = None,
+    update_min_steps: int = 1,
+) -> ProgressBar[V]: ...
+
+
+def progressbar(
+    iterable: cabc.Iterable[V] | None = None,
+    length: int | None = None,
+    label: str | None = None,
+    hidden: bool = False,
+    show_eta: bool = True,
+    show_percent: bool | None = None,
+    show_pos: bool = False,
+    item_show_func: t.Callable[[V | None], str | None] | None = None,
+    fill_char: str = "#",
+    empty_char: str = "-",
+    bar_template: str = "%(label)s  [%(bar)s]  %(info)s",
+    info_sep: str = "  ",
+    width: int = 36,
+    file: t.TextIO | None = None,
+    color: bool | None = None,
+    update_min_steps: int = 1,
+) -> ProgressBar[V]:
+    """This function creates an iterable context manager that can be used
+    to iterate over something while showing a progress bar.  It will
+    either iterate over the `iterable` or `length` items (that are counted
+    up).  While iteration happens, this function will print a rendered
+    progress bar to the given `file` (defaults to stdout) and will attempt
+    to calculate remaining time and more.  By default, this progress bar
+    will not be rendered if the file is not a terminal.
+
+    The context manager creates the progress bar.  When the context
+    manager is entered the progress bar is already created.  With every
+    iteration over the progress bar, the iterable passed to the bar is
+    advanced and the bar is updated.  When the context manager exits,
+    a newline is printed and the progress bar is finalized on screen.
+
+    Note: The progress bar is currently designed for use cases where the
+    total progress can be expected to take at least several seconds.
+    Because of this, the ProgressBar class object won't display
+    progress that is considered too fast, and progress where the time
+    between steps is less than a second.
+
+    No printing must happen or the progress bar will be unintentionally
+    destroyed.
+
+    Example usage::
+
+        with progressbar(items) as bar:
+            for item in bar:
+                do_something_with(item)
+
+    Alternatively, if no iterable is specified, one can manually update the
+    progress bar through the `update()` method instead of directly
+    iterating over the progress bar.  The update method accepts the number
+    of steps to increment the bar with::
+
+        with progressbar(length=chunks.total_bytes) as bar:
+            for chunk in chunks:
+                process_chunk(chunk)
+                bar.update(chunks.bytes)
+
+    The ``update()`` method also takes an optional value specifying the
+    ``current_item`` at the new position. This is useful when used
+    together with ``item_show_func`` to customize the output for each
+    manual step::
+
+        with click.progressbar(
+            length=total_size,
+            label='Unzipping archive',
+            item_show_func=lambda a: a.filename
+        ) as bar:
+            for archive in zip_file:
+                archive.extract()
+                bar.update(archive.size, archive)
+
+    :param iterable: an iterable to iterate over.  If not provided the length
+                     is required.
+    :param length: the number of items to iterate over.  By default the
+                   progressbar will attempt to ask the iterator about its
+                   length, which might or might not work.  If an iterable is
+                   also provided this parameter can be used to override the
+                   length.  If an iterable is not provided the progress bar
+                   will iterate over a range of that length.
+    :param label: the label to show next to the progress bar.
+    :param hidden: hide the progressbar. Defaults to ``False``. When no tty is
+        detected, it will only print the progressbar label. Setting this to
+        ``False`` also disables that.
+    :param show_eta: enables or disables the estimated time display.  This is
+                     automatically disabled if the length cannot be
+                     determined.
+    :param show_percent: enables or disables the percentage display.  The
+                         default is `True` if the iterable has a length or
+                         `False` if not.
+    :param show_pos: enables or disables the absolute position display.  The
+                     default is `False`.
+    :param item_show_func: A function called with the current item which
+        can return a string to show next to the progress bar. If the
+        function returns ``None`` nothing is shown. The current item can
+        be ``None``, such as when entering and exiting the bar.
+    :param fill_char: the character to use to show the filled part of the
+                      progress bar.
+    :param empty_char: the character to use to show the non-filled part of
+                       the progress bar.
+    :param bar_template: the format string to use as template for the bar.
+                         The parameters in it are ``label`` for the label,
+                         ``bar`` for the progress bar and ``info`` for the
+                         info section.
+    :param info_sep: the separator between multiple info items (eta etc.)
+    :param width: the width of the progress bar in characters, 0 means full
+                  terminal width
+    :param file: The file to write to. If this is not a terminal then
+        only the label is printed.
+    :param color: controls if the terminal supports ANSI colors or not.  The
+                  default is autodetection.  This is only needed if ANSI
+                  codes are included anywhere in the progress bar output
+                  which is not the case by default.
+    :param update_min_steps: Render only when this many updates have
+        completed. This allows tuning for very fast iterators.
+
+    .. versionadded:: 8.2
+        The ``hidden`` argument.
+
+    .. versionchanged:: 8.0
+        Output is shown even if execution time is less than 0.5 seconds.
+
+    .. versionchanged:: 8.0
+        ``item_show_func`` shows the current item, not the previous one.
+
+    .. versionchanged:: 8.0
+        Labels are echoed if the output is not a TTY. Reverts a change
+        in 7.0 that removed all output.
+
+    .. versionadded:: 8.0
+       The ``update_min_steps`` parameter.
+
+    .. versionadded:: 4.0
+        The ``color`` parameter and ``update`` method.
+
+    .. versionadded:: 2.0
+    """
+    from ._termui_impl import ProgressBar
+
+    color = resolve_color_default(color)
+    return ProgressBar(
+        iterable=iterable,
+        length=length,
+        hidden=hidden,
+        show_eta=show_eta,
+        show_percent=show_percent,
+        show_pos=show_pos,
+        item_show_func=item_show_func,
+        fill_char=fill_char,
+        empty_char=empty_char,
+        bar_template=bar_template,
+        info_sep=info_sep,
+        file=file,
+        label=label,
+        width=width,
+        color=color,
+        update_min_steps=update_min_steps,
+    )
+
+
+def clear() -> None:
+    """Clears the terminal screen.  This will have the effect of clearing
+    the whole visible space of the terminal and moving the cursor to the
+    top left.  This does not do anything if not connected to a terminal.
+
+    .. versionadded:: 2.0
+    """
+    if not isatty(sys.stdout):
+        return
+
+    # ANSI escape \033[2J clears the screen, \033[1;1H moves the cursor
+    echo("\033[2J\033[1;1H", nl=False)
+
+
+def _interpret_color(color: int | tuple[int, int, int] | str, offset: int = 0) -> str:
+    if isinstance(color, int):
+        return f"{38 + offset};5;{color:d}"
+
+    if isinstance(color, (tuple, list)):
+        r, g, b = color
+        return f"{38 + offset};2;{r:d};{g:d};{b:d}"
+
+    return str(_ansi_colors[color] + offset)
+
+
+def style(
+    text: t.Any,
+    fg: int | tuple[int, int, int] | str | None = None,
+    bg: int | tuple[int, int, int] | str | None = None,
+    bold: bool | None = None,
+    dim: bool | None = None,
+    underline: bool | None = None,
+    overline: bool | None = None,
+    italic: bool | None = None,
+    blink: bool | None = None,
+    reverse: bool | None = None,
+    strikethrough: bool | None = None,
+    reset: bool = True,
+) -> str:
+    """Styles a text with ANSI styles and returns the new string.  By
+    default the styling is self contained which means that at the end
+    of the string a reset code is issued.  This can be prevented by
+    passing ``reset=False``.
+
+    Examples::
+
+        click.echo(click.style('Hello World!', fg='green'))
+        click.echo(click.style('ATTENTION!', blink=True))
+        click.echo(click.style('Some things', reverse=True, fg='cyan'))
+        click.echo(click.style('More colors', fg=(255, 12, 128), bg=117))
+
+    Supported color names:
+
+    * ``black`` (might be a gray)
+    * ``red``
+    * ``green``
+    * ``yellow`` (might be an orange)
+    * ``blue``
+    * ``magenta``
+    * ``cyan``
+    * ``white`` (might be light gray)
+    * ``bright_black``
+    * ``bright_red``
+    * ``bright_green``
+    * ``bright_yellow``
+    * ``bright_blue``
+    * ``bright_magenta``
+    * ``bright_cyan``
+    * ``bright_white``
+    * ``reset`` (reset the color code only)
+
+    If the terminal supports it, color may also be specified as:
+
+    -   An integer in the interval [0, 255]. The terminal must support
+        8-bit/256-color mode.
+    -   An RGB tuple of three integers in [0, 255]. The terminal must
+        support 24-bit/true-color mode.
+
+    See https://en.wikipedia.org/wiki/ANSI_color and
+    https://gist.github.com/XVilka/8346728 for more information.
+
+    :param text: the string to style with ansi codes.
+    :param fg: if provided this will become the foreground color.
+    :param bg: if provided this will become the background color.
+    :param bold: if provided this will enable or disable bold mode.
+    :param dim: if provided this will enable or disable dim mode.  This is
+                badly supported.
+    :param underline: if provided this will enable or disable underline.
+    :param overline: if provided this will enable or disable overline.
+    :param italic: if provided this will enable or disable italic.
+    :param blink: if provided this will enable or disable blinking.
+    :param reverse: if provided this will enable or disable inverse
+                    rendering (foreground becomes background and the
+                    other way round).
+    :param strikethrough: if provided this will enable or disable
+        striking through text.
+    :param reset: by default a reset-all code is added at the end of the
+                  string which means that styles do not carry over.  This
+                  can be disabled to compose styles.
+
+    .. versionchanged:: 8.0
+        A non-string ``message`` is converted to a string.
+
+    .. versionchanged:: 8.0
+       Added support for 256 and RGB color codes.
+
+    .. versionchanged:: 8.0
+        Added the ``strikethrough``, ``italic``, and ``overline``
+        parameters.
+
+    .. versionchanged:: 7.0
+        Added support for bright colors.
+
+    .. versionadded:: 2.0
+    """
+    if not isinstance(text, str):
+        text = str(text)
+
+    bits = []
+
+    if fg:
+        try:
+            bits.append(f"\033[{_interpret_color(fg)}m")
+        except KeyError:
+            raise TypeError(f"Unknown color {fg!r}") from None
+
+    if bg:
+        try:
+            bits.append(f"\033[{_interpret_color(bg, 10)}m")
+        except KeyError:
+            raise TypeError(f"Unknown color {bg!r}") from None
+
+    if bold is not None:
+        bits.append(f"\033[{1 if bold else 22}m")
+    if dim is not None:
+        bits.append(f"\033[{2 if dim else 22}m")
+    if underline is not None:
+        bits.append(f"\033[{4 if underline else 24}m")
+    if overline is not None:
+        bits.append(f"\033[{53 if overline else 55}m")
+    if italic is not None:
+        bits.append(f"\033[{3 if italic else 23}m")
+    if blink is not None:
+        bits.append(f"\033[{5 if blink else 25}m")
+    if reverse is not None:
+        bits.append(f"\033[{7 if reverse else 27}m")
+    if strikethrough is not None:
+        bits.append(f"\033[{9 if strikethrough else 29}m")
+    bits.append(text)
+    if reset:
+        bits.append(_ansi_reset_all)
+    return "".join(bits)
+
+
+def unstyle(text: str) -> str:
+    """Removes ANSI styling information from a string.  Usually it's not
+    necessary to use this function as Click's echo function will
+    automatically remove styling if necessary.
+
+    .. versionadded:: 2.0
+
+    :param text: the text to remove style information from.
+    """
+    return strip_ansi(text)
+
+
+def secho(
+    message: t.Any | None = None,
+    file: t.IO[t.AnyStr] | None = None,
+    nl: bool = True,
+    err: bool = False,
+    color: bool | None = None,
+    **styles: t.Any,
+) -> None:
+    """This function combines :func:`echo` and :func:`style` into one
+    call.  As such the following two calls are the same::
+
+        click.secho('Hello World!', fg='green')
+        click.echo(click.style('Hello World!', fg='green'))
+
+    All keyword arguments are forwarded to the underlying functions
+    depending on which one they go with.
+
+    Non-string types will be converted to :class:`str`. However,
+    :class:`bytes` are passed directly to :meth:`echo` without applying
+    style. If you want to style bytes that represent text, call
+    :meth:`bytes.decode` first.
+
+    .. versionchanged:: 8.0
+        A non-string ``message`` is converted to a string. Bytes are
+        passed through without style applied.
+
+    .. versionadded:: 2.0
+    """
+    if message is not None and not isinstance(message, (bytes, bytearray)):
+        message = style(message, **styles)
+
+    return echo(message, file=file, nl=nl, err=err, color=color)
+
+
+@t.overload
+def edit(
+    text: bytes | bytearray,
+    editor: str | None = None,
+    env: cabc.Mapping[str, str] | None = None,
+    require_save: bool = False,
+    extension: str = ".txt",
+) -> bytes | None: ...
+
+
+@t.overload
+def edit(
+    text: str,
+    editor: str | None = None,
+    env: cabc.Mapping[str, str] | None = None,
+    require_save: bool = True,
+    extension: str = ".txt",
+) -> str | None: ...
+
+
+@t.overload
+def edit(
+    text: None = None,
+    editor: str | None = None,
+    env: cabc.Mapping[str, str] | None = None,
+    require_save: bool = True,
+    extension: str = ".txt",
+    filename: str | cabc.Iterable[str] | None = None,
+) -> None: ...
+
+
+def edit(
+    text: str | bytes | bytearray | None = None,
+    editor: str | None = None,
+    env: cabc.Mapping[str, str] | None = None,
+    require_save: bool = True,
+    extension: str = ".txt",
+    filename: str | cabc.Iterable[str] | None = None,
+) -> str | bytes | bytearray | None:
+    r"""Edits the given text in the defined editor.  If an editor is given
+    (should be the full path to the executable but the regular operating
+    system search path is used for finding the executable) it overrides
+    the detected editor.  Optionally, some environment variables can be
+    used.  If the editor is closed without changes, `None` is returned.  In
+    case a file is edited directly the return value is always `None` and
+    `require_save` and `extension` are ignored.
+
+    If the editor cannot be opened a :exc:`UsageError` is raised.
+
+    Note for Windows: to simplify cross-platform usage, the newlines are
+    automatically converted from POSIX to Windows and vice versa.  As such,
+    the message here will have ``\n`` as newline markers.
+
+    :param text: the text to edit.
+    :param editor: optionally the editor to use.  Defaults to automatic
+                   detection.
+    :param env: environment variables to forward to the editor.
+    :param require_save: if this is true, then not saving in the editor
+                         will make the return value become `None`.
+    :param extension: the extension to tell the editor about.  This defaults
+                      to `.txt` but changing this might change syntax
+                      highlighting.
+    :param filename: if provided it will edit this file instead of the
+                     provided text contents.  It will not use a temporary
+                     file as an indirection in that case. If the editor supports
+                     editing multiple files at once, a sequence of files may be
+                     passed as well. Invoke `click.file` once per file instead
+                     if multiple files cannot be managed at once or editing the
+                     files serially is desired.
+
+    .. versionchanged:: 8.2.0
+        ``filename`` now accepts any ``Iterable[str]`` in addition to a ``str``
+        if the ``editor`` supports editing multiple files at once.
+
+    """
+    from ._termui_impl import Editor
+
+    ed = Editor(editor=editor, env=env, require_save=require_save, extension=extension)
+
+    if filename is None:
+        return ed.edit(text)
+
+    if isinstance(filename, str):
+        filename = (filename,)
+
+    ed.edit_files(filenames=filename)
+    return None
+
+
+def launch(url: str, wait: bool = False, locate: bool = False) -> int:
+    """This function launches the given URL (or filename) in the default
+    viewer application for this file type.  If this is an executable, it
+    might launch the executable in a new session.  The return value is
+    the exit code of the launched application.  Usually, ``0`` indicates
+    success.
+
+    Examples::
+
+        click.launch('https://click.palletsprojects.com/')
+        click.launch('/my/downloaded/file', locate=True)
+
+    .. versionadded:: 2.0
+
+    :param url: URL or filename of the thing to launch.
+    :param wait: Wait for the program to exit before returning. This
+        only works if the launched program blocks. In particular,
+        ``xdg-open`` on Linux does not block.
+    :param locate: if this is set to `True` then instead of launching the
+                   application associated with the URL it will attempt to
+                   launch a file manager with the file located.  This
+                   might have weird effects if the URL does not point to
+                   the filesystem.
+    """
+    from ._termui_impl import open_url
+
+    return open_url(url, wait=wait, locate=locate)
+
+
+# If this is provided, getchar() calls into this instead.  This is used
+# for unittesting purposes.
+_getchar: t.Callable[[bool], str] | None = None
+
+
+def getchar(echo: bool = False) -> str:
+    """Fetches a single character from the terminal and returns it.  This
+    will always return a unicode character and under certain rare
+    circumstances this might return more than one character.  The
+    situations which more than one character is returned is when for
+    whatever reason multiple characters end up in the terminal buffer or
+    standard input was not actually a terminal.
+
+    Note that this will always read from the terminal, even if something
+    is piped into the standard input.
+
+    Note for Windows: in rare cases when typing non-ASCII characters, this
+    function might wait for a second character and then return both at once.
+    This is because certain Unicode characters look like special-key markers.
+
+    .. versionadded:: 2.0
+
+    :param echo: if set to `True`, the character read will also show up on
+                 the terminal.  The default is to not show it.
+    """
+    global _getchar
+
+    if _getchar is None:
+        from ._termui_impl import getchar as f
+
+        _getchar = f
+
+    return _getchar(echo)
+
+
+def raw_terminal() -> AbstractContextManager[int]:
+    from ._termui_impl import raw_terminal as f
+
+    return f()
+
+
+def pause(info: str | None = None, err: bool = False) -> None:
+    """This command stops execution and waits for the user to press any
+    key to continue.  This is similar to the Windows batch "pause"
+    command.  If the program is not run through a terminal, this command
+    will instead do nothing.
+
+    .. versionadded:: 2.0
+
+    .. versionadded:: 4.0
+       Added the `err` parameter.
+
+    :param info: The message to print before pausing. Defaults to
+        ``"Press any key to continue..."``.
+    :param err: if set to message goes to ``stderr`` instead of
+                ``stdout``, the same as with echo.
+    """
+    if not isatty(sys.stdin) or not isatty(sys.stdout):
+        return
+
+    if info is None:
+        info = _("Press any key to continue...")
+
+    try:
+        if info:
+            echo(info, nl=False, err=err)
+        try:
+            getchar()
+        except (KeyboardInterrupt, EOFError):
+            pass
+    finally:
+        if info:
+            echo(err=err)

python/py/Lib/site-packages/click/testing.py

@@ -0,0 +1,574 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+import contextlib
+import io
+import os
+import shlex
+import sys
+import tempfile
+import typing as t
+from types import TracebackType
+
+from . import _compat
+from . import formatting
+from . import termui
+from . import utils
+from ._compat import _find_binary_reader
+
+if t.TYPE_CHECKING:
+    from _typeshed import ReadableBuffer
+
+    from .core import Command
+
+
+class EchoingStdin:
+    def __init__(self, input: t.BinaryIO, output: t.BinaryIO) -> None:
+        self._input = input
+        self._output = output
+        self._paused = False
+
+    def __getattr__(self, x: str) -> t.Any:
+        return getattr(self._input, x)
+
+    def _echo(self, rv: bytes) -> bytes:
+        if not self._paused:
+            self._output.write(rv)
+
+        return rv
+
+    def read(self, n: int = -1) -> bytes:
+        return self._echo(self._input.read(n))
+
+    def read1(self, n: int = -1) -> bytes:
+        return self._echo(self._input.read1(n))  # type: ignore
+
+    def readline(self, n: int = -1) -> bytes:
+        return self._echo(self._input.readline(n))
+
+    def readlines(self) -> list[bytes]:
+        return [self._echo(x) for x in self._input.readlines()]
+
+    def __iter__(self) -> cabc.Iterator[bytes]:
+        return iter(self._echo(x) for x in self._input)
+
+    def __repr__(self) -> str:
+        return repr(self._input)
+
+
+@contextlib.contextmanager
+def _pause_echo(stream: EchoingStdin | None) -> cabc.Iterator[None]:
+    if stream is None:
+        yield
+    else:
+        stream._paused = True
+        yield
+        stream._paused = False
+
+
+class BytesIOCopy(io.BytesIO):
+    """Patch ``io.BytesIO`` to let the written stream be copied to another.
+
+    .. versionadded:: 8.2
+    """
+
+    def __init__(self, copy_to: io.BytesIO) -> None:
+        super().__init__()
+        self.copy_to = copy_to
+
+    def flush(self) -> None:
+        super().flush()
+        self.copy_to.flush()
+
+    def write(self, b: ReadableBuffer) -> int:
+        self.copy_to.write(b)
+        return super().write(b)
+
+
+class StreamMixer:
+    """Mixes `<stdout>` and `<stderr>` streams.
+
+    The result is available in the ``output`` attribute.
+
+    .. versionadded:: 8.2
+    """
+
+    def __init__(self) -> None:
+        self.output: io.BytesIO = io.BytesIO()
+        self.stdout: io.BytesIO = BytesIOCopy(copy_to=self.output)
+        self.stderr: io.BytesIO = BytesIOCopy(copy_to=self.output)
+
+
+class _NamedTextIOWrapper(io.TextIOWrapper):
+    def __init__(
+        self, buffer: t.BinaryIO, name: str, mode: str, **kwargs: t.Any
+    ) -> None:
+        super().__init__(buffer, **kwargs)
+        self._name = name
+        self._mode = mode
+
+    def close(self) -> None:
+        """
+        The buffer this object contains belongs to some other object, so
+        prevent the default __del__ implementation from closing that buffer.
+
+        .. versionadded:: 8.3.2
+        """
+        ...
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def mode(self) -> str:
+        return self._mode
+
+
+def make_input_stream(
+    input: str | bytes | t.IO[t.Any] | None, charset: str
+) -> t.BinaryIO:
+    # Is already an input stream.
+    if hasattr(input, "read"):
+        rv = _find_binary_reader(t.cast("t.IO[t.Any]", input))
+
+        if rv is not None:
+            return rv
+
+        raise TypeError("Could not find binary reader for input stream.")
+
+    if input is None:
+        input = b""
+    elif isinstance(input, str):
+        input = input.encode(charset)
+
+    return io.BytesIO(input)
+
+
+class Result:
+    """Holds the captured result of an invoked CLI script.
+
+    :param runner: The runner that created the result
+    :param stdout_bytes: The standard output as bytes.
+    :param stderr_bytes: The standard error as bytes.
+    :param output_bytes: A mix of ``stdout_bytes`` and ``stderr_bytes``, as the
+        user would see  it in its terminal.
+    :param return_value: The value returned from the invoked command.
+    :param exit_code: The exit code as integer.
+    :param exception: The exception that happened if one did.
+    :param exc_info: Exception information (exception type, exception instance,
+        traceback type).
+
+    .. versionchanged:: 8.2
+        ``stderr_bytes`` no longer optional, ``output_bytes`` introduced and
+        ``mix_stderr`` has been removed.
+
+    .. versionadded:: 8.0
+        Added ``return_value``.
+    """
+
+    def __init__(
+        self,
+        runner: CliRunner,
+        stdout_bytes: bytes,
+        stderr_bytes: bytes,
+        output_bytes: bytes,
+        return_value: t.Any,
+        exit_code: int,
+        exception: BaseException | None,
+        exc_info: tuple[type[BaseException], BaseException, TracebackType]
+        | None = None,
+    ):
+        self.runner = runner
+        self.stdout_bytes = stdout_bytes
+        self.stderr_bytes = stderr_bytes
+        self.output_bytes = output_bytes
+        self.return_value = return_value
+        self.exit_code = exit_code
+        self.exception = exception
+        self.exc_info = exc_info
+
+    @property
+    def output(self) -> str:
+        """The terminal output as unicode string, as the user would see it.
+
+        .. versionchanged:: 8.2
+            No longer a proxy for ``self.stdout``. Now has its own independent stream
+            that is mixing `<stdout>` and `<stderr>`, in the order they were written.
+        """
+        return self.output_bytes.decode(self.runner.charset, "replace").replace(
+            "\r\n", "\n"
+        )
+
+    @property
+    def stdout(self) -> str:
+        """The standard output as unicode string."""
+        return self.stdout_bytes.decode(self.runner.charset, "replace").replace(
+            "\r\n", "\n"
+        )
+
+    @property
+    def stderr(self) -> str:
+        """The standard error as unicode string.
+
+        .. versionchanged:: 8.2
+            No longer raise an exception, always returns the `<stderr>` string.
+        """
+        return self.stderr_bytes.decode(self.runner.charset, "replace").replace(
+            "\r\n", "\n"
+        )
+
+    def __repr__(self) -> str:
+        exc_str = repr(self.exception) if self.exception else "okay"
+        return f"<{type(self).__name__} {exc_str}>"
+
+
+class CliRunner:
+    """The CLI runner provides functionality to invoke a Click command line
+    script for unittesting purposes in a isolated environment.  This only
+    works in single-threaded systems without any concurrency as it changes the
+    global interpreter state.
+
+    :param charset: the character set for the input and output data.
+    :param env: a dictionary with environment variables for overriding.
+    :param echo_stdin: if this is set to `True`, then reading from `<stdin>` writes
+                       to `<stdout>`.  This is useful for showing examples in
+                       some circumstances.  Note that regular prompts
+                       will automatically echo the input.
+    :param catch_exceptions: Whether to catch any exceptions other than
+                             ``SystemExit`` when running :meth:`~CliRunner.invoke`.
+
+    .. versionchanged:: 8.2
+        Added the ``catch_exceptions`` parameter.
+
+    .. versionchanged:: 8.2
+        ``mix_stderr`` parameter has been removed.
+    """
+
+    def __init__(
+        self,
+        charset: str = "utf-8",
+        env: cabc.Mapping[str, str | None] | None = None,
+        echo_stdin: bool = False,
+        catch_exceptions: bool = True,
+    ) -> None:
+        self.charset = charset
+        self.env: cabc.Mapping[str, str | None] = env or {}
+        self.echo_stdin = echo_stdin
+        self.catch_exceptions = catch_exceptions
+
+    def get_default_prog_name(self, cli: Command) -> str:
+        """Given a command object it will return the default program name
+        for it.  The default is the `name` attribute or ``"root"`` if not
+        set.
+        """
+        return cli.name or "root"
+
+    def make_env(
+        self, overrides: cabc.Mapping[str, str | None] | None = None
+    ) -> cabc.Mapping[str, str | None]:
+        """Returns the environment overrides for invoking a script."""
+        rv = dict(self.env)
+        if overrides:
+            rv.update(overrides)
+        return rv
+
+    @contextlib.contextmanager
+    def isolation(
+        self,
+        input: str | bytes | t.IO[t.Any] | None = None,
+        env: cabc.Mapping[str, str | None] | None = None,
+        color: bool = False,
+    ) -> cabc.Iterator[tuple[io.BytesIO, io.BytesIO, io.BytesIO]]:
+        """A context manager that sets up the isolation for invoking of a
+        command line tool.  This sets up `<stdin>` with the given input data
+        and `os.environ` with the overrides from the given dictionary.
+        This also rebinds some internals in Click to be mocked (like the
+        prompt functionality).
+
+        This is automatically done in the :meth:`invoke` method.
+
+        :param input: the input stream to put into `sys.stdin`.
+        :param env: the environment overrides as dictionary.
+        :param color: whether the output should contain color codes. The
+                      application can still override this explicitly.
+
+        .. versionadded:: 8.2
+            An additional output stream is returned, which is a mix of
+            `<stdout>` and `<stderr>` streams.
+
+        .. versionchanged:: 8.2
+            Always returns the `<stderr>` stream.
+
+        .. versionchanged:: 8.0
+            `<stderr>` is opened with ``errors="backslashreplace"``
+            instead of the default ``"strict"``.
+
+        .. versionchanged:: 4.0
+            Added the ``color`` parameter.
+        """
+        bytes_input = make_input_stream(input, self.charset)
+        echo_input = None
+
+        old_stdin = sys.stdin
+        old_stdout = sys.stdout
+        old_stderr = sys.stderr
+        old_forced_width = formatting.FORCED_WIDTH
+        formatting.FORCED_WIDTH = 80
+
+        env = self.make_env(env)
+
+        stream_mixer = StreamMixer()
+
+        if self.echo_stdin:
+            bytes_input = echo_input = t.cast(
+                t.BinaryIO, EchoingStdin(bytes_input, stream_mixer.stdout)
+            )
+
+        sys.stdin = text_input = _NamedTextIOWrapper(
+            bytes_input, encoding=self.charset, name="<stdin>", mode="r"
+        )
+
+        if self.echo_stdin:
+            # Force unbuffered reads, otherwise TextIOWrapper reads a
+            # large chunk which is echoed early.
+            text_input._CHUNK_SIZE = 1  # type: ignore
+
+        sys.stdout = _NamedTextIOWrapper(
+            stream_mixer.stdout, encoding=self.charset, name="<stdout>", mode="w"
+        )
+
+        sys.stderr = _NamedTextIOWrapper(
+            stream_mixer.stderr,
+            encoding=self.charset,
+            name="<stderr>",
+            mode="w",
+            errors="backslashreplace",
+        )
+
+        @_pause_echo(echo_input)  # type: ignore
+        def visible_input(prompt: str | None = None) -> str:
+            sys.stdout.write(prompt or "")
+            try:
+                val = next(text_input).rstrip("\r\n")
+            except StopIteration as e:
+                raise EOFError() from e
+            sys.stdout.write(f"{val}\n")
+            sys.stdout.flush()
+            return val
+
+        @_pause_echo(echo_input)  # type: ignore
+        def hidden_input(prompt: str | None = None) -> str:
+            sys.stdout.write(f"{prompt or ''}\n")
+            sys.stdout.flush()
+            try:
+                return next(text_input).rstrip("\r\n")
+            except StopIteration as e:
+                raise EOFError() from e
+
+        @_pause_echo(echo_input)  # type: ignore
+        def _getchar(echo: bool) -> str:
+            char = sys.stdin.read(1)
+
+            if echo:
+                sys.stdout.write(char)
+
+            sys.stdout.flush()
+            return char
+
+        default_color = color
+
+        def should_strip_ansi(
+            stream: t.IO[t.Any] | None = None, color: bool | None = None
+        ) -> bool:
+            if color is None:
+                return not default_color
+            return not color
+
+        old_visible_prompt_func = termui.visible_prompt_func
+        old_hidden_prompt_func = termui.hidden_prompt_func
+        old__getchar_func = termui._getchar
+        old_should_strip_ansi = utils.should_strip_ansi  # type: ignore
+        old__compat_should_strip_ansi = _compat.should_strip_ansi
+        termui.visible_prompt_func = visible_input
+        termui.hidden_prompt_func = hidden_input
+        termui._getchar = _getchar
+        utils.should_strip_ansi = should_strip_ansi  # type: ignore
+        _compat.should_strip_ansi = should_strip_ansi
+
+        old_env = {}
+        try:
+            for key, value in env.items():
+                old_env[key] = os.environ.get(key)
+                if value is None:
+                    try:
+                        del os.environ[key]
+                    except Exception:
+                        pass
+                else:
+                    os.environ[key] = value
+            yield (stream_mixer.stdout, stream_mixer.stderr, stream_mixer.output)
+        finally:
+            for key, value in old_env.items():
+                if value is None:
+                    try:
+                        del os.environ[key]
+                    except Exception:
+                        pass
+                else:
+                    os.environ[key] = value
+            sys.stdout = old_stdout
+            sys.stderr = old_stderr
+            sys.stdin = old_stdin
+            termui.visible_prompt_func = old_visible_prompt_func
+            termui.hidden_prompt_func = old_hidden_prompt_func
+            termui._getchar = old__getchar_func
+            utils.should_strip_ansi = old_should_strip_ansi  # type: ignore
+            _compat.should_strip_ansi = old__compat_should_strip_ansi
+            formatting.FORCED_WIDTH = old_forced_width
+
+    def invoke(
+        self,
+        cli: Command,
+        args: str | cabc.Sequence[str] | None = None,
+        input: str | bytes | t.IO[t.Any] | None = None,
+        env: cabc.Mapping[str, str | None] | None = None,
+        catch_exceptions: bool | None = None,
+        color: bool = False,
+        **extra: t.Any,
+    ) -> Result:
+        """Invokes a command in an isolated environment.  The arguments are
+        forwarded directly to the command line script, the `extra` keyword
+        arguments are passed to the :meth:`~clickpkg.Command.main` function of
+        the command.
+
+        This returns a :class:`Result` object.
+
+        :param cli: the command to invoke
+        :param args: the arguments to invoke. It may be given as an iterable
+                     or a string. When given as string it will be interpreted
+                     as a Unix shell command. More details at
+                     :func:`shlex.split`.
+        :param input: the input data for `sys.stdin`.
+        :param env: the environment overrides.
+        :param catch_exceptions: Whether to catch any other exceptions than
+                                 ``SystemExit``. If :data:`None`, the value
+                                 from :class:`CliRunner` is used.
+        :param extra: the keyword arguments to pass to :meth:`main`.
+        :param color: whether the output should contain color codes. The
+                      application can still override this explicitly.
+
+        .. versionadded:: 8.2
+            The result object has the ``output_bytes`` attribute with
+            the mix of ``stdout_bytes`` and ``stderr_bytes``, as the user would
+            see it in its terminal.
+
+        .. versionchanged:: 8.2
+            The result object always returns the ``stderr_bytes`` stream.
+
+        .. versionchanged:: 8.0
+            The result object has the ``return_value`` attribute with
+            the value returned from the invoked command.
+
+        .. versionchanged:: 4.0
+            Added the ``color`` parameter.
+
+        .. versionchanged:: 3.0
+            Added the ``catch_exceptions`` parameter.
+
+        .. versionchanged:: 3.0
+            The result object has the ``exc_info`` attribute with the
+            traceback if available.
+        """
+        exc_info = None
+        if catch_exceptions is None:
+            catch_exceptions = self.catch_exceptions
+
+        with self.isolation(input=input, env=env, color=color) as outstreams:
+            return_value = None
+            exception: BaseException | None = None
+            exit_code = 0
+
+            if isinstance(args, str):
+                args = shlex.split(args)
+
+            try:
+                prog_name = extra.pop("prog_name")
+            except KeyError:
+                prog_name = self.get_default_prog_name(cli)
+
+            try:
+                return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
+            except SystemExit as e:
+                exc_info = sys.exc_info()
+                e_code = t.cast("int | t.Any | None", e.code)
+
+                if e_code is None:
+                    e_code = 0
+
+                if e_code != 0:
+                    exception = e
+
+                if not isinstance(e_code, int):
+                    sys.stdout.write(str(e_code))
+                    sys.stdout.write("\n")
+                    e_code = 1
+
+                exit_code = e_code
+
+            except Exception as e:
+                if not catch_exceptions:
+                    raise
+                exception = e
+                exit_code = 1
+                exc_info = sys.exc_info()
+            finally:
+                sys.stdout.flush()
+                sys.stderr.flush()
+                stdout = outstreams[0].getvalue()
+                stderr = outstreams[1].getvalue()
+                output = outstreams[2].getvalue()
+
+        return Result(
+            runner=self,
+            stdout_bytes=stdout,
+            stderr_bytes=stderr,
+            output_bytes=output,
+            return_value=return_value,
+            exit_code=exit_code,
+            exception=exception,
+            exc_info=exc_info,  # type: ignore
+        )
+
+    @contextlib.contextmanager
+    def isolated_filesystem(
+        self, temp_dir: str | os.PathLike[str] | None = None
+    ) -> cabc.Iterator[str]:
+        """A context manager that creates a temporary directory and
+        changes the current working directory to it. This isolates tests
+        that affect the contents of the CWD to prevent them from
+        interfering with each other.
+
+        :param temp_dir: Create the temporary directory under this
+            directory. If given, the created directory is not removed
+            when exiting.
+
+        .. versionchanged:: 8.0
+            Added the ``temp_dir`` parameter.
+        """
+        cwd = os.getcwd()
+        dt = tempfile.mkdtemp(dir=temp_dir)
+        os.chdir(dt)
+
+        try:
+            yield dt
+        finally:
+            os.chdir(cwd)
+
+            if temp_dir is None:
+                import shutil
+
+                try:
+                    shutil.rmtree(dt)
+                except OSError:
+                    pass

python/py/Lib/site-packages/click/types.py

@@ -0,0 +1,1209 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+import enum
+import os
+import stat
+import sys
+import typing as t
+from datetime import datetime
+from gettext import gettext as _
+from gettext import ngettext
+
+from ._compat import _get_argv_encoding
+from ._compat import open_stream
+from .exceptions import BadParameter
+from .utils import format_filename
+from .utils import LazyFile
+from .utils import safecall
+
+if t.TYPE_CHECKING:
+    import typing_extensions as te
+
+    from .core import Context
+    from .core import Parameter
+    from .shell_completion import CompletionItem
+
+ParamTypeValue = t.TypeVar("ParamTypeValue")
+
+
+class ParamType:
+    """Represents the type of a parameter. Validates and converts values
+    from the command line or Python into the correct type.
+
+    To implement a custom type, subclass and implement at least the
+    following:
+
+    -   The :attr:`name` class attribute must be set.
+    -   Calling an instance of the type with ``None`` must return
+        ``None``. This is already implemented by default.
+    -   :meth:`convert` must convert string values to the correct type.
+    -   :meth:`convert` must accept values that are already the correct
+        type.
+    -   It must be able to convert a value if the ``ctx`` and ``param``
+        arguments are ``None``. This can occur when converting prompt
+        input.
+    """
+
+    is_composite: t.ClassVar[bool] = False
+    arity: t.ClassVar[int] = 1
+
+    #: the descriptive name of this type
+    name: str
+
+    #: if a list of this type is expected and the value is pulled from a
+    #: string environment variable, this is what splits it up.  `None`
+    #: means any whitespace.  For all parameters the general rule is that
+    #: whitespace splits them up.  The exception are paths and files which
+    #: are split by ``os.path.pathsep`` by default (":" on Unix and ";" on
+    #: Windows).
+    envvar_list_splitter: t.ClassVar[str | None] = None
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        """Gather information that could be useful for a tool generating
+        user-facing documentation.
+
+        Use :meth:`click.Context.to_info_dict` to traverse the entire
+        CLI structure.
+
+        .. versionadded:: 8.0
+        """
+        # The class name without the "ParamType" suffix.
+        param_type = type(self).__name__.partition("ParamType")[0]
+        param_type = param_type.partition("ParameterType")[0]
+
+        # Custom subclasses might not remember to set a name.
+        if hasattr(self, "name"):
+            name = self.name
+        else:
+            name = param_type
+
+        return {"param_type": param_type, "name": name}
+
+    def __call__(
+        self,
+        value: t.Any,
+        param: Parameter | None = None,
+        ctx: Context | None = None,
+    ) -> t.Any:
+        if value is not None:
+            return self.convert(value, param, ctx)
+
+    def get_metavar(self, param: Parameter, ctx: Context) -> str | None:
+        """Returns the metavar default for this param if it provides one."""
+
+    def get_missing_message(self, param: Parameter, ctx: Context | None) -> str | None:
+        """Optionally might return extra information about a missing
+        parameter.
+
+        .. versionadded:: 2.0
+        """
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        """Convert the value to the correct type. This is not called if
+        the value is ``None`` (the missing value).
+
+        This must accept string values from the command line, as well as
+        values that are already the correct type. It may also convert
+        other compatible types.
+
+        The ``param`` and ``ctx`` arguments may be ``None`` in certain
+        situations, such as when converting prompt input.
+
+        If the value cannot be converted, call :meth:`fail` with a
+        descriptive message.
+
+        :param value: The value to convert.
+        :param param: The parameter that is using this type to convert
+            its value. May be ``None``.
+        :param ctx: The current context that arrived at this value. May
+            be ``None``.
+        """
+        return value
+
+    def split_envvar_value(self, rv: str) -> cabc.Sequence[str]:
+        """Given a value from an environment variable this splits it up
+        into small chunks depending on the defined envvar list splitter.
+
+        If the splitter is set to `None`, which means that whitespace splits,
+        then leading and trailing whitespace is ignored.  Otherwise, leading
+        and trailing splitters usually lead to empty items being included.
+        """
+        return (rv or "").split(self.envvar_list_splitter)
+
+    def fail(
+        self,
+        message: str,
+        param: Parameter | None = None,
+        ctx: Context | None = None,
+    ) -> t.NoReturn:
+        """Helper method to fail with an invalid value message."""
+        raise BadParameter(message, ctx=ctx, param=param)
+
+    def shell_complete(
+        self, ctx: Context, param: Parameter, incomplete: str
+    ) -> list[CompletionItem]:
+        """Return a list of
+        :class:`~click.shell_completion.CompletionItem` objects for the
+        incomplete value. Most types do not provide completions, but
+        some do, and this allows custom types to provide custom
+        completions as well.
+
+        :param ctx: Invocation context for this command.
+        :param param: The parameter that is requesting completion.
+        :param incomplete: Value being completed. May be empty.
+
+        .. versionadded:: 8.0
+        """
+        return []
+
+
+class CompositeParamType(ParamType):
+    is_composite = True
+
+    @property
+    def arity(self) -> int:  # type: ignore
+        raise NotImplementedError()
+
+
+class FuncParamType(ParamType):
+    def __init__(self, func: t.Callable[[t.Any], t.Any]) -> None:
+        self.name: str = func.__name__
+        self.func = func
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        info_dict = super().to_info_dict()
+        info_dict["func"] = self.func
+        return info_dict
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        try:
+            return self.func(value)
+        except ValueError:
+            try:
+                value = str(value)
+            except UnicodeError:
+                value = value.decode("utf-8", "replace")
+
+            self.fail(value, param, ctx)
+
+
+class UnprocessedParamType(ParamType):
+    name = "text"
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        return value
+
+    def __repr__(self) -> str:
+        return "UNPROCESSED"
+
+
+class StringParamType(ParamType):
+    name = "text"
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        if isinstance(value, bytes):
+            enc = _get_argv_encoding()
+            try:
+                value = value.decode(enc)
+            except UnicodeError:
+                fs_enc = sys.getfilesystemencoding()
+                if fs_enc != enc:
+                    try:
+                        value = value.decode(fs_enc)
+                    except UnicodeError:
+                        value = value.decode("utf-8", "replace")
+                else:
+                    value = value.decode("utf-8", "replace")
+            return value
+        return str(value)
+
+    def __repr__(self) -> str:
+        return "STRING"
+
+
+class Choice(ParamType, t.Generic[ParamTypeValue]):
+    """The choice type allows a value to be checked against a fixed set
+    of supported values.
+
+    You may pass any iterable value which will be converted to a tuple
+    and thus will only be iterated once.
+
+    The resulting value will always be one of the originally passed choices.
+    See :meth:`normalize_choice` for more info on the mapping of strings
+    to choices. See :ref:`choice-opts` for an example.
+
+    :param case_sensitive: Set to false to make choices case
+        insensitive. Defaults to true.
+
+    .. versionchanged:: 8.2.0
+        Non-``str`` ``choices`` are now supported. It can additionally be any
+        iterable. Before you were not recommended to pass anything but a list or
+        tuple.
+
+    .. versionadded:: 8.2.0
+        Choice normalization can be overridden via :meth:`normalize_choice`.
+    """
+
+    name = "choice"
+
+    def __init__(
+        self, choices: cabc.Iterable[ParamTypeValue], case_sensitive: bool = True
+    ) -> None:
+        self.choices: cabc.Sequence[ParamTypeValue] = tuple(choices)
+        self.case_sensitive = case_sensitive
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        info_dict = super().to_info_dict()
+        info_dict["choices"] = self.choices
+        info_dict["case_sensitive"] = self.case_sensitive
+        return info_dict
+
+    def _normalized_mapping(
+        self, ctx: Context | None = None
+    ) -> cabc.Mapping[ParamTypeValue, str]:
+        """
+        Returns mapping where keys are the original choices and the values are
+        the normalized values that are accepted via the command line.
+
+        This is a simple wrapper around :meth:`normalize_choice`, use that
+        instead which is supported.
+        """
+        return {
+            choice: self.normalize_choice(
+                choice=choice,
+                ctx=ctx,
+            )
+            for choice in self.choices
+        }
+
+    def normalize_choice(self, choice: ParamTypeValue, ctx: Context | None) -> str:
+        """
+        Normalize a choice value, used to map a passed string to a choice.
+        Each choice must have a unique normalized value.
+
+        By default uses :meth:`Context.token_normalize_func` and if not case
+        sensitive, convert it to a casefolded value.
+
+        .. versionadded:: 8.2.0
+        """
+        normed_value = choice.name if isinstance(choice, enum.Enum) else str(choice)
+
+        if ctx is not None and ctx.token_normalize_func is not None:
+            normed_value = ctx.token_normalize_func(normed_value)
+
+        if not self.case_sensitive:
+            normed_value = normed_value.casefold()
+
+        return normed_value
+
+    def get_metavar(self, param: Parameter, ctx: Context) -> str | None:
+        if param.param_type_name == "option" and not param.show_choices:  # type: ignore
+            choice_metavars = [
+                convert_type(type(choice)).name.upper() for choice in self.choices
+            ]
+            choices_str = "|".join([*dict.fromkeys(choice_metavars)])
+        else:
+            choices_str = "|".join(
+                [str(i) for i in self._normalized_mapping(ctx=ctx).values()]
+            )
+
+        # Use curly braces to indicate a required argument.
+        if param.required and param.param_type_name == "argument":
+            return f"{{{choices_str}}}"
+
+        # Use square braces to indicate an option or optional argument.
+        return f"[{choices_str}]"
+
+    def get_missing_message(self, param: Parameter, ctx: Context | None) -> str:
+        """
+        Message shown when no choice is passed.
+
+        .. versionchanged:: 8.2.0 Added ``ctx`` argument.
+        """
+        return _("Choose from:\n\t{choices}").format(
+            choices=",\n\t".join(self._normalized_mapping(ctx=ctx).values())
+        )
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> ParamTypeValue:
+        """
+        For a given value from the parser, normalize it and find its
+        matching normalized value in the list of choices. Then return the
+        matched "original" choice.
+        """
+        normed_value = self.normalize_choice(choice=value, ctx=ctx)
+        normalized_mapping = self._normalized_mapping(ctx=ctx)
+
+        try:
+            return next(
+                original
+                for original, normalized in normalized_mapping.items()
+                if normalized == normed_value
+            )
+        except StopIteration:
+            self.fail(
+                self.get_invalid_choice_message(value=value, ctx=ctx),
+                param=param,
+                ctx=ctx,
+            )
+
+    def get_invalid_choice_message(self, value: t.Any, ctx: Context | None) -> str:
+        """Get the error message when the given choice is invalid.
+
+        :param value: The invalid value.
+
+        .. versionadded:: 8.2
+        """
+        choices_str = ", ".join(map(repr, self._normalized_mapping(ctx=ctx).values()))
+        return ngettext(
+            "{value!r} is not {choice}.",
+            "{value!r} is not one of {choices}.",
+            len(self.choices),
+        ).format(value=value, choice=choices_str, choices=choices_str)
+
+    def __repr__(self) -> str:
+        return f"Choice({list(self.choices)})"
+
+    def shell_complete(
+        self, ctx: Context, param: Parameter, incomplete: str
+    ) -> list[CompletionItem]:
+        """Complete choices that start with the incomplete value.
+
+        :param ctx: Invocation context for this command.
+        :param param: The parameter that is requesting completion.
+        :param incomplete: Value being completed. May be empty.
+
+        .. versionadded:: 8.0
+        """
+        from click.shell_completion import CompletionItem
+
+        str_choices = map(str, self.choices)
+
+        if self.case_sensitive:
+            matched = (c for c in str_choices if c.startswith(incomplete))
+        else:
+            incomplete = incomplete.lower()
+            matched = (c for c in str_choices if c.lower().startswith(incomplete))
+
+        return [CompletionItem(c) for c in matched]
+
+
+class DateTime(ParamType):
+    """The DateTime type converts date strings into `datetime` objects.
+
+    The format strings which are checked are configurable, but default to some
+    common (non-timezone aware) ISO 8601 formats.
+
+    When specifying *DateTime* formats, you should only pass a list or a tuple.
+    Other iterables, like generators, may lead to surprising results.
+
+    The format strings are processed using ``datetime.strptime``, and this
+    consequently defines the format strings which are allowed.
+
+    Parsing is tried using each format, in order, and the first format which
+    parses successfully is used.
+
+    :param formats: A list or tuple of date format strings, in the order in
+                    which they should be tried. Defaults to
+                    ``'%Y-%m-%d'``, ``'%Y-%m-%dT%H:%M:%S'``,
+                    ``'%Y-%m-%d %H:%M:%S'``.
+    """
+
+    name = "datetime"
+
+    def __init__(self, formats: cabc.Sequence[str] | None = None):
+        self.formats: cabc.Sequence[str] = formats or [
+            "%Y-%m-%d",
+            "%Y-%m-%dT%H:%M:%S",
+            "%Y-%m-%d %H:%M:%S",
+        ]
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        info_dict = super().to_info_dict()
+        info_dict["formats"] = self.formats
+        return info_dict
+
+    def get_metavar(self, param: Parameter, ctx: Context) -> str | None:
+        return f"[{'|'.join(self.formats)}]"
+
+    def _try_to_convert_date(self, value: t.Any, format: str) -> datetime | None:
+        try:
+            return datetime.strptime(value, format)
+        except ValueError:
+            return None
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        if isinstance(value, datetime):
+            return value
+
+        for format in self.formats:
+            converted = self._try_to_convert_date(value, format)
+
+            if converted is not None:
+                return converted
+
+        formats_str = ", ".join(map(repr, self.formats))
+        self.fail(
+            ngettext(
+                "{value!r} does not match the format {format}.",
+                "{value!r} does not match the formats {formats}.",
+                len(self.formats),
+            ).format(value=value, format=formats_str, formats=formats_str),
+            param,
+            ctx,
+        )
+
+    def __repr__(self) -> str:
+        return "DateTime"
+
+
+class _NumberParamTypeBase(ParamType):
+    _number_class: t.ClassVar[type[t.Any]]
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        try:
+            return self._number_class(value)
+        except ValueError:
+            self.fail(
+                _("{value!r} is not a valid {number_type}.").format(
+                    value=value, number_type=self.name
+                ),
+                param,
+                ctx,
+            )
+
+
+class _NumberRangeBase(_NumberParamTypeBase):
+    def __init__(
+        self,
+        min: float | None = None,
+        max: float | None = None,
+        min_open: bool = False,
+        max_open: bool = False,
+        clamp: bool = False,
+    ) -> None:
+        self.min = min
+        self.max = max
+        self.min_open = min_open
+        self.max_open = max_open
+        self.clamp = clamp
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        info_dict = super().to_info_dict()
+        info_dict.update(
+            min=self.min,
+            max=self.max,
+            min_open=self.min_open,
+            max_open=self.max_open,
+            clamp=self.clamp,
+        )
+        return info_dict
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        import operator
+
+        rv = super().convert(value, param, ctx)
+        lt_min: bool = self.min is not None and (
+            operator.le if self.min_open else operator.lt
+        )(rv, self.min)
+        gt_max: bool = self.max is not None and (
+            operator.ge if self.max_open else operator.gt
+        )(rv, self.max)
+
+        if self.clamp:
+            if lt_min:
+                return self._clamp(self.min, 1, self.min_open)  # type: ignore
+
+            if gt_max:
+                return self._clamp(self.max, -1, self.max_open)  # type: ignore
+
+        if lt_min or gt_max:
+            self.fail(
+                _("{value} is not in the range {range}.").format(
+                    value=rv, range=self._describe_range()
+                ),
+                param,
+                ctx,
+            )
+
+        return rv
+
+    def _clamp(self, bound: float, dir: t.Literal[1, -1], open: bool) -> float:
+        """Find the valid value to clamp to bound in the given
+        direction.
+
+        :param bound: The boundary value.
+        :param dir: 1 or -1 indicating the direction to move.
+        :param open: If true, the range does not include the bound.
+        """
+        raise NotImplementedError
+
+    def _describe_range(self) -> str:
+        """Describe the range for use in help text."""
+        if self.min is None:
+            op = "<" if self.max_open else "<="
+            return f"x{op}{self.max}"
+
+        if self.max is None:
+            op = ">" if self.min_open else ">="
+            return f"x{op}{self.min}"
+
+        lop = "<" if self.min_open else "<="
+        rop = "<" if self.max_open else "<="
+        return f"{self.min}{lop}x{rop}{self.max}"
+
+    def __repr__(self) -> str:
+        clamp = " clamped" if self.clamp else ""
+        return f"<{type(self).__name__} {self._describe_range()}{clamp}>"
+
+
+class IntParamType(_NumberParamTypeBase):
+    name = "integer"
+    _number_class = int
+
+    def __repr__(self) -> str:
+        return "INT"
+
+
+class IntRange(_NumberRangeBase, IntParamType):
+    """Restrict an :data:`click.INT` value to a range of accepted
+    values. See :ref:`ranges`.
+
+    If ``min`` or ``max`` are not passed, any value is accepted in that
+    direction. If ``min_open`` or ``max_open`` are enabled, the
+    corresponding boundary is not included in the range.
+
+    If ``clamp`` is enabled, a value outside the range is clamped to the
+    boundary instead of failing.
+
+    .. versionchanged:: 8.0
+        Added the ``min_open`` and ``max_open`` parameters.
+    """
+
+    name = "integer range"
+
+    def _clamp(  # type: ignore
+        self, bound: int, dir: t.Literal[1, -1], open: bool
+    ) -> int:
+        if not open:
+            return bound
+
+        return bound + dir
+
+
+class FloatParamType(_NumberParamTypeBase):
+    name = "float"
+    _number_class = float
+
+    def __repr__(self) -> str:
+        return "FLOAT"
+
+
+class FloatRange(_NumberRangeBase, FloatParamType):
+    """Restrict a :data:`click.FLOAT` value to a range of accepted
+    values. See :ref:`ranges`.
+
+    If ``min`` or ``max`` are not passed, any value is accepted in that
+    direction. If ``min_open`` or ``max_open`` are enabled, the
+    corresponding boundary is not included in the range.
+
+    If ``clamp`` is enabled, a value outside the range is clamped to the
+    boundary instead of failing. This is not supported if either
+    boundary is marked ``open``.
+
+    .. versionchanged:: 8.0
+        Added the ``min_open`` and ``max_open`` parameters.
+    """
+
+    name = "float range"
+
+    def __init__(
+        self,
+        min: float | None = None,
+        max: float | None = None,
+        min_open: bool = False,
+        max_open: bool = False,
+        clamp: bool = False,
+    ) -> None:
+        super().__init__(
+            min=min, max=max, min_open=min_open, max_open=max_open, clamp=clamp
+        )
+
+        if (min_open or max_open) and clamp:
+            raise TypeError("Clamping is not supported for open bounds.")
+
+    def _clamp(self, bound: float, dir: t.Literal[1, -1], open: bool) -> float:
+        if not open:
+            return bound
+
+        # Could use math.nextafter here, but clamping an
+        # open float range doesn't seem to be particularly useful. It's
+        # left up to the user to write a callback to do it if needed.
+        raise RuntimeError("Clamping is not supported for open bounds.")
+
+
+class BoolParamType(ParamType):
+    name = "boolean"
+
+    bool_states: dict[str, bool] = {
+        "1": True,
+        "0": False,
+        "yes": True,
+        "no": False,
+        "true": True,
+        "false": False,
+        "on": True,
+        "off": False,
+        "t": True,
+        "f": False,
+        "y": True,
+        "n": False,
+        # Absence of value is considered False.
+        "": False,
+    }
+    """A mapping of string values to boolean states.
+
+    Mapping is inspired by :py:attr:`configparser.ConfigParser.BOOLEAN_STATES`
+    and extends it.
+
+    .. caution::
+        String values are lower-cased, as the ``str_to_bool`` comparison function
+        below is case-insensitive.
+
+    .. warning::
+        The mapping is not exhaustive, and does not cover all possible boolean strings
+        representations. It will remains as it is to avoid endless bikeshedding.
+
+        Future work my be considered to make this mapping user-configurable from public
+        API.
+    """
+
+    @staticmethod
+    def str_to_bool(value: str | bool) -> bool | None:
+        """Convert a string to a boolean value.
+
+        If the value is already a boolean, it is returned as-is. If the value is a
+        string, it is stripped of whitespaces and lower-cased, then checked against
+        the known boolean states pre-defined in the `BoolParamType.bool_states` mapping
+        above.
+
+        Returns `None` if the value does not match any known boolean state.
+        """
+        if isinstance(value, bool):
+            return value
+        return BoolParamType.bool_states.get(value.strip().lower())
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> bool:
+        normalized = self.str_to_bool(value)
+        if normalized is None:
+            self.fail(
+                _(
+                    "{value!r} is not a valid boolean. Recognized values: {states}"
+                ).format(value=value, states=", ".join(sorted(self.bool_states))),
+                param,
+                ctx,
+            )
+        return normalized
+
+    def __repr__(self) -> str:
+        return "BOOL"
+
+
+class UUIDParameterType(ParamType):
+    name = "uuid"
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        import uuid
+
+        if isinstance(value, uuid.UUID):
+            return value
+
+        value = value.strip()
+
+        try:
+            return uuid.UUID(value)
+        except ValueError:
+            self.fail(
+                _("{value!r} is not a valid UUID.").format(value=value), param, ctx
+            )
+
+    def __repr__(self) -> str:
+        return "UUID"
+
+
+class File(ParamType):
+    """Declares a parameter to be a file for reading or writing.  The file
+    is automatically closed once the context tears down (after the command
+    finished working).
+
+    Files can be opened for reading or writing.  The special value ``-``
+    indicates stdin or stdout depending on the mode.
+
+    By default, the file is opened for reading text data, but it can also be
+    opened in binary mode or for writing.  The encoding parameter can be used
+    to force a specific encoding.
+
+    The `lazy` flag controls if the file should be opened immediately or upon
+    first IO. The default is to be non-lazy for standard input and output
+    streams as well as files opened for reading, `lazy` otherwise. When opening a
+    file lazily for reading, it is still opened temporarily for validation, but
+    will not be held open until first IO. lazy is mainly useful when opening
+    for writing to avoid creating the file until it is needed.
+
+    Files can also be opened atomically in which case all writes go into a
+    separate file in the same folder and upon completion the file will
+    be moved over to the original location.  This is useful if a file
+    regularly read by other users is modified.
+
+    See :ref:`file-args` for more information.
+
+    .. versionchanged:: 2.0
+        Added the ``atomic`` parameter.
+    """
+
+    name = "filename"
+    envvar_list_splitter: t.ClassVar[str] = os.path.pathsep
+
+    def __init__(
+        self,
+        mode: str = "r",
+        encoding: str | None = None,
+        errors: str | None = "strict",
+        lazy: bool | None = None,
+        atomic: bool = False,
+    ) -> None:
+        self.mode = mode
+        self.encoding = encoding
+        self.errors = errors
+        self.lazy = lazy
+        self.atomic = atomic
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        info_dict = super().to_info_dict()
+        info_dict.update(mode=self.mode, encoding=self.encoding)
+        return info_dict
+
+    def resolve_lazy_flag(self, value: str | os.PathLike[str]) -> bool:
+        if self.lazy is not None:
+            return self.lazy
+        if os.fspath(value) == "-":
+            return False
+        elif "w" in self.mode:
+            return True
+        return False
+
+    def convert(
+        self,
+        value: str | os.PathLike[str] | t.IO[t.Any],
+        param: Parameter | None,
+        ctx: Context | None,
+    ) -> t.IO[t.Any]:
+        if _is_file_like(value):
+            return value
+
+        value = t.cast("str | os.PathLike[str]", value)
+
+        try:
+            lazy = self.resolve_lazy_flag(value)
+
+            if lazy:
+                lf = LazyFile(
+                    value, self.mode, self.encoding, self.errors, atomic=self.atomic
+                )
+
+                if ctx is not None:
+                    ctx.call_on_close(lf.close_intelligently)
+
+                return t.cast("t.IO[t.Any]", lf)
+
+            f, should_close = open_stream(
+                value, self.mode, self.encoding, self.errors, atomic=self.atomic
+            )
+
+            # If a context is provided, we automatically close the file
+            # at the end of the context execution (or flush out).  If a
+            # context does not exist, it's the caller's responsibility to
+            # properly close the file.  This for instance happens when the
+            # type is used with prompts.
+            if ctx is not None:
+                if should_close:
+                    ctx.call_on_close(safecall(f.close))
+                else:
+                    ctx.call_on_close(safecall(f.flush))
+
+            return f
+        except OSError as e:
+            self.fail(f"'{format_filename(value)}': {e.strerror}", param, ctx)
+
+    def shell_complete(
+        self, ctx: Context, param: Parameter, incomplete: str
+    ) -> list[CompletionItem]:
+        """Return a special completion marker that tells the completion
+        system to use the shell to provide file path completions.
+
+        :param ctx: Invocation context for this command.
+        :param param: The parameter that is requesting completion.
+        :param incomplete: Value being completed. May be empty.
+
+        .. versionadded:: 8.0
+        """
+        from click.shell_completion import CompletionItem
+
+        return [CompletionItem(incomplete, type="file")]
+
+
+def _is_file_like(value: t.Any) -> te.TypeGuard[t.IO[t.Any]]:
+    return hasattr(value, "read") or hasattr(value, "write")
+
+
+class Path(ParamType):
+    """The ``Path`` type is similar to the :class:`File` type, but
+    returns the filename instead of an open file. Various checks can be
+    enabled to validate the type of file and permissions.
+
+    :param exists: The file or directory needs to exist for the value to
+        be valid. If this is not set to ``True``, and the file does not
+        exist, then all further checks are silently skipped.
+    :param file_okay: Allow a file as a value.
+    :param dir_okay: Allow a directory as a value.
+    :param readable: if true, a readable check is performed.
+    :param writable: if true, a writable check is performed.
+    :param executable: if true, an executable check is performed.
+    :param resolve_path: Make the value absolute and resolve any
+        symlinks. A ``~`` is not expanded, as this is supposed to be
+        done by the shell only.
+    :param allow_dash: Allow a single dash as a value, which indicates
+        a standard stream (but does not open it). Use
+        :func:`~click.open_file` to handle opening this value.
+    :param path_type: Convert the incoming path value to this type. If
+        ``None``, keep Python's default, which is ``str``. Useful to
+        convert to :class:`pathlib.Path`.
+
+    .. versionchanged:: 8.1
+        Added the ``executable`` parameter.
+
+    .. versionchanged:: 8.0
+        Allow passing ``path_type=pathlib.Path``.
+
+    .. versionchanged:: 6.0
+        Added the ``allow_dash`` parameter.
+    """
+
+    envvar_list_splitter: t.ClassVar[str] = os.path.pathsep
+
+    def __init__(
+        self,
+        exists: bool = False,
+        file_okay: bool = True,
+        dir_okay: bool = True,
+        writable: bool = False,
+        readable: bool = True,
+        resolve_path: bool = False,
+        allow_dash: bool = False,
+        path_type: type[t.Any] | None = None,
+        executable: bool = False,
+    ):
+        self.exists = exists
+        self.file_okay = file_okay
+        self.dir_okay = dir_okay
+        self.readable = readable
+        self.writable = writable
+        self.executable = executable
+        self.resolve_path = resolve_path
+        self.allow_dash = allow_dash
+        self.type = path_type
+
+        if self.file_okay and not self.dir_okay:
+            self.name: str = _("file")
+        elif self.dir_okay and not self.file_okay:
+            self.name = _("directory")
+        else:
+            self.name = _("path")
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        info_dict = super().to_info_dict()
+        info_dict.update(
+            exists=self.exists,
+            file_okay=self.file_okay,
+            dir_okay=self.dir_okay,
+            writable=self.writable,
+            readable=self.readable,
+            allow_dash=self.allow_dash,
+        )
+        return info_dict
+
+    def coerce_path_result(
+        self, value: str | os.PathLike[str]
+    ) -> str | bytes | os.PathLike[str]:
+        if self.type is not None and not isinstance(value, self.type):
+            if self.type is str:
+                return os.fsdecode(value)
+            elif self.type is bytes:
+                return os.fsencode(value)
+            else:
+                return t.cast("os.PathLike[str]", self.type(value))
+
+        return value
+
+    def convert(
+        self,
+        value: str | os.PathLike[str],
+        param: Parameter | None,
+        ctx: Context | None,
+    ) -> str | bytes | os.PathLike[str]:
+        rv = value
+
+        is_dash = self.file_okay and self.allow_dash and rv in (b"-", "-")
+
+        if not is_dash:
+            if self.resolve_path:
+                rv = os.path.realpath(rv)
+
+            try:
+                st = os.stat(rv)
+            except OSError:
+                if not self.exists:
+                    return self.coerce_path_result(rv)
+                self.fail(
+                    _("{name} {filename!r} does not exist.").format(
+                        name=self.name.title(), filename=format_filename(value)
+                    ),
+                    param,
+                    ctx,
+                )
+
+            if not self.file_okay and stat.S_ISREG(st.st_mode):
+                self.fail(
+                    _("{name} {filename!r} is a file.").format(
+                        name=self.name.title(), filename=format_filename(value)
+                    ),
+                    param,
+                    ctx,
+                )
+            if not self.dir_okay and stat.S_ISDIR(st.st_mode):
+                self.fail(
+                    _("{name} {filename!r} is a directory.").format(
+                        name=self.name.title(), filename=format_filename(value)
+                    ),
+                    param,
+                    ctx,
+                )
+
+            if self.readable and not os.access(rv, os.R_OK):
+                self.fail(
+                    _("{name} {filename!r} is not readable.").format(
+                        name=self.name.title(), filename=format_filename(value)
+                    ),
+                    param,
+                    ctx,
+                )
+
+            if self.writable and not os.access(rv, os.W_OK):
+                self.fail(
+                    _("{name} {filename!r} is not writable.").format(
+                        name=self.name.title(), filename=format_filename(value)
+                    ),
+                    param,
+                    ctx,
+                )
+
+            if self.executable and not os.access(value, os.X_OK):
+                self.fail(
+                    _("{name} {filename!r} is not executable.").format(
+                        name=self.name.title(), filename=format_filename(value)
+                    ),
+                    param,
+                    ctx,
+                )
+
+        return self.coerce_path_result(rv)
+
+    def shell_complete(
+        self, ctx: Context, param: Parameter, incomplete: str
+    ) -> list[CompletionItem]:
+        """Return a special completion marker that tells the completion
+        system to use the shell to provide path completions for only
+        directories or any paths.
+
+        :param ctx: Invocation context for this command.
+        :param param: The parameter that is requesting completion.
+        :param incomplete: Value being completed. May be empty.
+
+        .. versionadded:: 8.0
+        """
+        from click.shell_completion import CompletionItem
+
+        type = "dir" if self.dir_okay and not self.file_okay else "file"
+        return [CompletionItem(incomplete, type=type)]
+
+
+class Tuple(CompositeParamType):
+    """The default behavior of Click is to apply a type on a value directly.
+    This works well in most cases, except for when `nargs` is set to a fixed
+    count and different types should be used for different items.  In this
+    case the :class:`Tuple` type can be used.  This type can only be used
+    if `nargs` is set to a fixed number.
+
+    For more information see :ref:`tuple-type`.
+
+    This can be selected by using a Python tuple literal as a type.
+
+    :param types: a list of types that should be used for the tuple items.
+    """
+
+    def __init__(self, types: cabc.Sequence[type[t.Any] | ParamType]) -> None:
+        self.types: cabc.Sequence[ParamType] = [convert_type(ty) for ty in types]
+
+    def to_info_dict(self) -> dict[str, t.Any]:
+        info_dict = super().to_info_dict()
+        info_dict["types"] = [t.to_info_dict() for t in self.types]
+        return info_dict
+
+    @property
+    def name(self) -> str:  # type: ignore
+        return f"<{' '.join(ty.name for ty in self.types)}>"
+
+    @property
+    def arity(self) -> int:  # type: ignore
+        return len(self.types)
+
+    def convert(
+        self, value: t.Any, param: Parameter | None, ctx: Context | None
+    ) -> t.Any:
+        len_type = len(self.types)
+        len_value = len(value)
+
+        if len_value != len_type:
+            self.fail(
+                ngettext(
+                    "{len_type} values are required, but {len_value} was given.",
+                    "{len_type} values are required, but {len_value} were given.",
+                    len_value,
+                ).format(len_type=len_type, len_value=len_value),
+                param=param,
+                ctx=ctx,
+            )
+
+        return tuple(
+            ty(x, param, ctx) for ty, x in zip(self.types, value, strict=False)
+        )
+
+
+def convert_type(ty: t.Any | None, default: t.Any | None = None) -> ParamType:
+    """Find the most appropriate :class:`ParamType` for the given Python
+    type. If the type isn't provided, it can be inferred from a default
+    value.
+    """
+    guessed_type = False
+
+    if ty is None and default is not None:
+        if isinstance(default, (tuple, list)):
+            # If the default is empty, ty will remain None and will
+            # return STRING.
+            if default:
+                item = default[0]
+
+                # A tuple of tuples needs to detect the inner types.
+                # Can't call convert recursively because that would
+                # incorrectly unwind the tuple to a single type.
+                if isinstance(item, (tuple, list)):
+                    ty = tuple(map(type, item))
+                else:
+                    ty = type(item)
+        else:
+            ty = type(default)
+
+        guessed_type = True
+
+    if isinstance(ty, tuple):
+        return Tuple(ty)
+
+    if isinstance(ty, ParamType):
+        return ty
+
+    if ty is str or ty is None:
+        return STRING
+
+    if ty is int:
+        return INT
+
+    if ty is float:
+        return FLOAT
+
+    if ty is bool:
+        return BOOL
+
+    if guessed_type:
+        return STRING
+
+    if __debug__:
+        try:
+            if issubclass(ty, ParamType):
+                raise AssertionError(
+                    f"Attempted to use an uninstantiated parameter type ({ty})."
+                )
+        except TypeError:
+            # ty is an instance (correct), so issubclass fails.
+            pass
+
+    return FuncParamType(ty)
+
+
+#: A dummy parameter type that just does nothing.  From a user's
+#: perspective this appears to just be the same as `STRING` but
+#: internally no string conversion takes place if the input was bytes.
+#: This is usually useful when working with file paths as they can
+#: appear in bytes and unicode.
+#:
+#: For path related uses the :class:`Path` type is a better choice but
+#: there are situations where an unprocessed type is useful which is why
+#: it is is provided.
+#:
+#: .. versionadded:: 4.0
+UNPROCESSED = UnprocessedParamType()
+
+#: A unicode string parameter type which is the implicit default.  This
+#: can also be selected by using ``str`` as type.
+STRING = StringParamType()
+
+#: An integer parameter.  This can also be selected by using ``int`` as
+#: type.
+INT = IntParamType()
+
+#: A floating point value parameter.  This can also be selected by using
+#: ``float`` as type.
+FLOAT = FloatParamType()
+
+#: A boolean parameter.  This is the default for boolean flags.  This can
+#: also be selected by using ``bool`` as a type.
+BOOL = BoolParamType()
+
+#: A UUID parameter.
+UUID = UUIDParameterType()
+
+
+class OptionHelpExtra(t.TypedDict, total=False):
+    envvars: tuple[str, ...]
+    default: str
+    range: str
+    required: str

python/py/Lib/site-packages/click/utils.py

@@ -0,0 +1,627 @@
+from __future__ import annotations
+
+import collections.abc as cabc
+import os
+import re
+import sys
+import typing as t
+from functools import update_wrapper
+from types import ModuleType
+from types import TracebackType
+
+from ._compat import _default_text_stderr
+from ._compat import _default_text_stdout
+from ._compat import _find_binary_writer
+from ._compat import auto_wrap_for_ansi
+from ._compat import binary_streams
+from ._compat import open_stream
+from ._compat import should_strip_ansi
+from ._compat import strip_ansi
+from ._compat import text_streams
+from ._compat import WIN
+from .globals import resolve_color_default
+
+if t.TYPE_CHECKING:
+    import typing_extensions as te
+
+    P = te.ParamSpec("P")
+
+R = t.TypeVar("R")
+
+
+def _posixify(name: str) -> str:
+    return "-".join(name.split()).lower()
+
+
+def safecall(func: t.Callable[P, R]) -> t.Callable[P, R | None]:
+    """Wraps a function so that it swallows exceptions."""
+
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> R | None:
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            pass
+        return None
+
+    return update_wrapper(wrapper, func)
+
+
+def make_str(value: t.Any) -> str:
+    """Converts a value into a valid string."""
+    if isinstance(value, bytes):
+        try:
+            return value.decode(sys.getfilesystemencoding())
+        except UnicodeError:
+            return value.decode("utf-8", "replace")
+    return str(value)
+
+
+def make_default_short_help(help: str, max_length: int = 45) -> str:
+    """Returns a condensed version of help string."""
+    # Consider only the first paragraph.
+    paragraph_end = help.find("\n\n")
+
+    if paragraph_end != -1:
+        help = help[:paragraph_end]
+
+    # Collapse newlines, tabs, and spaces.
+    words = help.split()
+
+    if not words:
+        return ""
+
+    # The first paragraph started with a "no rewrap" marker, ignore it.
+    if words[0] == "\b":
+        words = words[1:]
+
+    total_length = 0
+    last_index = len(words) - 1
+
+    for i, word in enumerate(words):
+        total_length += len(word) + (i > 0)
+
+        if total_length > max_length:  # too long, truncate
+            break
+
+        if word[-1] == ".":  # sentence end, truncate without "..."
+            return " ".join(words[: i + 1])
+
+        if total_length == max_length and i != last_index:
+            break  # not at sentence end, truncate with "..."
+    else:
+        return " ".join(words)  # no truncation needed
+
+    # Account for the length of the suffix.
+    total_length += len("...")
+
+    # remove words until the length is short enough
+    while i > 0:
+        total_length -= len(words[i]) + (i > 0)
+
+        if total_length <= max_length:
+            break
+
+        i -= 1
+
+    return " ".join(words[:i]) + "..."
+
+
+class LazyFile:
+    """A lazy file works like a regular file but it does not fully open
+    the file but it does perform some basic checks early to see if the
+    filename parameter does make sense.  This is useful for safely opening
+    files for writing.
+    """
+
+    def __init__(
+        self,
+        filename: str | os.PathLike[str],
+        mode: str = "r",
+        encoding: str | None = None,
+        errors: str | None = "strict",
+        atomic: bool = False,
+    ):
+        self.name: str = os.fspath(filename)
+        self.mode = mode
+        self.encoding = encoding
+        self.errors = errors
+        self.atomic = atomic
+        self._f: t.IO[t.Any] | None
+        self.should_close: bool
+
+        if self.name == "-":
+            self._f, self.should_close = open_stream(filename, mode, encoding, errors)
+        else:
+            if "r" in mode:
+                # Open and close the file in case we're opening it for
+                # reading so that we can catch at least some errors in
+                # some cases early.
+                open(filename, mode).close()
+            self._f = None
+            self.should_close = True
+
+    def __getattr__(self, name: str) -> t.Any:
+        return getattr(self.open(), name)
+
+    def __repr__(self) -> str:
+        if self._f is not None:
+            return repr(self._f)
+        return f"<unopened file '{format_filename(self.name)}' {self.mode}>"
+
+    def open(self) -> t.IO[t.Any]:
+        """Opens the file if it's not yet open.  This call might fail with
+        a :exc:`FileError`.  Not handling this error will produce an error
+        that Click shows.
+        """
+        if self._f is not None:
+            return self._f
+        try:
+            rv, self.should_close = open_stream(
+                self.name, self.mode, self.encoding, self.errors, atomic=self.atomic
+            )
+        except OSError as e:
+            from .exceptions import FileError
+
+            raise FileError(self.name, hint=e.strerror) from e
+        self._f = rv
+        return rv
+
+    def close(self) -> None:
+        """Closes the underlying file, no matter what."""
+        if self._f is not None:
+            self._f.close()
+
+    def close_intelligently(self) -> None:
+        """This function only closes the file if it was opened by the lazy
+        file wrapper.  For instance this will never close stdin.
+        """
+        if self.should_close:
+            self.close()
+
+    def __enter__(self) -> LazyFile:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close_intelligently()
+
+    def __iter__(self) -> cabc.Iterator[t.AnyStr]:
+        self.open()
+        return iter(self._f)  # type: ignore
+
+
+class KeepOpenFile:
+    def __init__(self, file: t.IO[t.Any]) -> None:
+        self._file: t.IO[t.Any] = file
+
+    def __getattr__(self, name: str) -> t.Any:
+        return getattr(self._file, name)
+
+    def __enter__(self) -> KeepOpenFile:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        pass
+
+    def __repr__(self) -> str:
+        return repr(self._file)
+
+    def __iter__(self) -> cabc.Iterator[t.AnyStr]:
+        return iter(self._file)
+
+
+def echo(
+    message: t.Any | None = None,
+    file: t.IO[t.Any] | None = None,
+    nl: bool = True,
+    err: bool = False,
+    color: bool | None = None,
+) -> None:
+    """Print a message and newline to stdout or a file. This should be
+    used instead of :func:`print` because it provides better support
+    for different data, files, and environments.
+
+    Compared to :func:`print`, this does the following:
+
+    -   Ensures that the output encoding is not misconfigured on Linux.
+    -   Supports Unicode in the Windows console.
+    -   Supports writing to binary outputs, and supports writing bytes
+        to text outputs.
+    -   Supports colors and styles on Windows.
+    -   Removes ANSI color and style codes if the output does not look
+        like an interactive terminal.
+    -   Always flushes the output.
+
+    :param message: The string or bytes to output. Other objects are
+        converted to strings.
+    :param file: The file to write to. Defaults to ``stdout``.
+    :param err: Write to ``stderr`` instead of ``stdout``.
+    :param nl: Print a newline after the message. Enabled by default.
+    :param color: Force showing or hiding colors and other styles. By
+        default Click will remove color if the output does not look like
+        an interactive terminal.
+
+    .. versionchanged:: 6.0
+        Support Unicode output on the Windows console. Click does not
+        modify ``sys.stdout``, so ``sys.stdout.write()`` and ``print()``
+        will still not support Unicode.
+
+    .. versionchanged:: 4.0
+        Added the ``color`` parameter.
+
+    .. versionadded:: 3.0
+        Added the ``err`` parameter.
+
+    .. versionchanged:: 2.0
+        Support colors on Windows if colorama is installed.
+    """
+    if file is None:
+        if err:
+            file = _default_text_stderr()
+        else:
+            file = _default_text_stdout()
+
+        # There are no standard streams attached to write to. For example,
+        # pythonw on Windows.
+        if file is None:
+            return
+
+    # Convert non bytes/text into the native string type.
+    if message is not None and not isinstance(message, (str, bytes, bytearray)):
+        out: str | bytes | bytearray | None = str(message)
+    else:
+        out = message
+
+    if nl:
+        out = out or ""
+        if isinstance(out, str):
+            out += "\n"
+        else:
+            out += b"\n"
+
+    if not out:
+        file.flush()
+        return
+
+    # If there is a message and the value looks like bytes, we manually
+    # need to find the binary stream and write the message in there.
+    # This is done separately so that most stream types will work as you
+    # would expect. Eg: you can write to StringIO for other cases.
+    if isinstance(out, (bytes, bytearray)):
+        binary_file = _find_binary_writer(file)
+
+        if binary_file is not None:
+            file.flush()
+            binary_file.write(out)
+            binary_file.flush()
+            return
+
+    # ANSI style code support. For no message or bytes, nothing happens.
+    # When outputting to a file instead of a terminal, strip codes.
+    else:
+        color = resolve_color_default(color)
+
+        if should_strip_ansi(file, color):
+            out = strip_ansi(out)
+        elif WIN:
+            if auto_wrap_for_ansi is not None:
+                file = auto_wrap_for_ansi(file, color)  # type: ignore
+            elif not color:
+                out = strip_ansi(out)
+
+    file.write(out)  # type: ignore
+    file.flush()
+
+
+def get_binary_stream(name: t.Literal["stdin", "stdout", "stderr"]) -> t.BinaryIO:
+    """Returns a system stream for byte processing.
+
+    :param name: the name of the stream to open.  Valid names are ``'stdin'``,
+                 ``'stdout'`` and ``'stderr'``
+    """
+    opener = binary_streams.get(name)
+    if opener is None:
+        raise TypeError(f"Unknown standard stream '{name}'")
+    return opener()
+
+
+def get_text_stream(
+    name: t.Literal["stdin", "stdout", "stderr"],
+    encoding: str | None = None,
+    errors: str | None = "strict",
+) -> t.TextIO:
+    """Returns a system stream for text processing.  This usually returns
+    a wrapped stream around a binary stream returned from
+    :func:`get_binary_stream` but it also can take shortcuts for already
+    correctly configured streams.
+
+    :param name: the name of the stream to open.  Valid names are ``'stdin'``,
+                 ``'stdout'`` and ``'stderr'``
+    :param encoding: overrides the detected default encoding.
+    :param errors: overrides the default error mode.
+    """
+    opener = text_streams.get(name)
+    if opener is None:
+        raise TypeError(f"Unknown standard stream '{name}'")
+    return opener(encoding, errors)
+
+
+def open_file(
+    filename: str | os.PathLike[str],
+    mode: str = "r",
+    encoding: str | None = None,
+    errors: str | None = "strict",
+    lazy: bool = False,
+    atomic: bool = False,
+) -> t.IO[t.Any]:
+    """Open a file, with extra behavior to handle ``'-'`` to indicate
+    a standard stream, lazy open on write, and atomic write. Similar to
+    the behavior of the :class:`~click.File` param type.
+
+    If ``'-'`` is given to open ``stdout`` or ``stdin``, the stream is
+    wrapped so that using it in a context manager will not close it.
+    This makes it possible to use the function without accidentally
+    closing a standard stream:
+
+    .. code-block:: python
+
+        with open_file(filename) as f:
+            ...
+
+    :param filename: The name or Path of the file to open, or ``'-'`` for
+        ``stdin``/``stdout``.
+    :param mode: The mode in which to open the file.
+    :param encoding: The encoding to decode or encode a file opened in
+        text mode.
+    :param errors: The error handling mode.
+    :param lazy: Wait to open the file until it is accessed. For read
+        mode, the file is temporarily opened to raise access errors
+        early, then closed until it is read again.
+    :param atomic: Write to a temporary file and replace the given file
+        on close.
+
+    .. versionadded:: 3.0
+    """
+    if lazy:
+        return t.cast(
+            "t.IO[t.Any]", LazyFile(filename, mode, encoding, errors, atomic=atomic)
+        )
+
+    f, should_close = open_stream(filename, mode, encoding, errors, atomic=atomic)
+
+    if not should_close:
+        f = t.cast("t.IO[t.Any]", KeepOpenFile(f))
+
+    return f
+
+
+def format_filename(
+    filename: str | bytes | os.PathLike[str] | os.PathLike[bytes],
+    shorten: bool = False,
+) -> str:
+    """Format a filename as a string for display. Ensures the filename can be
+    displayed by replacing any invalid bytes or surrogate escapes in the name
+    with the replacement character ``�``.
+
+    Invalid bytes or surrogate escapes will raise an error when written to a
+    stream with ``errors="strict"``. This will typically happen with ``stdout``
+    when the locale is something like ``en_GB.UTF-8``.
+
+    Many scenarios *are* safe to write surrogates though, due to PEP 538 and
+    PEP 540, including:
+
+    -   Writing to ``stderr``, which uses ``errors="backslashreplace"``.
+    -   The system has ``LANG=C.UTF-8``, ``C``, or ``POSIX``. Python opens
+        stdout and stderr with ``errors="surrogateescape"``.
+    -   None of ``LANG/LC_*`` are set. Python assumes ``LANG=C.UTF-8``.
+    -   Python is started in UTF-8 mode  with  ``PYTHONUTF8=1`` or ``-X utf8``.
+        Python opens stdout and stderr with ``errors="surrogateescape"``.
+
+    :param filename: formats a filename for UI display.  This will also convert
+                     the filename into unicode without failing.
+    :param shorten: this optionally shortens the filename to strip of the
+                    path that leads up to it.
+    """
+    if shorten:
+        filename = os.path.basename(filename)
+    else:
+        filename = os.fspath(filename)
+
+    if isinstance(filename, bytes):
+        filename = filename.decode(sys.getfilesystemencoding(), "replace")
+    else:
+        filename = filename.encode("utf-8", "surrogateescape").decode(
+            "utf-8", "replace"
+        )
+
+    return filename
+
+
+def get_app_dir(app_name: str, roaming: bool = True, force_posix: bool = False) -> str:
+    r"""Returns the config folder for the application.  The default behavior
+    is to return whatever is most appropriate for the operating system.
+
+    To give you an idea, for an app called ``"Foo Bar"``, something like
+    the following folders could be returned:
+
+    Mac OS X:
+      ``~/Library/Application Support/Foo Bar``
+    Mac OS X (POSIX):
+      ``~/.foo-bar``
+    Unix:
+      ``~/.config/foo-bar``
+    Unix (POSIX):
+      ``~/.foo-bar``
+    Windows (roaming):
+      ``C:\Users\<user>\AppData\Roaming\Foo Bar``
+    Windows (not roaming):
+      ``C:\Users\<user>\AppData\Local\Foo Bar``
+
+    .. versionadded:: 2.0
+
+    :param app_name: the application name.  This should be properly capitalized
+                     and can contain whitespace.
+    :param roaming: controls if the folder should be roaming or not on Windows.
+                    Has no effect otherwise.
+    :param force_posix: if this is set to `True` then on any POSIX system the
+                        folder will be stored in the home folder with a leading
+                        dot instead of the XDG config home or darwin's
+                        application support folder.
+    """
+    if WIN:
+        key = "APPDATA" if roaming else "LOCALAPPDATA"
+        folder = os.environ.get(key)
+        if folder is None:
+            folder = os.path.expanduser("~")
+        return os.path.join(folder, app_name)
+    if force_posix:
+        return os.path.join(os.path.expanduser(f"~/.{_posixify(app_name)}"))
+    if sys.platform == "darwin":
+        return os.path.join(
+            os.path.expanduser("~/Library/Application Support"), app_name
+        )
+    return os.path.join(
+        os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
+        _posixify(app_name),
+    )
+
+
+class PacifyFlushWrapper:
+    """This wrapper is used to catch and suppress BrokenPipeErrors resulting
+    from ``.flush()`` being called on broken pipe during the shutdown/final-GC
+    of the Python interpreter. Notably ``.flush()`` is always called on
+    ``sys.stdout`` and ``sys.stderr``. So as to have minimal impact on any
+    other cleanup code, and the case where the underlying file is not a broken
+    pipe, all calls and attributes are proxied.
+    """
+
+    def __init__(self, wrapped: t.IO[t.Any]) -> None:
+        self.wrapped = wrapped
+
+    def flush(self) -> None:
+        try:
+            self.wrapped.flush()
+        except OSError as e:
+            import errno
+
+            if e.errno != errno.EPIPE:
+                raise
+
+    def __getattr__(self, attr: str) -> t.Any:
+        return getattr(self.wrapped, attr)
+
+
+def _detect_program_name(
+    path: str | None = None, _main: ModuleType | None = None
+) -> str:
+    """Determine the command used to run the program, for use in help
+    text. If a file or entry point was executed, the file name is
+    returned. If ``python -m`` was used to execute a module or package,
+    ``python -m name`` is returned.
+
+    This doesn't try to be too precise, the goal is to give a concise
+    name for help text. Files are only shown as their name without the
+    path. ``python`` is only shown for modules, and the full path to
+    ``sys.executable`` is not shown.
+
+    :param path: The Python file being executed. Python puts this in
+        ``sys.argv[0]``, which is used by default.
+    :param _main: The ``__main__`` module. This should only be passed
+        during internal testing.
+
+    .. versionadded:: 8.0
+        Based on command args detection in the Werkzeug reloader.
+
+    :meta private:
+    """
+    if _main is None:
+        _main = sys.modules["__main__"]
+
+    if not path:
+        path = sys.argv[0]
+
+    # The value of __package__ indicates how Python was called. It may
+    # not exist if a setuptools script is installed as an egg. It may be
+    # set incorrectly for entry points created with pip on Windows.
+    # It is set to "" inside a Shiv or PEX zipapp.
+    if getattr(_main, "__package__", None) in {None, ""} or (
+        os.name == "nt"
+        and _main.__package__ == ""
+        and not os.path.exists(path)
+        and os.path.exists(f"{path}.exe")
+    ):
+        # Executed a file, like "python app.py".
+        return os.path.basename(path)
+
+    # Executed a module, like "python -m example".
+    # Rewritten by Python from "-m script" to "/path/to/script.py".
+    # Need to look at main module to determine how it was executed.
+    py_module = t.cast(str, _main.__package__)
+    name = os.path.splitext(os.path.basename(path))[0]
+
+    # A submodule like "example.cli".
+    if name != "__main__":
+        py_module = f"{py_module}.{name}"
+
+    return f"python -m {py_module.lstrip('.')}"
+
+
+def _expand_args(
+    args: cabc.Iterable[str],
+    *,
+    user: bool = True,
+    env: bool = True,
+    glob_recursive: bool = True,
+) -> list[str]:
+    """Simulate Unix shell expansion with Python functions.
+
+    See :func:`glob.glob`, :func:`os.path.expanduser`, and
+    :func:`os.path.expandvars`.
+
+    This is intended for use on Windows, where the shell does not do any
+    expansion. It may not exactly match what a Unix shell would do.
+
+    :param args: List of command line arguments to expand.
+    :param user: Expand user home directory.
+    :param env: Expand environment variables.
+    :param glob_recursive: ``**`` matches directories recursively.
+
+    .. versionchanged:: 8.1
+        Invalid glob patterns are treated as empty expansions rather
+        than raising an error.
+
+    .. versionadded:: 8.0
+
+    :meta private:
+    """
+    from glob import glob
+
+    out = []
+
+    for arg in args:
+        if user:
+            arg = os.path.expanduser(arg)
+
+        if env:
+            arg = os.path.expandvars(arg)
+
+        try:
+            matches = glob(arg, recursive=glob_recursive)
+        except re.error:
+            matches = []
+
+        if not matches:
+            out.append(arg)
+        else:
+            out.extend(matches)
+
+    return out

python/py/Lib/site-packages/decompyle3-3.9.3.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

python/py/Lib/site-packages/decompyle3-3.9.3.dist-info/METADATA

@@ -0,0 +1,242 @@
+Metadata-Version: 2.4
+Name: decompyle3
+Version: 3.9.3
+Summary: Python cross-version byte-code library and disassembler
+Author-email: Rocky Bernstein <rb@dustyfeet.com>
+License-Expression: GPL-3.0-or-later
+Project-URL: Homepage, https://github.com/rocky/python-decompile3
+Project-URL: Downloads, https://github.com/rocky/python-decompile3/releases
+Keywords: Python bytecode,bytecode,disassembler
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Description-Content-Type: text/x-rst
+License-File: COPYING
+Requires-Dist: click
+Requires-Dist: spark-parser<1.9.2,>=1.8.9
+Requires-Dist: xdis<6.3,>=6.1.0
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+|buildstatus| |Pypi Installs| |Latest Version| |Supported Python Versions|
+
+|packagestatus|
+
+.. contents::
+
+decompyle3
+==========
+
+A native Python cross-version decompiler and fragment decompiler.
+A reworking of uncompyle6_.
+
+I gave a talk on this at `BlackHat Asia 2024 <https://youtu.be/H-7ZNrpsV50?si=nOaixgYHr7RbILVS>`_.
+
+Introduction
+------------
+
+*decompyle3* translates Python bytecode back into equivalent Python
+source code. It accepts bytecodes from Python version 3.7 on.
+
+For decompilation of older Python bytecode, see uncompyle6_.
+
+Why this?
+---------
+
+Uncompyle6 is awesome, but it has a fundamental problem in the way
+it handles control flow. In the early days of Python, when there was
+little optimization and code was generated in a very template-oriented way, figuring out control flow structures could be done by simply looking at code patterns.
+
+Over the years, more code optimization, specifically around handling jumps, has made it harder to support detecting control flow strictly
+from code patterns. This was noticed as far back as Python 2.4 (2004), but since this is a difficult problem, so far it hasn't been tackled
+in a satisfactory way.
+
+The initial attempt to fix to this problem was to add markers in the
+instruction stream, initially this was a ``COME_FROM`` instruction, and
+then use that in pattern detection.
+
+Over the years, I've extended that to be more specific, so
+``COME_FROM_LOOP`` and ``COME_FROM_WITH`` were added. And I added checks
+at grammar-reduce time to make try to make sure jumps match with
+supposed ``COME_FROM`` targets.
+
+However, all of this is complicated, not robust, has greatly slowed down deparsing and is not really tenable.
+
+In this project, we began rewriting and refactoring the grammar.
+
+However, even this isn't enough. Control flow needs
+to be addressed by using dominators and reverse-dominators, which the python-control-flow_ project can give.
+
+This I am *finally* slowly doing in yet another non-public project. It
+is a lot of work.  Funding in the form of sponsorship while greatly
+appreciated isn't commensurate with the amount of effort, and
+currently I have a full-time job. So it may take time before it is
+available publicly, if at all.
+
+Requirements
+------------
+
+The code here can be run on Python versions 3.7 or 3.8. The bytecode
+files it can read have been tested on Python bytecodes from versions
+3.7 and 3.8.
+
+Installation
+------------
+
+You can install from PyPI using the name ``decompyle3``::
+
+    pip install decompyle3
+
+
+To install from source code, this project uses setup.py, so it follows the standard Python routine::
+
+    $ pip install -e .  # set up to run from source tree
+
+or::
+
+    $ python setup.py install # may need sudo
+
+A GNU Makefile is also provided, so :code:`make install` (possibly as root or
+sudo) will do the steps above.
+
+Running Tests
+-------------
+
+::
+
+   make check
+
+A GNU makefile has been added to smooth over setting up and running the right
+command, and running tests from fastest to slowest.
+
+If you have remake_ installed, you can see the list of all tasks
+including tests via :code:`remake --tasks`
+
+
+Usage
+-----
+
+Run
+
+::
+
+$ decompyle3 *compiled-python-file-pyc-or-pyo*
+
+For usage help:
+
+::
+
+   $ decompyle3 -h
+
+Verification
+------------
+
+If you want Python syntax verification of the correctness of the
+decompilation process, add the :code:`--syntax-verify` option. However since
+Python syntax changes. You should use this option if the bytecode is
+the right bytecode for the Python interpreter that will be checking
+the syntax.
+
+You can also cross-compare the results with another Python decompiler
+like unpyc37_ . Since they work differently, bugs here often aren't in
+that, and vice versa.
+
+There is an interesting class of these programs that is readily
+available to give stronger verification: those programs that, when run, test themselves. Our test suite includes these.
+
+And Python comes with another set of programs like this: its test
+suite for the standard library. We have some code in :code:`test/stdlib` to
+facilitate this kind of checking too.
+
+Known Bugs/Restrictions
+-----------------------
+
+**We support only released versions, not candidate versions.** Note however
+that the magic of a released version is usually the same as the *last* candidate version prior to release.
+
+We also don't handle PJOrion_ or otherwise obfuscated code. For
+PJOrion try: PJOrion Deobfuscator_ to unscramble the bytecode to get
+valid bytecode before trying this tool; pydecipher_ might help with that.
+
+This program can't decompile Microsoft Windows EXE files created by
+Py2EXE_, although we can probably decompile the code after you extract
+the bytecode properly. `Pydeinstaller <https://github.com/charles-dyfis-net/pydeinstaller>`_ may help with unpacking Pyinstaller bundlers.
+
+Handling pathologically long lists of expressions or statements is slow. We don't handle Cython_ or MicroPython, which don't use bytecode.
+
+There are numerous bugs in decompilation. And that's true for every
+other CPython decompilers I have encountered, even the ones that
+claimed to be "perfect" on some particular version like 2.4.
+
+As Python progresses, decompilation also gets harder because the
+compilation is more sophisticated and the language itself is more
+sophisticated. I suspect that attempts there will be fewer ad-hoc
+attempts like unpyc37_ (which is based on a 3.3 decompiler) simply
+because it is harder to do so. The good news, at least from my
+standpoint, is that I think I understand what's needed to address the
+problems in a more robust way. But right now, until such time as
+the project is better funded, I do not intend to make any serious effort
+to support Python versions 3.8 or 3.9, including bugs that might come
+in. I imagine at some point I may be interested in it.
+
+You can easily find bugs by running the tests against the standard
+test suite that Python uses to check itself. At any given time, there are
+dozens of known problems that are pretty well isolated and that could
+be solved if one were to put in the time to do so. The problem is that
+there aren't that many people who have been working on bug fixing.
+
+You may run across a bug, that you want to report. Please do so. But
+be aware that it might not get my attention for a while. If you
+sponsor or support the project in some way, I'll prioritize your
+issues above the queue of other things I might be doing instead. In
+rare situations, I can do a hand decompilation of bytecode for a fee.
+However, this is expensive, usually beyond what most people are willing
+to spend.
+
+See Also
+--------
+
+* https://github.com/andrew-tavera/unpyc37/ : indirect fork of https://code.google.com/archive/p/unpyc3/ The above projects use a different decompiling technique than what is used here. Instructions are walked. Some instructions use the stack to generate strings, while others don't. Because control flow isn't dealt with directly, it too suffers the same problems as the various ``uncompyle`` and ``decompyle`` programs.
+* https://github.com/rocky/python-xdis : Cross Python version disassembler
+* https://github.com/rocky/python-xasm : Cross Python version assembler
+* https://github.com/rocky/python-decompile3/wiki : Wiki Documents that describe the code and aspects of it in more detail
+
+.. |buildstatus| image:: https://dl.circleci.com/status-badge/img/gh/rocky/python-decompile3/tree/master.svg?style=svg
+        :target: https://dl.circleci.com/status-badge/redirect/gh/rocky/python-decompile3/tree/master
+.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
+		 :target: https://repology.org/project/python:decompyle3/versions
+.. _Cython: https://en.wikipedia.org/wiki/Cython
+.. _MicroPython: https://micropython.org
+.. _uncompyle6: https://pypi.python.org/pypi/uncompyle6
+.. _python-control-flow: https://github.com/rocky/python-control-flow
+.. _trepan: https://pypi.python.org/pypi/trepan3k
+.. _compiler: https://pypi.python.org/pypi/spark_parser
+.. _HISTORY: https://github.com/rocky/python-decompile3/blob/master/HISTORY.md
+.. _debuggers: https://pypi.python.org/pypi/trepan3k
+.. _remake: https://bashdb.sf.net/remake
+.. _unpyc37: https://github.com/andrew-tavera/unpyc37/
+.. _this: https://github.com/rocky/python-decompile3/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
+.. |TravisCI| image:: https://travis-ci.org/rocky/python-decompile3.svg
+		 :target: https://travis-ci.org/rocky/python-decompile3
+.. |CircleCI| image:: https://circleci.com/gh/rocky/python-decompile3.svg?style=svg
+	  :target: https://circleci.com/gh/rocky/python-decompile3
+
+.. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
+.. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
+.. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
+.. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/decompyle3.svg
+.. |Latest Version| image:: https://badge.fury.io/py/decompyle3.svg
+		 :target: https://badge.fury.io/py/decompyle3
+.. |PyPI Installs| image:: https://pepy.tech/badge/decompyle3/month
+.. _pydecipher: https://github.com/mitre/pydecipher

python/py/Lib/site-packages/decompyle3-3.9.3.dist-info/RECORD

@@ -0,0 +1,199 @@
+../../Scripts/decompyle3-code.exe,sha256=PURUmK7G1avZlm-qCpxP64hui1y1yPq055JQRTe29Ug,108378
+../../Scripts/decompyle3-tokenize.exe,sha256=5rURjkLSivHG0Ksi-FOIn-tOX9mVBgOpF5YiQFaPDv0,108375
+../../Scripts/decompyle3.exe,sha256=ywoF0fm2G2h-8u_q39aIaH31i8-w3zaPF0EPCq7Z0lk,108376
+decompyle3-3.9.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+decompyle3-3.9.3.dist-info/METADATA,sha256=c3aFNjNz_sz7p1asaX7kz_2cpxVtZNMiet0h_lcW3kE,10675
+decompyle3-3.9.3.dist-info/RECORD,,
+decompyle3-3.9.3.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+decompyle3-3.9.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+decompyle3-3.9.3.dist-info/entry_points.txt,sha256=LKFx8gMDwu_poH96XBuLgqaIP_iR-ADnnJ6mB-RPTZs,182
+decompyle3-3.9.3.dist-info/licenses/COPYING,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+decompyle3-3.9.3.dist-info/top_level.txt,sha256=TzYLpK-dz7QDceD6HWvI4Boa9LNz6IFcspkQbrSgfX4,11
+decompyle3/__init__.py,sha256=6IJE9E638HJY8d2w6I5qwwjTJ7tCS1taSNVfoq3JxJU,1983
+decompyle3/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/__pycache__/code_fns.cpython-312.pyc,,
+decompyle3/__pycache__/disas.cpython-312.pyc,,
+decompyle3/__pycache__/linenumbers.cpython-312.pyc,,
+decompyle3/__pycache__/main.cpython-312.pyc,,
+decompyle3/__pycache__/scanner.cpython-312.pyc,,
+decompyle3/__pycache__/show.cpython-312.pyc,,
+decompyle3/__pycache__/util.cpython-312.pyc,,
+decompyle3/__pycache__/version.cpython-312.pyc,,
+decompyle3/bin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+decompyle3/bin/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/bin/__pycache__/decompile.cpython-312.pyc,,
+decompyle3/bin/__pycache__/decompile_code_type.cpython-312.pyc,,
+decompyle3/bin/__pycache__/decompile_tokens.cpython-312.pyc,,
+decompyle3/bin/decompile.py,sha256=W9gE0HFkZk-L9HJxBEhFv6YSiJMZxOhie7te3nfGuX4,7330
+decompyle3/bin/decompile_code_type.py,sha256=LdqLmfnXZDJMEboAdmYLDHOAMm6zSBJMvdkQheuOHgE,6649
+decompyle3/bin/decompile_tokens.py,sha256=W0YmLK5s_xIOgt9sG3FO-RR1bo842vwX4gRii6_7z_Y,3258
+decompyle3/code_fns.py,sha256=knuvqPwlpC6Zo7jjUbfraWP5-HZy-R06K2Ro_IurVGc,9782
+decompyle3/disas.py,sha256=9NZvXR3f2Mi_z4CLKY6iwx8JMz3KLKEmyFZlo5fsGe4,3907
+decompyle3/linenumbers.py,sha256=X-TYU88DctM-GAmzcqbVruXRurfxYctTElhYj7Y7_8g,2926
+decompyle3/main.py,sha256=PNA93jQ6uf10pOtFYbCmGWDAKGsYIhT4fAJ_USQzsSE,16658
+decompyle3/parsers/.gitignore,sha256=ShyC3LUBkvAj9iRHG3X1dmY9jNHY_yZLug9B1xffGqM,24
+decompyle3/parsers/__init__.py,sha256=I9m4MgCyILHl6ouh6K9s8uaTa50S6hrR1KEKuUqgPZ8,577
+decompyle3/parsers/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/parsers/__pycache__/dump.cpython-312.pyc,,
+decompyle3/parsers/__pycache__/main.cpython-312.pyc,,
+decompyle3/parsers/__pycache__/parse_heads.cpython-312.pyc,,
+decompyle3/parsers/__pycache__/treenode.cpython-312.pyc,,
+decompyle3/parsers/dump.py,sha256=XGTrzJqGUqcsCLPkGE3JFVQcL2H4p_ndole3VwSRKks,2070
+decompyle3/parsers/main.py,sha256=ruhmRTPej-hWeXxFfUA1yTkF1n5Q8IUFNDcKvjLlwn4,7295
+decompyle3/parsers/p37/Makefile,sha256=4GKcucN866rPbZC5gXFpfrigrNlneZft_qIeMIfkoW8,222
+decompyle3/parsers/p37/__init__.py,sha256=ynSCIYyBPx75dG_E5SgTMvktU7FfKFNqns87LSA8D8g,140
+decompyle3/parsers/p37/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/parsers/p37/__pycache__/base.cpython-312.pyc,,
+decompyle3/parsers/p37/__pycache__/full.cpython-312.pyc,,
+decompyle3/parsers/p37/__pycache__/heads.cpython-312.pyc,,
+decompyle3/parsers/p37/__pycache__/lambda_custom.cpython-312.pyc,,
+decompyle3/parsers/p37/__pycache__/lambda_expr.cpython-312.pyc,,
+decompyle3/parsers/p37/base.py,sha256=kyyXUrW1n3UhkhE8pNoJXhUbBQT9MYdsOQJVbHwVfaw,56966
+decompyle3/parsers/p37/full.py,sha256=hSQ1MnaHyKcAwO1zPbme8petuhyQLWUovE695DwYsFo,37500
+decompyle3/parsers/p37/heads.py,sha256=M24KjIvb9GKqoQhkKlnhpLJDKR8SqYmNC0T4lhDsHGM,2226
+decompyle3/parsers/p37/lambda_custom.py,sha256=d_F4naUjrUeTfQ6m3Ur34ZPJeDvGIy-BYuNPyGMUWoU,31269
+decompyle3/parsers/p37/lambda_expr.py,sha256=FZnGfH-ED-BKMHFzWaNgauh_Q4G3OoYdtfst5e3dm8M,28688
+decompyle3/parsers/p38/__init__.py,sha256=cxrcSU9WjZcauau67LRvj0nbFqz3DWIrBYdjm2Dwzu0,140
+decompyle3/parsers/p38/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/parsers/p38/__pycache__/base.cpython-312.pyc,,
+decompyle3/parsers/p38/__pycache__/full.cpython-312.pyc,,
+decompyle3/parsers/p38/__pycache__/full_custom.cpython-312.pyc,,
+decompyle3/parsers/p38/__pycache__/heads.cpython-312.pyc,,
+decompyle3/parsers/p38/__pycache__/lambda_custom.cpython-312.pyc,,
+decompyle3/parsers/p38/__pycache__/lambda_expr.cpython-312.pyc,,
+decompyle3/parsers/p38/base.py,sha256=liBXpDFbn1akK64qrov7L8Yz2DyHzcKdsTB6OYiINH8,2207
+decompyle3/parsers/p38/full.py,sha256=vWGiq193yu-8jjeSv-JMnkuJEslaZSne4pruNx4Mx2Y,29731
+decompyle3/parsers/p38/full_custom.py,sha256=gMzdZw1ysQ_UtgjrZXW3hlSlICjYSYwThBVYdqSs-Ek,59863
+decompyle3/parsers/p38/heads.py,sha256=cSV7xnTHgtTyRvZDTHqaJZhZrdLJZETMJ63WQaH71VY,1909
+decompyle3/parsers/p38/lambda_custom.py,sha256=SubYA4ZifcTPydndMfwK0olidzfwKEOvbViI4iYtWNo,33735
+decompyle3/parsers/p38/lambda_expr.py,sha256=hS0ZqUCwr-GDy450dJkXDvy8EuAKmKb0PGh9fv4iY2g,2897
+decompyle3/parsers/p38pypy/__init__.py,sha256=bd_5RtVBjGCaHPVhcAUrQ7VtV1SEkvYg0y-xD6nYUIY,145
+decompyle3/parsers/p38pypy/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/parsers/p38pypy/__pycache__/base.cpython-312.pyc,,
+decompyle3/parsers/p38pypy/__pycache__/full.cpython-312.pyc,,
+decompyle3/parsers/p38pypy/__pycache__/full_custom.cpython-312.pyc,,
+decompyle3/parsers/p38pypy/__pycache__/heads.cpython-312.pyc,,
+decompyle3/parsers/p38pypy/__pycache__/lambda_custom.cpython-312.pyc,,
+decompyle3/parsers/p38pypy/__pycache__/lambda_expr.cpython-312.pyc,,
+decompyle3/parsers/p38pypy/base.py,sha256=OTCCxpkqhnaUI0qntZE1kWsabQqd01hCm7PjSIeUk0U,2123
+decompyle3/parsers/p38pypy/full.py,sha256=3iyScA1jlmO9VH-fI7HtzTa4ClYVZGuoyKGczpSJ_d0,29750
+decompyle3/parsers/p38pypy/full_custom.py,sha256=Ow6jqihMNH-JLjfLfOuGzzFrYYUJyqR-zO3yBelVSns,59605
+decompyle3/parsers/p38pypy/heads.py,sha256=8HqDO0B8rr7og7ET2Pcw9dATHk3AJTXuqM4fvN34ukU,1952
+decompyle3/parsers/p38pypy/lambda_custom.py,sha256=7uJRrzdC3XzrNs0uZPqApDzkESwQaCNwxvwjSka6_Qo,33702
+decompyle3/parsers/p38pypy/lambda_expr.py,sha256=OecHWQ_AxmrpuCm6QQKvj0xLgTbblC7AfCYgA2g6EGg,3365
+decompyle3/parsers/parse_heads.py,sha256=aGMICdG1ybmiDqYcZSididg-cNRA33jZBudEhilEAP4,15264
+decompyle3/parsers/reduce_check/__init__.py,sha256=TzhrAqEdOzMTGfKKcyVm1vXkQ8tWZbgZN7kBHXIRTYw,2479
+decompyle3/parsers/reduce_check/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/and_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/and_cond_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/and_not_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/break38_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/c_tryelsestmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/for38_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/forelse38_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/if_and_elsestmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/if_and_stmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/if_not_stmtc.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/ifelsestmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/iflaststmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/ifstmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/ifstmts_jump.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/joined_str_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/lastc_stmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/list_if_not.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/not_or_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/or_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/or_cond_check.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/pop_return.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/testtrue.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/tryexcept.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/while1elsestmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/while1stmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/whileTruestmt38.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/whilestmt.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/__pycache__/whilestmt38.cpython-312.pyc,,
+decompyle3/parsers/reduce_check/and_check.py,sha256=DVwOOp6UTMG6qdo-Z0gcKiIpHz8CCwOt0BtSt5PHUCo,4026
+decompyle3/parsers/reduce_check/and_cond_check.py,sha256=GRy_zLfvyS2KK8W_7nBjfrmOa1fKSFEfKJHrytBtA4A,1340
+decompyle3/parsers/reduce_check/and_not_check.py,sha256=OTVaXZ4Dk7RS1hES3unpGfOQdzXGemkWigzgi1q4ciw,669
+decompyle3/parsers/reduce_check/break38_check.py,sha256=yorW6MY9KXRa3CshpQjTySjFD7AJVPP827ByzdsTDSc,1383
+decompyle3/parsers/reduce_check/c_tryelsestmt.py,sha256=Mk8rSpUWc-RarDqS0v5aRE2UWknKDtR2WTCgT1i0T_0,2839
+decompyle3/parsers/reduce_check/for38_check.py,sha256=sTLfesPl2DWtkPcwtEyzsTOpR81NhxfNRhKqThE44oc,3528
+decompyle3/parsers/reduce_check/forelse38_check.py,sha256=B7vJDzVX3PJiOjGZrmUl77NToLRFe1fCMI5_OKlznog,2113
+decompyle3/parsers/reduce_check/if_and_elsestmt.py,sha256=DbTRfCdR1NSZD7FWkZZA3-IpoD8UvrXWvKegPuTmy60,1978
+decompyle3/parsers/reduce_check/if_and_stmt.py,sha256=hpS6aLeNdYbXXuDrQVRzL6l_5It_8igEjjGCW9hm9Gw,2180
+decompyle3/parsers/reduce_check/if_not_stmtc.py,sha256=k10QUkQuQGImVtrjgwImmFJA0IRPRLCB8Jwe3sQ2JeU,803
+decompyle3/parsers/reduce_check/ifelsestmt.py,sha256=_pi5XlbX_0aQDJCUeq5YscIN-Z3JZo_H3z3TKAIqtRU,11488
+decompyle3/parsers/reduce_check/iflaststmt.py,sha256=Mfl-DVuScuFfRvrLMechEygMlcw4_Qmc80KvJiYdYy0,8254
+decompyle3/parsers/reduce_check/ifstmt.py,sha256=BF1QdJfp42hwppDGOiCukZyDGUAwl70u9YpcEGYp95c,9328
+decompyle3/parsers/reduce_check/ifstmts_jump.py,sha256=jz7bjsAsoL3ojYEj4-lDjGRd1IMpYs8xdjvzbSZ_pLU,1870
+decompyle3/parsers/reduce_check/joined_str_check.py,sha256=_6doAIubU2BtRmIFy3umoZ6yFcnFNvgvjAJHjIioQuw,1859
+decompyle3/parsers/reduce_check/lastc_stmt.py,sha256=JWPapiDe_7waR-eGogXS-e0CqcOqgndYeDT-5bCCsw4,1538
+decompyle3/parsers/reduce_check/list_if_not.py,sha256=QhNsNWB2T0WxrYsXhdXOUtYPoy8ND6uUhN97H17jPEs,1332
+decompyle3/parsers/reduce_check/not_or_check.py,sha256=lsJSzUdTkLekfv3K1x7muL-tw1bh11wZOR5XJPnCNnw,1958
+decompyle3/parsers/reduce_check/or_check.py,sha256=R0LhMzKJz2WTHmekJOJMUNaU2U4fukvghLVOG2uIRUc,2910
+decompyle3/parsers/reduce_check/or_cond_check.py,sha256=CM55W7x65qOlD7wesU1IHmPw-x0b5WKFgtTLhU25UJU,1212
+decompyle3/parsers/reduce_check/pop_return.py,sha256=TBU4IXKoGkFwT7bojp0eahLPDKpDZR1J5MjeX1Y7kqk,414
+decompyle3/parsers/reduce_check/testtrue.py,sha256=0VJVEIOQGKODf47M5B6E6sj9nx24dbFZbtvwcoleb90,987
+decompyle3/parsers/reduce_check/tryexcept.py,sha256=nVwpV__p_L_WtJGL7VJvjpZXO8x597RxgB2-zuIW30U,2652
+decompyle3/parsers/reduce_check/while1elsestmt.py,sha256=vBmR6lLDYwMx2OYwoe_vXN8tr0RpJJ4T6bT6BwI242Y,809
+decompyle3/parsers/reduce_check/while1stmt.py,sha256=urRsP72LVTXnHO1s8ETh8aW1c5uf4XVKF6acRb7aqbI,2273
+decompyle3/parsers/reduce_check/whileTruestmt38.py,sha256=snzo9kKO1cbpYx8ZMPsgHOLG__vjkNtZ1ycg2ZYlZQQ,2967
+decompyle3/parsers/reduce_check/whilestmt.py,sha256=a1zKKlQIpB6mAbmL6ABPmEDhY41cBe-O4rkBTpy2ptg,1256
+decompyle3/parsers/reduce_check/whilestmt38.py,sha256=KVly2XinUqZL_AbqLkKsf2Zkf-6_8pIJY03PwoBemIM,1669
+decompyle3/parsers/treenode.py,sha256=oitXkt6o5_hsGxKn1EV0ix1sY9gdLvcdCLPe5APVIuU,2553
+decompyle3/scanner.py,sha256=SveNSdR0qqYEvsZ6mepnfh_mWQ6i3H57uRodnv_e6cc,21026
+decompyle3/scanners/__init__.py,sha256=FZ43nESPw2U4w117WdQbMmXzIJbCORF_EnyDjmF9RXM,1218
+decompyle3/scanners/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/scanners/__pycache__/pypy37.cpython-312.pyc,,
+decompyle3/scanners/__pycache__/pypy38.cpython-312.pyc,,
+decompyle3/scanners/__pycache__/scanner37.cpython-312.pyc,,
+decompyle3/scanners/__pycache__/scanner37base.cpython-312.pyc,,
+decompyle3/scanners/__pycache__/scanner38-next.cpython-312.pyc,,
+decompyle3/scanners/__pycache__/scanner38.cpython-312.pyc,,
+decompyle3/scanners/__pycache__/scanner39.cpython-312.pyc,,
+decompyle3/scanners/__pycache__/tok.cpython-312.pyc,,
+decompyle3/scanners/pypy37.py,sha256=FZV0jbgjshgcogAN3iwfHO41EbxAM719UvuKNEq_oMc,732
+decompyle3/scanners/pypy38.py,sha256=PDOt0u2loJXVGLeJNzkaYg9Ptpw6H7CBTJDjAS8pk4c,686
+decompyle3/scanners/scanner37.py,sha256=Brxw35OdJWoaXfJx9F7dr8wqcj3wDoMBVjtvJ8QPFJs,4870
+decompyle3/scanners/scanner37base.py,sha256=igJGSMrT6JwuiLDYwj9ZDjlBILVln1gME9m1eKxKPYY,44559
+decompyle3/scanners/scanner38-next.py,sha256=xFLZ3L_X7WI-5vM1_vulwH_RiEKo2_nfo2eJSszEtOs,6197
+decompyle3/scanners/scanner38.py,sha256=yOUaZ78LXZubBbQMTsqNiu2_EJnLMFO_sPsAqdXm4jM,7998
+decompyle3/scanners/scanner39.py,sha256=KYj4Bx3ORDmhg62MTo8aR6h2-uhQ6XdNB69ENGrUHwg,1722
+decompyle3/scanners/tok.py,sha256=jY5fvq8tRgC312BGdciCcxNlXp3ZP6CLrNks3NFOpKU,8294
+decompyle3/semantics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+decompyle3/semantics/__pycache__/__init__.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/aligner.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/check_ast.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/consts.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/customize.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/customize3.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/customize37.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/customize38.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/fragments.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/gencomp.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/helper.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/linemap.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/make_function36.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/n_actions.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/parser_error.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/pysource.cpython-312.pyc,,
+decompyle3/semantics/__pycache__/transform.cpython-312.pyc,,
+decompyle3/semantics/aligner.py,sha256=3TwoEt7TlBvA3VFle89N79uhY_toXY5pfj6FIvXzhD4,6190
+decompyle3/semantics/check_ast.py,sha256=D_WgMBrEP_VDcw4CSRH8bAIpcMg_aidvOFnAAb95sm4,1854
+decompyle3/semantics/consts.py,sha256=w-9qqWawVnNGjWddIP0lU-QuhLgcG8jtH2HqK9t4vUA,21990
+decompyle3/semantics/customize.py,sha256=TDt52JTgAA8iX_sMGowKzezJXPaoDHfj1wbF-HQF8Ec,5265
+decompyle3/semantics/customize3.py,sha256=ngOEO-RZmxmagkej0hlnAN1RiP57cfYs6MWVNsfq59M,7200
+decompyle3/semantics/customize37.py,sha256=2o-B8rrA6MRAFXScsUuCAr468YvFcCqopOTR0uFCRzs,51346
+decompyle3/semantics/customize38.py,sha256=8CTa55DP1m8N4SgnO5uiJ24YPoQXNerR6K8MyqixtYs,16523
+decompyle3/semantics/fragments.py,sha256=xcJeDorzs6bxoFVeDhZ2JZcWzzP1hBn24om3nnE17W4,75673
+decompyle3/semantics/gencomp.py,sha256=ukyaf8JqU4Q-djTORJexbvEWzK7KtCd5TdoOv-dhbXs,26103
+decompyle3/semantics/helper.py,sha256=fRZgBNQhAgogVst2gKFlQvhsLGe6UCZTbJZqr3Z6MS0,8177
+decompyle3/semantics/linemap.py,sha256=gzCfQOjb3ozXO28D_m5Hdi55RTMUiWbaymi5CK7dj00,3578
+decompyle3/semantics/make_function36.py,sha256=-CUhbOCNMCrXsCvh4GfWi_oj-Gmovc2z9DG0iYl43ps,12572
+decompyle3/semantics/n_actions.py,sha256=PisPSAprzJBX5xJmq-B0oBLoNucZnL35c4DXPjNtCKA,43341
+decompyle3/semantics/parser_error.py,sha256=awgRfGkOuyLC4kJCH5aWWOQj6-NLvkSRwuMqhdagtmo,1292
+decompyle3/semantics/pysource.py,sha256=XWuBTG62GjvwFYhopRqmtqdssW-vQvV4AF-6jLDPPlc,43451
+decompyle3/semantics/transform.py,sha256=_9yAOllXcCZJRcLpPu7fzgwb4FABIZEwqxI2OLG8hL8,20647
+decompyle3/show.py,sha256=HdXtGrDbvxeT7Lq_XGQERK1uNkttXcN9Xl04FvvgptQ,3286
+decompyle3/util.py,sha256=YbmvluGYUAlJs8l6b3Fg4x716cQXJMPStyH7XL4HZCY,1690
+decompyle3/version.py,sha256=Z-d_UuAqttPfzuiZgjPVMazNPUuZjK0XzvxybdXNPnE,767

python/py/Lib/site-packages/decompyle3-3.9.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

python/py/Lib/site-packages/decompyle3-3.9.3.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+decompyle3 = decompyle3.bin.decompile:main_bin
+decompyle3-code = decompyle3.bin.decompile_code_type:main
+decompyle3-tokenize = decompyle3.bin.decompile_tokens:main

python/py/Lib/site-packages/decompyle3-3.9.3.dist-info/licenses/COPYING

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

python/py/Lib/site-packages/decompyle3-3.9.3.dist-info/top_level.txt

@@ -0,0 +1 @@
+decompyle3

python/py/Lib/site-packages/decompyle3/__init__.py

@@ -0,0 +1,52 @@
+"""
+  Copyright (c) 2015, 2018, 2020-2022 by Rocky Bernstein
+  Copyright (c) 2000 by hartmut Goebel <h.goebel@crazy-compilers.com>
+  Copyright (c) 1999 John Aycock
+
+  Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+
+  The above copyright notice and this permission notice shall be
+  included in all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+  NB. This is not a masterpiece of software, but became more like a hack.
+  Probably a complete rewrite would be sensefull. hG/2000-12-27
+"""
+
+import sys
+
+__docformat__ = "restructuredtext"
+
+if hasattr(sys, "setrecursionlimit"):
+    # pyston doesn't have setrecursionlimit
+    sys.setrecursionlimit(5000)
+
+# Export some convenience functions so you can say:
+# from decompyle3 import (code_deparse, deparse_code2str)
+
+from decompyle3.main import decompile_file
+from decompyle3.semantics import fragments, pysource
+from decompyle3.semantics.pysource import code_deparse, deparse_code2str
+from decompyle3.version import __version__
+
+__all__ = [
+    "__version__",
+    "code_deparse",
+    "decompile_file",
+    "deparse_code2str",
+    "fragments",
+    "pysource",
+]

python/py/Lib/site-packages/decompyle3/bin/decompile.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python
+# Mode: -*- python -*-
+#
+# Copyright (c) 2015-2017, 2019-2024 by Rocky Bernstein
+# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#
+
+import os
+import sys
+from typing import List
+
+import click
+from xdis.version_info import version_tuple_to_str
+
+from decompyle3.main import main, status_msg
+from decompyle3.version import __version__
+
+case_sensitive = {"case_sensitive": False}
+program = "decompyle3"
+
+
+def usage():
+    print(__doc__)
+    sys.exit(1)
+
+
+@click.command()
+@click.option(
+    "--asm++/--no-asm++",
+    "-A",
+    "asm_plus",
+    default=False,
+    help="show xdis assembler and tokenized assembler",
+)
+@click.option("--asm/--no-asm", "-a", default=False)
+@click.option("--grammar/--no-grammar", "-g", "show_grammar", default=False)
+@click.option("--tree/--no-tree", "-t", default=False)
+@click.option(
+    "--tree++/--no-tree++",
+    "-T",
+    "tree_plus",
+    default=False,
+    help="show parse tree and Abstract Syntax Tree",
+)
+@click.option(
+    "--linemaps/--no-linemaps",
+    default=False,
+    help="show line number correspondencies between byte-code "
+    "and generated source output",
+)
+@click.option(
+    "--verify",
+    type=click.Choice(["run", "syntax"]),
+    default=None,
+)
+@click.option(
+    "--recurse/--no-recurse",
+    "-r",
+    "recurse_dirs",
+    default=False,
+)
+@click.option(
+    "--output",
+    "-o",
+    "outfile",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+    required=False,
+)
+@click.version_option(version=__version__)
+@click.option(
+    "--start-offset",
+    "start_offset",
+    default=0,
+    help="start decomplation at offset; default is 0 or the starting offset.",
+)
+@click.version_option(version=__version__)
+@click.option(
+    "--stop-offset",
+    "stop_offset",
+    default=-1,
+    help="stop decomplation when seeing an offset greater or equal to this; default is "
+    "-1 which indicates no stopping point.",
+)
+@click.argument("files", nargs=-1, type=click.Path(readable=True), required=True)
+def main_bin(
+    asm_plus: bool,
+    asm: bool,
+    show_grammar,
+    tree: bool,
+    tree_plus: bool,
+    linemaps: bool,
+    verify,
+    recurse_dirs: bool,
+    outfile,
+    start_offset: int,
+    stop_offset: int,
+    files: List[str],
+):
+    """
+    Cross Python bytecode decompiler for Python 3.7-3.8 bytecode
+    """
+    version_tuple = sys.version_info[0:2]
+    if version_tuple < (3, 7):
+        print(
+            f"Error: {program} runs from Python 3.7 or greater."
+            f""" \n\tYou have version: {version_tuple_to_str()}."""
+        )
+        sys.exit(-1)
+
+    out_base = None
+    source_paths: List[str] = []
+    # timestamp = False
+    # timestampfmt = "# %Y.%m.%d %H:%M:%S %Z"
+
+    pyc_paths = files
+
+    # Expand directory if "recurse" was specified.
+    if recurse_dirs:
+        expanded_files = []
+        for f in pyc_paths:
+            if os.path.isdir(f):
+                for root, _, dir_files in os.walk(f):
+                    for df in dir_files:
+                        if df.endswith(".pyc") or df.endswith(".pyo"):
+                            expanded_files.append(os.path.join(root, df))
+        pyc_paths = expanded_files
+
+    # argl, commonprefix works on strings, not on path parts,
+    # thus we must handle the case with files in 'some/classes'
+    # and 'some/cmds'
+    src_base = os.path.commonprefix(pyc_paths)
+    if src_base[-1:] != os.sep:
+        src_base = os.path.dirname(src_base)
+    if src_base:
+        sb_len = len(os.path.join(src_base, ""))
+        pyc_paths = [f[sb_len:] for f in pyc_paths]
+
+    if not pyc_paths and not source_paths:
+        print("No input files given to decompile", file=sys.stderr)
+        usage()
+
+    if outfile == "-":
+        outfile = None  # use stdout
+    elif outfile and os.path.isdir(outfile):
+        out_base = outfile
+        outfile = None
+    elif outfile and len(pyc_paths) > 1:
+        out_base = outfile
+        outfile = None
+
+    # A second -a turns show_asm="after" into show_asm="before"
+    if asm_plus or asm:
+        asm_opt = "both" if asm_plus else "after"
+    else:
+        asm_opt = None
+
+    # if timestamp:
+    #     print(time.strftime(timestampfmt))
+
+    show_grammar = {
+        "rules": False,
+        "transition": False,
+        "reduce": show_grammar,
+        "errorstack": "full",
+        "context": True,
+        "dups": False,
+    }
+
+    numproc = 1
+    if numproc <= 1:
+        show_ast = {"before": tree or tree_plus, "after": tree_plus}
+        try:
+            result = main(
+                src_base,
+                out_base,
+                pyc_paths,
+                source_paths,
+                outfile,
+                showasm=asm_opt,
+                showgrammar=show_grammar,
+                showast=show_ast,
+                do_verify=verify,
+                do_linemaps=linemaps,
+                start_offset=start_offset,
+                stop_offset=stop_offset,
+            )
+
+            if len(pyc_paths) > 1:
+                mess = status_msg(verify, *result)
+                print("# " + mess)
+                pass
+        except ImportError as e:
+            print(str(e))
+            sys.exit(2)
+        except KeyboardInterrupt:
+            pass
+    else:
+        from multiprocessing import Process, Queue
+        from queue import Empty
+
+        fqueue = Queue(len(pyc_paths) + numproc)
+        for f in pyc_paths:
+            fqueue.put(f)
+        for i in range(numproc):
+            fqueue.put(None)
+
+        rqueue = Queue(numproc)
+
+        def process_func():
+            (tot_files, okay_files, failed_files, verify_failed_files) = (
+                0,
+                0,
+                0,
+                0,
+            )
+            try:
+                while True:
+                    f = fqueue.get()
+                    if f is None:
+                        break
+                    (t, o, f, v) = main(src_base, out_base, [f], [], outfile)
+                    tot_files += t
+                    okay_files += o
+                    failed_files += f
+                    verify_failed_files += v
+            except (Empty, KeyboardInterrupt):
+                pass
+            rqueue.put((tot_files, okay_files, failed_files, verify_failed_files))
+            rqueue.close()
+
+        try:
+            procs = [Process(target=process_func) for _ in range(numproc)]
+            for p in procs:
+                p.start()
+            for p in procs:
+                p.join()
+            (tot_files, okay_files, failed_files, verify_failed_files) = (
+                0,
+                0,
+                0,
+                0,
+            )
+            try:
+                while True:
+                    (t, o, f, v) = rqueue.get(False)
+                    tot_files += t
+                    okay_files += o
+                    failed_files += f
+                    verify_failed_files += v
+            except Empty:
+                pass
+            print(
+                "# decompiled %i files: %i okay, %i failed, %i verify failed"
+                % (tot_files, okay_files, failed_files, verify_failed_files)
+            )
+        except (KeyboardInterrupt, OSError):
+            pass
+
+    # if timestamp:
+    #     print(time.strftime(timestampfmt))
+
+    return
+
+
+if __name__ == "__main__":
+    main_bin()

python/py/Lib/site-packages/decompyle3/bin/decompile_code_type.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+# Mode: -*- python -*-
+#
+# Copyright (c) 2015-2016, 2018, 2020-2023 by Rocky Bernstein <rb@dustyfeet.com>
+#
+import os
+import sys
+
+import click
+from xdis.version_info import version_tuple_to_str
+
+from decompyle3.code_fns import (
+    decompile_all_fragments,
+    decompile_dict_comprehensions,
+    decompile_generators,
+    decompile_lambda_fns,
+    decompile_list_comprehensions,
+    decompile_set_comprehensions,
+)
+from decompyle3.main import decompile_file
+from decompyle3.version import __version__
+
+if click.__version__ >= "7.":
+    case_sensitive = {"case_sensitive": False}
+else:
+    case_sensitive = {}
+
+program, ext = os.path.splitext(os.path.basename(__file__))
+
+PATTERNS = ("*.pyc", "*.pyo")
+
+
+@click.command()
+@click.option(
+    "--format",
+    "-F",
+    "code_format",
+    type=click.Choice(
+        [
+            "code-fragments",
+            "dict-comprehension",
+            "exec",
+            "generator",
+            "lambda",
+            "list-comprehension",
+            "set-comprehension",
+        ],
+        **case_sensitive,
+    ),
+)
+@click.version_option(version=__version__)
+@click.option("--asm", "-a", "show_asm", count=True)
+@click.option("--grammar/--no-grammar", "-g", default=False)
+@click.option("--tree/--no-tree", "-t", default=False)
+@click.option("--tree++/--no-tree++", "-T", "tree_plus", default=False)
+@click.option(
+    "--output",
+    "-o",
+    "outfile",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+    required=False,
+)
+@click.option(
+    "--start-offset",
+    "start_offset",
+    default=0,
+    help="start decomplation at offset; default is 0 or the starting offset.",
+)
+@click.version_option(version=__version__)
+@click.option(
+    "--stop-offset",
+    "stop_offset",
+    default=-1,
+    help="stop decomplation when seeing an offset greater or equal to this; default is "
+    "-1 which indicates no stopping point.",
+)
+@click.argument("files", nargs=-1, type=click.Path(readable=True), required=True)
+def main(
+    code_format,
+    show_asm: int,
+    grammar,
+    tree,
+    tree_plus,
+    outfile,
+    start_offset: int,
+    stop_offset: int,
+    files,
+):
+    """Decompile all code objects of a certain format."""
+
+    version_tuple = sys.version_info[0:2]
+    if version_tuple < (3, 7):
+        print(
+            f"Error: {program} runs from Python 3.7 or greater."
+            f""" \n\tYou have version: {version_tuple_to_str()}."""
+        )
+        sys.exit(-1)
+
+    # FIXME is there a "click" way to do this?
+    if code_format is None:
+        code_format = "lambda"
+
+    if code_format == "code-fragments":
+        decompile_fn = decompile_all_fragments
+    elif code_format == "generator":
+        decompile_fn = decompile_generators
+    elif code_format == "lambda":
+        decompile_fn = decompile_lambda_fns
+    elif code_format == "dict-comprehension":
+        decompile_fn = decompile_dict_comprehensions
+    elif code_format == "list-comprehension":
+        decompile_fn = decompile_list_comprehensions
+    elif code_format == "set-comprehension":
+        decompile_fn = decompile_set_comprehensions
+    elif code_format == "exec":
+        decompile_fn = decompile_file
+    else:
+        print(f"Unexpected code_format {code_format}")
+        return 1
+
+    # Use stdout if outfile is None
+    if outfile is None:
+        outfile = sys.stdout
+    else:
+        if os.path.isdir(outfile):
+            outfile = None
+
+    # A second -a turns show_asm="after" into show_asm="before"
+    if show_asm > 0:
+        asm_opt = "both" if show_asm > 1 else "after"
+    else:
+        asm_opt = None
+
+    if tree_plus:
+        tree = True
+    show_ast = {"before": tree, "after": tree_plus}
+    show_grammar = {
+        "rules": False,
+        "transition": False,
+        "reduce": grammar,
+        "errorstack": "full",
+        "context": True,
+        "dups": False,
+    }
+
+    success = 0
+    skipped = 0
+    skipped = 0
+    total = 0
+    for filename in files:
+        print(f"total: {total}, success: {success}")
+        try:
+            if os.path.isdir(filename):
+                for subdir, _, files in os.walk(filename):
+                    for filename in files:
+                        filepath = subdir + os.sep + filename
+                        if (
+                            filepath.endswith(".pyc")
+                            or filepath.endswith(".py")
+                            or filepath.endswith(".pyo")
+                        ):
+                            succeeded = decompile_fn(
+                                filepath,
+                                outfile,
+                                showasm=asm_opt,
+                                showgrammar=show_grammar,
+                                showast=show_ast,
+                                start_offset=start_offset,
+                                stop_offset=stop_offset,
+                            )
+                            print()
+                            if succeeded:
+                                success += 1
+                            elif succeeded is None:
+                                skipped += 1
+                            success += 1
+                            total += 1
+            elif os.path.exists(filename) and not os.path.islink(filename):
+                if (
+                    filename.endswith(".pyc")
+                    or filename.endswith(".py")
+                    or filename.endswith(".pyo")
+                    or os.path.isdir(filename)
+                ):
+                    succeeded = decompile_fn(
+                        filename,
+                        outfile,
+                        showasm=asm_opt,
+                        showgrammar=show_grammar,
+                        showast=show_ast,
+                        start_offset=start_offset,
+                        stop_offset=stop_offset,
+                    )
+                    print()
+                    if succeeded:
+                        success += 1
+                    elif succeeded is None:
+                        skipped += 1
+                    total += 1
+            else:
+                print(f"Can't read {filename}; skipping", file=outfile)
+                skipped += 1
+                total += 1
+                pass
+            pass
+        # except RuntimeError:  # uncomment out and comment out below to see traceback
+        except RuntimeError:
+            print("Failure")
+            print(sys.exc_info()[1])
+            total += 1
+        pass
+    print(f"total: {total}, success: {success}, skipped: {skipped}")
+    return
+
+
+if __name__ == "__main__":
+    main()

python/py/Lib/site-packages/decompyle3/bin/decompile_tokens.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+#
+#  Copyright (c) 2015-2016, 2018, 2020, 2022-2024
+#  by Rocky Bernstein <rb@dustyfeet.com>
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+Command-line interface to first phase of decompliation -
+taking a disassembly and "tokenizing" this to make it easier for
+parsing.
+"""
+
+import getopt
+import os
+import sys
+
+from decompyle3.disas import disassemble_file
+from decompyle3.version import __version__
+
+program = "decompile-tokens"
+
+__doc__ = """
+Usage:
+  {0} [OPTIONS]... FILE
+  {0} [--help | -h | -V | --version]
+
+Disassemble/Tokenize FILE with in the way that is done to assist
+decompyle3 in parsing the instruction stream. For example instructions
+with variable-length arguments like CALL_FUNCTION and BUILD_LIST have
+argument counts appended to the instruction name, and COME_FROM pseudo
+instructions are inserted into the instruction stream.  Bit flag
+values encoded in an operand are expanding, EXTENDED_ARG value are
+folded into the following instruction operand.
+
+Like the parser, you may find this more high-level and or helpful.
+However if you want a true disassembler see the Standard built-in
+Python library module "dis", or pydisasm from the cross-version
+Python bytecode package "xdis".
+
+Examples:
+  {0} foo.pyc
+  {0} foo.py    # same thing as above but find the file
+  {0} foo.pyc bar.pyc  # disassemble foo.pyc and bar.pyc
+
+See also `pydisasm' from the `xdis' package.
+
+Options:
+  -V | --version     show version and stop
+  -h | --help        show this message
+
+""".format(
+    program
+)
+
+PATTERNS = ("*.pyc", "*.pyo")
+
+
+def main():
+    usage_short = f"""usage: {program} FILE...
+Type -h for for full help."""
+
+    if len(sys.argv) == 1:
+        print("No file(s) given", file=sys.stderr)
+        print(usage_short, file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        opts, files = getopt.getopt(
+            sys.argv[1:], "hVU", ["help", "version", "decompyle3"]
+        )
+    except getopt.GetoptError as e:
+        print(f"{os.path.basename(sys.argv[0])}: {e}", file=sys.stderr)
+        sys.exit(-1)
+
+    for opt, val in opts:
+        if opt in ("-h", "--help"):
+            print(__doc__)
+            sys.exit(1)
+        elif opt in ("-V", "--version"):
+            print(f"{program} {__version__}")
+            sys.exit(0)
+        else:
+            print(opt)
+            print(usage_short, file=sys.stderr)
+            sys.exit(1)
+
+    for file in files:
+        if os.path.exists(files[0]):
+            disassemble_file(file, sys.stdout)
+        else:
+            print(f"Can't read {files[0]} - skipping", file=sys.stderr)
+            pass
+        pass
+    return
+
+
+if __name__ == "__main__":
+    main()

python/py/Lib/site-packages/decompyle3/code_fns.py

@@ -0,0 +1,370 @@
+#  Copyright (c) 2015-2016, 2818-2024 by Rocky Bernstein
+#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 1999 John Aycock
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+CPython magic- and version- independent disassembly routines
+
+There are two reasons we can't use Python's built-in routines
+from dis. First, the bytecode we are extracting may be from a different
+version of Python (different magic number) than the version of Python
+that is doing the extraction.
+
+Second, we need structured instruction information for the
+(de)-parsing step. Python 3.4 and up provides this, but we still do
+want to run on earlier Python versions.
+"""
+
+import sys
+from collections import deque
+from py_compile import PyCompileError
+from typing import Optional
+
+from xdis import check_object_path, iscode, load_module
+
+from decompyle3.scanner import get_scanner
+from decompyle3.semantics.pysource import (
+    PARSER_DEFAULT_DEBUG,
+    TREE_DEFAULT_DEBUG,
+    code_deparse,
+)
+
+
+def disco_deparse(
+    version: Optional[tuple],
+    co,
+    codename_map: dict,
+    out,
+    is_pypy,
+    debug_opts,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> None:
+    """
+    diassembles and deparses a given code block 'co'
+    """
+
+    assert iscode(co)
+
+    # store final output stream for case of error
+    real_out = out or sys.stdout
+    print(f"# Python {version}", file=real_out)
+    if co.co_filename:
+        print(f"# Embedded file name: {co.co_filename}", file=real_out)
+
+    scanner = get_scanner(version, is_pypy=is_pypy)
+
+    queue = deque([co])
+    disco_deparse_loop(
+        version,
+        scanner.ingest,
+        codename_map,
+        queue,
+        real_out,
+        is_pypy,
+        debug_opts,
+        start_offset=start_offset,
+        stop_offset=stop_offset,
+    )
+
+
+def disco_deparse_loop(
+    version: Optional[tuple],
+    disasm,
+    codename_map: dict,
+    queue,
+    real_out,
+    is_pypy,
+    debug_opts,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+):
+    while len(queue) > 0:
+        co = queue.popleft()
+        skip_token_scan = False
+        if co.co_name in codename_map:
+            print(
+                "\n# %s line %d of %s"
+                % (co.co_name, co.co_firstlineno, co.co_filename),
+                file=real_out,
+            )
+
+            code_deparse(
+                co,
+                real_out,
+                version=version,
+                debug_opts=debug_opts,
+                is_pypy=is_pypy,
+                compile_mode=codename_map[co.co_name],
+                start_offset=start_offset,
+                stop_offset=stop_offset,
+            )
+            skip_token_scan = True
+
+        tokens, _ = disasm(co, show_asm=debug_opts.get("asm", None))
+        if skip_token_scan:
+            continue
+        for t in tokens:
+            if iscode(t.pattr):
+                queue.append(t.pattr)
+            elif iscode(t.attr):
+                queue.append(t.attr)
+            pass
+        pass
+
+
+def decompile_code_type(
+    filename: str,
+    codename_map: dict,
+    outstream=None,
+    showasm=None,
+    showast=TREE_DEFAULT_DEBUG,
+    showgrammar=PARSER_DEFAULT_DEBUG,
+    start_offset=0,
+    stop_offset=-1,
+) -> bool:
+    """
+    decompile all lambda functions in a python byte-code file (.pyc)
+
+    If given a Python source file (".py") file, we'll
+    decompile all lambdas of the corresponding compiled object.
+    """
+    try:
+        filename = check_object_path(filename)
+    except (PyCompileError, ValueError) as e:
+        print(f"Skipping {filename}:\n{e}")
+        return False
+
+    (version, _, _, co, is_pypy, _, _) = load_module(filename)
+
+    # maybe a second -a will do before as well
+    # asm = "after" if showasm else None
+
+    debug_opts = {"asm": showasm, "tree": showast, "grammar": showgrammar}
+    if isinstance(co, list):
+        for bytecode in co:
+            disco_deparse(
+                version,
+                bytecode,
+                codename_map,
+                outstream,
+                is_pypy,
+                debug_opts,
+                start_offset=start_offset,
+                stop_offset=stop_offset,
+            )
+    else:
+        disco_deparse(
+            version,
+            co,
+            codename_map,
+            outstream,
+            is_pypy,
+            debug_opts,
+            start_offset=start_offset,
+            stop_offset=stop_offset,
+        )
+    return True
+
+
+def decompile_dict_comprehensions(
+    filename: str,
+    outstream=None,
+    showasm=None,
+    showast=TREE_DEFAULT_DEBUG,
+    showgrammar=PARSER_DEFAULT_DEBUG,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> Optional[bool]:
+    """
+    decompile all the dictionary-comprehension functions in a python byte-code
+    file (.pyc)
+
+    If given a Python source file (".py") file, we'll
+    decompile all dict_comprehensions of the corresponding compiled object.
+    """
+    return decompile_code_type(
+        filename,
+        {"<dictcomp>": "dictcomp"},
+        outstream,
+        showasm,
+        showast,
+        showgrammar,
+        start_offset,
+        stop_offset,
+    )
+
+
+def decompile_all_fragments(
+    filename: str,
+    outstream=None,
+    showasm=None,
+    showast=TREE_DEFAULT_DEBUG,
+    showgrammar=PARSER_DEFAULT_DEBUG,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> Optional[bool]:
+    """
+    decompile all comprehensions, generators, and lambda in a python byte-code
+    file (.pyc)
+
+    If given a Python source file (".py") file, we'll
+    decompile all dict_comprehensions of the corresponding compiled object.
+    """
+    return decompile_code_type(
+        filename,
+        {
+            "<dictcomp>": "dictcomp",
+            "<genexpr>": "genexpr",
+            "<lambda>": "lambda",
+            "<listcomp>": "listcomp",
+            "<setcomp>": "setcomp",
+        },
+        outstream,
+        showasm,
+        showast,
+        showgrammar,
+        start_offset=start_offset,
+        stop_offset=stop_offset,
+    )
+
+
+def decompile_generators(
+    filename: str,
+    outstream=None,
+    showasm=None,
+    showast=TREE_DEFAULT_DEBUG,
+    showgrammar=PARSER_DEFAULT_DEBUG,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> Optional[bool]:
+    """
+    decompile all the generator functions in a python byte-code file (.pyc)
+
+    If given a Python source file (".py") file, we'll
+    decompile all dict_comprehensions of the corresponding compiled object.
+    """
+    return decompile_code_type(
+        filename,
+        {"<genexpr>": "genexpr"},
+        outstream,
+        showasm,
+        showast,
+        showgrammar,
+        start_offset,
+        stop_offset,
+    )
+
+
+def decompile_lambda_fns(
+    filename: str,
+    outstream=None,
+    showasm=None,
+    showast=TREE_DEFAULT_DEBUG,
+    showgrammar=PARSER_DEFAULT_DEBUG,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> Optional[bool]:
+    """
+    decompile all the lambda functions in a python byte-code file (.pyc)
+
+    If given a Python source file (".py") file, we'll
+    decompile all lambdas of the corresponding compiled object.
+    """
+    return decompile_code_type(
+        filename,
+        {"<lambda>": "lambda"},
+        outstream,
+        showasm,
+        showast,
+        showgrammar,
+        start_offset=start_offset,
+        stop_offset=stop_offset,
+    )
+
+
+def decompile_list_comprehensions(
+    filename: str,
+    outstream=None,
+    showasm=None,
+    showast=TREE_DEFAULT_DEBUG,
+    showgrammar=PARSER_DEFAULT_DEBUG,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> Optional[bool]:
+    """
+    decompile all of the lambda functions in a python byte-code file (.pyc)
+
+    If given a Python source file (".py") file, we'll
+    decompile all list_comprehensions of the corresponding compiled object.
+    """
+    return decompile_code_type(
+        filename,
+        {"<listcomp>": "listcomp"},
+        outstream,
+        showasm,
+        showast,
+        showgrammar,
+        start_offset=start_offset,
+        stop_offset=stop_offset,
+    )
+
+
+def decompile_set_comprehensions(
+    filename: str,
+    code_type,
+    outstream=None,
+    showasm=None,
+    showast=TREE_DEFAULT_DEBUG,
+    showgrammar=PARSER_DEFAULT_DEBUG,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> Optional[bool]:
+    """
+    decompile all lambda functions in a python byte-code file (.pyc)
+
+    If given a Python source file (".py") file, we'll
+    decompile all list_comprehensions of the corresponding compiled object.
+    """
+    return decompile_code_type(
+        filename,
+        {"<setcomp>": "setcomp"},
+        outstream,
+        showasm,
+        showast,
+        showgrammar,
+        start_offset=start_offset,
+        stop_offset=stop_offset,
+    )
+
+
+def _test() -> None:
+    """Simple test program to disassemble a file."""
+    argc = len(sys.argv)
+    if argc != 2:
+        if argc == 1:
+            fn = __file__
+        else:
+            sys.stderr.write("usage: %s [-|CPython compiled file]\n" % __file__)
+            sys.exit(2)
+    else:
+        fn = sys.argv[1]
+    decompile_all_fragments(fn)
+
+
+if __name__ == "__main__":
+    _test()

python/py/Lib/site-packages/decompyle3/disas.py

@@ -0,0 +1,127 @@
+#  Copyright (c) 2015-2016, 2818-2020, 2024 by Rocky Bernstein
+#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 1999 John Aycock
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+CPython magic- and version- independent disassembly routines
+
+There are two reasons we can't use Python's built-in routines
+from dis. First, the bytecode we are extracting may be from a different
+version of Python (different magic number) than the version of Python
+that is doing the extraction.
+
+Second, we need structured instruction information for the
+(de)-parsing step. Python 3.4 and up provides this, but we still do
+want to run on earlier Python versions.
+"""
+
+import sys
+from collections import deque
+
+from xdis import check_object_path, iscode, load_module
+
+from decompyle3.scanner import get_scanner
+
+
+def disco(version: str, co, out=None, is_pypy=False) -> None:
+    """
+    diassembles and deparses a given code block 'co'
+    """
+
+    assert iscode(co)
+
+    # store final output stream for case of error
+    real_out = out or sys.stdout
+    print(f"# Python {version}", file=real_out)
+    if co.co_filename:
+        print(f"# Embedded file name: {co.co_filename}", file=real_out)
+
+    scanner = get_scanner(version, is_pypy=is_pypy)
+
+    queue = deque([co])
+    disco_loop(scanner.ingest, queue, real_out)
+
+
+def disco_loop(disasm, queue, real_out):
+    while len(queue) > 0:
+        co = queue.popleft()
+        if co.co_name != "<module>":
+            print(
+                "\n# %s line %d of %s"
+                % (co.co_name, co.co_firstlineno, co.co_filename),
+                file=real_out,
+            )
+        tokens, customize = disasm(co)
+        for t in tokens:
+            if iscode(t.pattr):
+                queue.append(t.pattr)
+            elif iscode(t.attr):
+                queue.append(t.attr)
+            print(t, file=real_out)
+            pass
+        pass
+
+
+# def disassemble_fp(fp, outstream=None):
+#     """
+#     disassemble Python byte-code from an open file
+#     """
+#     (version, timestamp, magic_int, co, is_pypy,
+#      source_size) = load_from_fp(fp)
+#     if type(co) == list:
+#         for con in co:
+#             disco(version, con, outstream)
+#     else:
+#         disco(version, co, outstream, is_pypy=is_pypy)
+#     co = None
+
+
+def disassemble_file(filename: str, outstream=None) -> None:
+    """
+    disassemble Python byte-code file (.pyc)
+
+    If given a Python source file (".py") file, we'll
+    try to find the corresponding compiled object.
+    """
+    filename = check_object_path(filename)
+    (version, timestamp, magic_int, co, is_pypy, source_size, sip_hash) = load_module(
+        filename
+    )
+    if isinstance(co, list):
+        for con in co:
+            disco(version, con, outstream)
+    else:
+        disco(version, co, outstream, is_pypy=is_pypy)
+    co = None
+
+
+def _test() -> None:
+    """Simple test program to disassemble a file."""
+    argc = len(sys.argv)
+    if argc != 2:
+        if argc == 1:
+            fn = __file__
+        else:
+            sys.stderr.write(f"usage: {__file__} [-|CPython compiled file]\n")
+            sys.exit(2)
+    else:
+        fn = sys.argv[1]
+    disassemble_file(fn)
+
+
+if __name__ == "__main__":
+    _test()

python/py/Lib/site-packages/decompyle3/linenumbers.py

@@ -0,0 +1,92 @@
+#  Copyright (c) 2015-2016, 2018-2020 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from collections import deque
+
+from xdis import (
+    Bytecode,
+    iscode,
+    findlinestarts,
+    get_opcode,
+    offset2line,
+    load_file,
+    load_module,
+)
+
+
+def line_number_mapping(pyc_filename, src_filename):
+    (
+        version,
+        timestamp,
+        magic_int,
+        code1,
+        is_pypy,
+        source_size,
+        sip_hash,
+    ) = load_module(pyc_filename)
+    try:
+        code2 = load_file(src_filename)
+    except SyntaxError as e:
+        return str(e)
+
+    queue = deque([code1, code2])
+
+    mappings = []
+
+    opc = get_opcode(version, is_pypy)
+    number_loop(queue, mappings, opc)
+    return sorted(mappings, key=lambda x: x[1])
+
+
+def number_loop(queue, mappings, opc):
+    while len(queue) > 0:
+        code1 = queue.popleft()
+        code2 = queue.popleft()
+        assert code1.co_name == code2.co_name
+        linestarts_orig = findlinestarts(code1)
+        linestarts_uncompiled = list(findlinestarts(code2))
+        mappings += [
+            [line, offset2line(offset, linestarts_uncompiled)]
+            for offset, line in linestarts_orig
+        ]
+        bytecode1 = Bytecode(code1, opc)
+        bytecode2 = Bytecode(code2, opc)
+        instr2s = bytecode2.get_instructions(code2)
+        seen = set([code1.co_name])
+        for instr in bytecode1.get_instructions(code1):
+            next_code1 = None
+            if iscode(instr.argval):
+                next_code1 = instr.argval
+            if next_code1:
+                next_code2 = None
+                while not next_code2:
+                    try:
+                        instr2 = next(instr2s)
+                        if iscode(instr2.argval):
+                            next_code2 = instr2.argval
+                            pass
+                    except StopIteration:
+                        break
+                    pass
+                if next_code2:
+                    assert next_code1.co_name == next_code2.co_name
+                    if next_code1.co_name not in seen:
+                        seen.add(next_code1.co_name)
+                        queue.append(next_code1)
+                        queue.append(next_code2)
+                        pass
+                    pass
+            pass
+        pass

python/py/Lib/site-packages/decompyle3/main.py

@@ -0,0 +1,501 @@
+# Copyright (C) 2018-2025 Rocky Bernstein <rocky@gnu.org>
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import ast
+import datetime
+import os
+import os.path as osp
+import py_compile
+import subprocess
+import sys
+import tempfile
+from typing import Any, Optional, TextIO, Tuple
+
+from xdis import iscode, load_module
+from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE, version_tuple_to_str
+
+from decompyle3.disas import check_object_path
+from decompyle3.parsers.parse_heads import ParserError
+from decompyle3.semantics import pysource
+from decompyle3.semantics.fragments import code_deparse as code_deparse_fragments
+from decompyle3.semantics.linemap import deparse_code_with_map
+from decompyle3.semantics.pysource import PARSER_DEFAULT_DEBUG, code_deparse
+from decompyle3.version import __version__
+
+# from decompyle3.linenumbers import line_number_mapping
+
+
+def _get_outstream(outfile: str) -> Any:
+    dir = osp.dirname(outfile)
+    failed_file = outfile + "_failed"
+    if osp.exists(failed_file):
+        os.remove(failed_file)
+    try:
+        os.makedirs(dir)
+    except OSError:
+        pass
+    return open(outfile, mode="w", encoding="utf-8")
+
+
+def syntax_check(filename: str) -> bool:
+    with open(filename) as f:
+        source = f.read()
+    valid = True
+    try:
+        ast.parse(source)
+    except SyntaxError:
+        valid = False
+    return valid
+
+
+def decompile(
+    co,
+    bytecode_version: Tuple[int] = PYTHON_VERSION_TRIPLE,
+    out: Optional[TextIO] = sys.stdout,
+    showasm: Optional[str] = None,
+    showast={},
+    timestamp=None,
+    showgrammar=False,
+    source_encoding=None,
+    code_objects={},
+    source_size=None,
+    is_pypy: bool = False,
+    magic_int=None,
+    mapstream=None,
+    do_fragments=False,
+    compile_mode="exec",
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> Any:
+    """
+    ingests and deparses a given code block 'co'
+
+    if `bytecode_version` is None, use the current Python interpreter
+    version.
+
+    Caller is responsible for closing `out` and `mapstream`
+    """
+    if bytecode_version is None:
+        bytecode_version = PYTHON_VERSION_TRIPLE
+
+    # store final output stream for case of error
+    real_out = out or sys.stdout
+
+    def write(s):
+        s += "\n"
+        real_out.write(s)
+
+    assert iscode(co), f"""{co} does not smell like code"""
+
+    co_pypy_str = "PyPy " if is_pypy else ""
+    run_pypy_str = "PyPy " if IS_PYPY else ""
+    sys_version_lines = sys.version.split("\n")
+    if source_encoding:
+        write(f"# -*- coding: {source_encoding} -*-")
+    write(
+        "# decompyle3 version %s\n"
+        "# %sPython bytecode version base %s%s\n# Decompiled from: %sPython %s"
+        % (
+            __version__,
+            co_pypy_str,
+            version_tuple_to_str(bytecode_version),
+            " (%s)" % str(magic_int) if magic_int else "",
+            run_pypy_str,
+            "\n# ".join(sys_version_lines),
+        )
+    )
+    if co.co_filename:
+        write(f"# Embedded file name: {co.co_filename}")
+    if timestamp:
+        write(f"# Compiled at: {datetime.datetime.fromtimestamp(timestamp)}")
+    if source_size:
+        write("# Size of source mod 2**32: %d bytes" % source_size)
+
+    grammar = dict(PARSER_DEFAULT_DEBUG)
+    if showgrammar:
+        grammar["reduce"] = True
+
+    debug_opts = {"asm": showasm, "tree": showast, "grammar": grammar}
+
+    try:
+        if mapstream:
+            if isinstance(mapstream, str):
+                mapstream = _get_outstream(mapstream)
+
+            debug_opts = {"asm": showasm, "tree": showast, "grammar": grammar}
+
+            deparsed = deparse_code_with_map(
+                co=co,
+                out=out,
+                version=bytecode_version,
+                code_objects=code_objects,
+                is_pypy=is_pypy,
+                debug_opts=debug_opts,
+            )
+            header_count = 3 + len(sys_version_lines)
+            if deparsed is not None:
+                linemap = [
+                    (line_no, deparsed.source_linemap[line_no] + header_count)
+                    for line_no in sorted(deparsed.source_linemap.keys())
+                ]
+                mapstream.write("\n\n# %s\n" % linemap)
+        else:
+            if do_fragments:
+                deparse_fn = code_deparse_fragments
+            else:
+                deparse_fn = code_deparse
+            deparsed = deparse_fn(
+                co,
+                out,
+                bytecode_version,
+                is_pypy=is_pypy,
+                debug_opts=debug_opts,
+                compile_mode=compile_mode,
+                start_offset=start_offset,
+                stop_offset=stop_offset,
+            )
+            pass
+        real_out.write("\n")
+        return deparsed
+    except pysource.SourceWalkerError as e:
+        # deparsing failed
+        raise pysource.SourceWalkerError(str(e))
+
+
+def compile_file(source_path: str) -> str:
+    if source_path.endswith(".py"):
+        basename = source_path[:-3]
+    else:
+        basename = source_path
+
+    if hasattr(sys, "pypy_version_info"):
+        bytecode_path = f"{basename}-pypy{version_tuple_to_str()}.pyc"
+    else:
+        bytecode_path = f"{basename}-{version_tuple_to_str()}.pyc"
+
+    print(f"compiling {source_path} to {bytecode_path}")
+    py_compile.compile(source_path, bytecode_path, "exec")
+    return bytecode_path
+
+
+def decompile_file(
+    filename: str,
+    outstream: Optional[TextIO] = None,
+    showasm: Optional[str] = None,
+    showast={},
+    showgrammar=dict(PARSER_DEFAULT_DEBUG),
+    source_encoding=None,
+    mapstream=None,
+    do_fragments=False,
+    start_offset=0,
+    stop_offset=-1,
+) -> Any:
+    """
+    decompile Python byte-code file (.pyc). Return objects to
+    all of the deparsed objects found in `filename`.
+    """
+
+    filename = check_object_path(filename)
+    code_objects = {}
+    version, timestamp, magic_int, co, is_pypy, source_size, _ = load_module(
+        filename, code_objects
+    )
+
+    if isinstance(co, list):
+        deparsed = []
+        for bytecode in co:
+            deparsed.append(
+                decompile(
+                    bytecode,
+                    version,
+                    outstream,
+                    showasm,
+                    showast,
+                    timestamp,
+                    showgrammar,
+                    source_encoding,
+                    code_objects=code_objects,
+                    is_pypy=is_pypy,
+                    magic_int=magic_int,
+                    mapstream=mapstream,
+                    start_offset=start_offset,
+                    stop_offset=stop_offset,
+                ),
+            )
+    else:
+        deparsed = [
+            decompile(
+                co,
+                version,
+                outstream,
+                showasm,
+                showast,
+                timestamp,
+                showgrammar,
+                source_encoding,
+                code_objects=code_objects,
+                source_size=source_size,
+                is_pypy=is_pypy,
+                magic_int=magic_int,
+                mapstream=mapstream,
+                do_fragments=do_fragments,
+                compile_mode="exec",
+                start_offset=start_offset,
+                stop_offset=stop_offset,
+            )
+        ]
+    return deparsed
+
+
+# FIXME: combine into an options parameter
+def main(
+    in_base: str,
+    out_base: Optional[str],
+    compiled_files: list,
+    source_files: list,
+    outfile: Optional[str] = None,
+    showasm: Optional[str] = None,
+    showast={},
+    do_verify: Optional[str] = None,
+    showgrammar: bool = False,
+    source_encoding=None,
+    do_linemaps=False,
+    do_fragments=False,
+    start_offset: int = 0,
+    stop_offset: int = -1,
+) -> Tuple[int, int, int, int]:
+    """
+    in_base	base directory for input files
+    out_base	base directory for output files (ignored when
+    files	list of filenames to be uncompyled (relative to in_base)
+    outfile	write output to this filename (overwrites out_base)
+
+    For redirecting output to
+    - <filename>		outfile=<filename> (out_base is ignored)
+    - files below out_base	out_base=...
+    - stdout			out_base=None, outfile=None
+    """
+    tot_files = okay_files = failed_files = 0
+    verify_failed_files = 0 if do_verify else 0
+    current_outfile = outfile
+    linemap_stream = None
+
+    for source_path in source_files:
+        compiled_files.append(compile_file(source_path))
+
+    for filename in compiled_files:
+        infile = osp.join(in_base, filename)
+        # print("XXX", infile)
+        if not osp.exists(infile):
+            sys.stderr.write(f"File '{infile}' doesn't exist. Skipped\n")
+            continue
+
+        if do_linemaps:
+            linemap_stream = infile + ".pymap"
+            pass
+
+        # print (infile, file=sys.stderr)
+
+        if outfile:  # outfile was given as parameter
+            outstream = _get_outstream(outfile)
+        elif out_base is None:
+            out_base = tempfile.mkdtemp(prefix="py-dis-")
+            if do_verify and filename.endswith(".pyc"):
+                current_outfile = osp.join(out_base, filename[0:-1])
+                outstream = open(current_outfile, "w")
+            else:
+                outstream = sys.stdout
+            if do_linemaps:
+                linemap_stream = sys.stdout
+        else:
+            if filename.endswith(".pyc"):
+                current_outfile = osp.join(out_base, filename[0:-1])
+            else:
+                current_outfile = osp.join(out_base, filename) + "_dis"
+                pass
+            pass
+
+            outstream = _get_outstream(current_outfile)
+
+        # print(current_outfile, file=sys.stderr)
+
+        # Try to decompile the input file.
+        try:
+            deparsed_objects = decompile_file(
+                infile,
+                outstream,
+                showasm,
+                showast,
+                showgrammar,
+                source_encoding,
+                linemap_stream,
+                do_fragments,
+                start_offset,
+                stop_offset,
+            )
+            if do_fragments:
+                for deparsed_object in deparsed_objects:
+                    last_mod = None
+                    offsets = deparsed_object.offsets
+                    for e in sorted(
+                        [k for k in offsets.keys() if isinstance(k[1], int)]
+                    ):
+                        if e[0] != last_mod:
+                            line = "=" * len(e[0])
+                            outstream.write(f"{line}\n{e[0]}\n{line}\n")
+                        last_mod = e[0]
+                        info = offsets[e]
+                        extract_info = deparse_object.extract_node_info(info)
+                        outstream.write(f"{info.node.format().strip()}" + "\n")
+                        outstream.write(extract_info.selectedLine + "\n")
+                        outstream.write(extract_info.markerLine + "\n\n")
+                    pass
+                pass
+            if do_verify:
+                for deparsed_object in deparsed_objects:
+                    deparsed_object.f.close()
+                    if PYTHON_VERSION_TRIPLE[:2] != deparsed_object.version[:2]:
+                        sys.stdout.write(
+                            f"\n# skipping running {deparsed_object.f.name}; it is "
+                            f"{version_tuple_to_str(deparsed_object.version, end=2)}, "
+                            "and we are "
+                            f"{version_tuple_to_str(PYTHON_VERSION_TRIPLE, end=2)}\n"
+                        )
+                    else:
+                        check_type = "syntax check"
+                        if do_verify == "run":
+                            check_type = "run"
+                            result = subprocess.run(
+                                [sys.executable, deparsed_object.f.name],
+                                capture_output=True,
+                            )
+                            valid = result.returncode == 0
+                            output = result.stdout.decode()
+                            if output:
+                                print(output)
+                            pass
+                            if not valid:
+                                print(result.stderr.decode())
+
+                        else:
+                            valid = syntax_check(deparsed_object.f.name)
+
+                        if not valid:
+                            verify_failed_files += 1
+                            sys.stderr.write(
+                                f"\n# {check_type} failed on file {deparsed_object.f.name}\n"
+                            )
+
+                    # sys.stderr.write(f"Ran {deparsed_object.f.name}\n")
+                pass
+            tot_files += 1
+        except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
+            sys.stdout.write("\n")
+            sys.stderr.write(f"\n# file {infile}\n# {e}\n")
+            failed_files += 1
+            tot_files += 1
+        except KeyboardInterrupt:
+            if outfile:
+                outstream.close()
+                os.remove(outfile)
+            sys.stdout.write("\n")
+            sys.stderr.write(f"\nLast file: {infile}   ")
+            raise
+        except RuntimeError as e:
+            sys.stdout.write(f"\n{str(e)}\n")
+            if str(e).startswith("Unsupported Python"):
+                sys.stdout.write("\n")
+                sys.stderr.write(f"\n# Unsupported bytecode in file {infile}\n# {e}\n")
+            else:
+                if outfile:
+                    outstream.close()
+                    os.remove(outfile)
+                sys.stdout.write("\n")
+                sys.stderr.write(f"\nLast file: {infile}   ")
+                raise
+
+        # except:
+        #     failed_files += 1
+        #     if current_outfile:
+        #         outstream.close()
+        #         os.rename(current_outfile, current_outfile + "_failed")
+        #     else:
+        #         sys.stderr.write("\n# %s" % sys.exc_info()[1])
+        #         sys.stderr.write("\n# Can't uncompile %s\n" % infile)
+        else:  # uncompile successful
+            if current_outfile:
+                outstream.close()
+                okay_files += 1
+                pass
+            else:
+                okay_files += 1
+                if not current_outfile:
+                    mess = "\n# okay decompiling"
+                    print(mess, infile)
+        if current_outfile:
+            sys.stdout.write(
+                "%s -- %s\r"
+                % (
+                    infile,
+                    status_msg(
+                        do_verify,
+                        tot_files,
+                        okay_files,
+                        failed_files,
+                        verify_failed_files,
+                    ),
+                )
+            )
+            try:
+                # FIXME: Something is weird with Pypy here
+                sys.stdout.flush()
+            except Exception:
+                pass
+    if current_outfile:
+        sys.stdout.write("\n")
+        try:
+            # FIXME: Something is weird with Pypy here
+            sys.stdout.flush()
+        except Exception:
+            pass
+        pass
+    return tot_files, okay_files, failed_files, verify_failed_files
+
+
+# ---- main ----
+
+
+def status_msg(
+    do_verify: Optional[str],
+    tot_files: int,
+    okay_files: int,
+    failed_files: int,
+    verify_failed_files: Optional[int],
+):
+    if tot_files == 1:
+        if failed_files:
+            return "\n# decompile failed"
+        elif verify_failed_files:
+            return "\n# decompile run verification failed"
+        elif do_verify:
+            return "\n# Successfully decompiled and ran or syntax-checked all files"
+        else:
+            return "\n# Successfully decompiled all files"
+            pass
+        pass
+    mess = f"decompiled {tot_files} files: {okay_files} okay, {failed_files} failed"
+    if do_verify:
+        mess += f", {verify_failed_files} failed verification"
+    return mess

python/py/Lib/site-packages/decompyle3/parsers/.gitignore

@@ -0,0 +1,2 @@
+/.coverage
+/.mypy_cache

python/py/Lib/site-packages/decompyle3/parsers/__init__.py

@@ -0,0 +1,13 @@
+"""Here we have parser grammars for the different Python versions.
+Instead of full grammars, we have full grammars for certain Python versions
+and the others indicate differences between a neighboring version.
+
+For example Python 2.6, 2.7, 3.2, and 3.7 are largely "base" versions
+which work off off parse2.py, parse3.py, and parse37base.py.
+
+Some examples:
+Python 3.3 diffs off of 3.2; 3.1 and 3.0 diff off of 3.2; Python 1.0..Python 2.5 diff off of
+Python 2.6 and Python 3.8 diff off of 3.7
+"""
+from decompyle3.parsers.main import *
+from decompyle3.parsers.treenode import *

python/py/Lib/site-packages/decompyle3/parsers/dump.py

@@ -0,0 +1,55 @@
+#  Copyright (c) 2020-2022 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Common grammar dump and check routine"""
+
+
+def dump_and_check(p, version: tuple, modified_tokens: set) -> None:
+
+    p.dump_grammar()
+    print("=" * 50, "\n")
+
+    p.check_grammar()
+    from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+
+    if PYTHON_VERSION_TRIPLE[:2] == version[:2]:
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
+        from decompyle3.scanner import get_scanner
+
+        s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
+        modified_tokens = set(
+            """JUMP_LOOP CONTINUE RETURN_END_IF COME_FROM
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+               LAMBDA_MARKER RETURN_LAST
+            """.split()
+        )
+        print("\nModified opcodes:", modified_tokens)
+        opcode_set = set(s.opc.opname).union(modified_tokens)
+
+        pseudo_tokens = set(tokens) - opcode_set
+        import re
+
+        pseudo_tokens = set([re.sub(r"_\d+$", "", t) for t in pseudo_tokens])
+        pseudo_tokens = set([re.sub("_CONT$", "", t) for t in pseudo_tokens])
+        pseudo_tokens = set(pseudo_tokens) - opcode_set
+
+        print("\nPseudo tokens:")
+        print(pseudo_tokens)
+        import sys
+
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))

python/py/Lib/site-packages/decompyle3/parsers/main.py

@@ -0,0 +1,208 @@
+#  Copyright (c) 2019-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Common decompyle3 parser routines. From the outside, of the module
+you'll usually import a call something here, such as:
+* get_python_parser().parse(), or
+* python_parser() which does the above
+
+"""
+
+import sys
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from xdis import iscode
+from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE, version_tuple_to_str
+
+from decompyle3.parsers.p37.heads import (
+    Python37ParserEval,
+    Python37ParserExec,
+    Python37ParserExpr,
+    Python37ParserLambda,
+    Python37ParserSingle,
+)
+from decompyle3.parsers.p38.heads import (
+    Python38ParserEval,
+    Python38ParserExec,
+    Python38ParserExpr,
+    Python38ParserLambda,
+    Python38ParserSingle,
+)
+from decompyle3.parsers.p38pypy.heads import (
+    Python38PyPyParserEval,
+    Python38PyPyParserExec,
+    Python38PyPyParserExpr,
+    Python38PyPyParserLambda,
+    Python38PyPyParserSingle,
+)
+from decompyle3.parsers.treenode import SyntaxTree
+from decompyle3.show import maybe_show_asm
+
+
+def parse(p, tokens, customize, is_lambda: bool) -> SyntaxTree:
+    was_lambda = p.is_lambda
+    p.is_lambda = is_lambda
+    p.customize_grammar_rules(tokens, customize)
+    tree = p.parse(tokens)
+    p.is_lambda = was_lambda
+    #  p.cleanup()
+    return tree
+
+
+def get_python_parser(
+    version, debug_parser=PARSER_DEFAULT_DEBUG, compile_mode="exec", is_pypy=False
+):
+    """
+    Returns parser object for Python version 3.7, 3.8, etc. depending on the parameters
+    passed.
+
+    *compile_mode* is one of:
+
+    * "lambda": is for the grammar that can appear in lambda statements.
+    * "eval_expr:" is for grammar "expr" kinds of expressions - this is a smaller kind
+       of "eval" that users only grammar inside lambdas.
+    * "eval:" is for Python eval() kinds of expressions or eval compile mode
+    * "exec": is for Python exec() kind of expressions, or exec compile mode
+    * "single": is python compile "single" compile mode
+
+    See https://docs.python.org/3/library/functions.html#compile for an
+    explanation of the different modes.
+    """
+
+    # FIXME: there has to be a better way...
+    # We could do this as a table lookup, but that would force us
+    # in import all of the parsers all of the time. Perhaps there is
+    # a lazy way of doing the import?
+
+    version = version[:2]
+    if version < (3, 7):
+        raise RuntimeError(f"Unsupported Python version {version}")
+    elif version == (3, 7):
+        if compile_mode == "exec":
+            p = Python37ParserExec(debug_parser=debug_parser)
+        elif compile_mode == "single":
+            p = Python37ParserSingle(debug_parser=debug_parser)
+        elif compile_mode == "lambda":
+            p = Python37ParserLambda(debug_parser=debug_parser)
+        elif compile_mode == "eval":
+            p = Python37ParserEval(debug_parser=debug_parser)
+        elif compile_mode == "expr":
+            p = Python37ParserExpr(debug_parser=debug_parser)
+        else:
+            p = Python37ParserSingle(debug_parser)
+    elif version == (3, 8):
+        if compile_mode == "exec":
+            if is_pypy:
+                p = Python38PyPyParserExec(debug_parser=debug_parser)
+            else:
+                p = Python38ParserExec(debug_parser=debug_parser)
+
+        elif compile_mode == "single":
+            if is_pypy:
+                p = Python38PyPyParserSingle(debug_parser=debug_parser)
+            else:
+                p = Python38ParserSingle(debug_parser=debug_parser)
+        elif compile_mode == "lambda":
+            if is_pypy:
+                p = Python38PyPyParserLambda(debug_parser=debug_parser)
+            else:
+                p = Python38ParserLambda(debug_parser=debug_parser)
+        elif compile_mode == "eval":
+            if is_pypy:
+                p = Python38PyPyParserEval(debug_parser=debug_parser)
+            else:
+                p = Python38ParserEval(debug_parser=debug_parser)
+        elif compile_mode == "expr":
+            if is_pypy:
+                p = Python38PyPyParserExpr(debug_parser=debug_parser)
+            else:
+                p = Python38ParserExpr(debug_parser=debug_parser)
+        elif is_pypy:
+            p = Python38PyPyParserSingle(debug_parser)
+        else:
+            p = Python38ParserSingle(debug_parser)
+
+    elif version > (3, 8):
+        raise RuntimeError(
+            f"""Version {version_tuple_to_str(version)} is not supported."""
+        )
+
+    p.version = version
+    # p.dump_grammar() # debug
+    return p
+
+
+def python_parser(
+    co,
+    version: tuple = PYTHON_VERSION_TRIPLE,
+    out=sys.stdout,
+    showasm: bool = False,
+    parser_debug=PARSER_DEFAULT_DEBUG,
+    compile_mode: str = "exec",
+    is_pypy: bool = False,
+    is_lambda: bool = False,
+) -> SyntaxTree:
+    """
+    Parse a code object to an abstract syntax tree representation.
+
+    :param co:              The code object to parse.
+    :param version:         The python version of this code is from as a float, for
+                            example, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5 etc.
+    :param out:             File like object to write the output to.
+    :param showasm:         Flag which determines whether the disassembled and
+                            ingested code is written to sys.stdout or not.
+    :param parser_debug:    dict containing debug flags for the spark parser.
+    :param compile_mode:    compile mode that we want to parse input `co` as.
+                            This is either "exec", "eval" or, "single".
+    :param is_pypy:         True if ``co`` comes is PyPy code
+    :param is_lambda        True if ``co`` is a lambda expression
+
+    :return: Abstract syntax tree representation of the code object.
+    """
+
+    assert iscode(co)
+    from decompyle3.scanner import get_scanner
+
+    scanner = get_scanner(version, is_pypy)
+    tokens, customize = scanner.ingest(co)
+    maybe_show_asm(showasm, tokens)
+
+    # For heavy grammar debugging
+    # parser_debug = {'rules': True, 'transition': True, 'reduce' : True,
+    #                 'showstack': 'full'}
+    p = get_python_parser(
+        version, parser_debug, compile_mode=compile_mode, is_pypy=IS_PYPY
+    )
+
+    # FIXME: have p.insts update in a better way
+    # modularity is broken here
+    p.insts = scanner.insts
+    p.offset2inst_index = scanner.offset2inst_index
+    p.opc = scanner.opc
+
+    return parse(p, tokens, customize, is_lambda)
+
+
+if __name__ == "__main__":
+
+    def parse_test(co) -> None:
+
+        tree = python_parser(co, (3, 8, 2), showasm=True, is_pypy=IS_PYPY)
+        print(tree)
+        print("+" * 30)
+        return
+
+    parse_test(parse_test.__code__)

python/py/Lib/site-packages/decompyle3/parsers/p37/Makefile

@@ -0,0 +1,13 @@
+# Whatever it is you want to do, it should be forwarded to the
+# to top-level irectories
+PHONY=check all type-check
+all: check
+
+MYPYPATH=../../..
+
+#: Static type checking
+type-check:
+	mypy *.py
+
+%:
+	$(MAKE) -C ../../.. $@

python/py/Lib/site-packages/decompyle3/parsers/p37/__init__.py

@@ -0,0 +1,4 @@
+"""
+Here we have Python 3.7 grammars and associated customization
+for the both full language and the subset used in lambda expressions.
+"""

python/py/Lib/site-packages/decompyle3/parsers/p37/base.py

@@ -0,0 +1,1306 @@
+#  Copyright (c) 2016-2017, 2019-2024 Rocky Bernstein
+
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Python 3.7 base code. We keep non-custom-generated grammar rules out of this file.
+"""
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from spark_parser.spark import rule2str
+
+from decompyle3.parsers.parse_heads import ParserError, PythonBaseParser, nop_func
+from decompyle3.parsers.reduce_check import (
+    and_cond_check,
+    and_invalid,
+    and_not_check,
+    c_tryelsestmt,
+    if_and_elsestmt,
+    if_and_stmt,
+    ifelsestmt,
+    iflaststmt,
+    ifstmt,
+    ifstmts_jump_invalid,
+    lastc_stmt,
+    list_if_not,
+    not_or_check,
+    or_check37_invalid,
+    or_cond_check_invalid,
+    testtrue,
+    tryexcept,
+    while1elsestmt,
+    while1stmt,
+    whilestmt,
+)
+from decompyle3.parsers.treenode import SyntaxTree
+
+
+class Python37BaseParser(PythonBaseParser):
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+
+        self.added_rules = set()
+        super(Python37BaseParser, self).__init__(SyntaxTree, debug=debug_parser)
+        self.new_rules = set()
+
+    @staticmethod
+    def call_fn_name(token):
+        """Customize CALL_FUNCTION to add the number of positional arguments"""
+        if token.attr is not None:
+            return "%s_%i" % (token.kind, token.attr)
+        else:
+            return "%s_0" % (token.kind)
+
+    def add_make_function_rule(self, rule, opname, attr, customize):
+        """Python 3.3 added a an additional LOAD_STR before MAKE_FUNCTION and
+        this has an effect on many rules.
+        """
+        new_rule = rule % "LOAD_STR "
+        self.add_unique_rule(new_rule, opname, attr, customize)
+
+    def custom_build_class_rule(self, opname, i, token, tokens, customize):
+        """
+        # Should the first rule be somehow folded into the 2nd one?
+        build_class ::= LOAD_BUILD_CLASS mkfunc
+                        LOAD_CLASSNAME {expr}^n-1 CALL_FUNCTION_n
+                        LOAD_CONST CALL_FUNCTION_n
+        build_class ::= LOAD_BUILD_CLASS mkfunc
+                        expr
+                        call
+                        CALL_FUNCTION_3
+        """
+        # FIXME: I bet this can be simplified
+        # look for next MAKE_FUNCTION
+        for i in range(i + 1, len(tokens)):
+            if tokens[i].kind.startswith("MAKE_FUNCTION"):
+                break
+            elif tokens[i].kind.startswith("MAKE_CLOSURE"):
+                break
+            pass
+        assert i < len(
+            tokens
+        ), "build_class needs to find MAKE_FUNCTION or MAKE_CLOSURE"
+        assert (
+            tokens[i + 1].kind == "LOAD_STR"
+        ), "build_class expecting CONST after MAKE_FUNCTION/MAKE_CLOSURE"
+        call_fn_tok = None
+        for i in range(i, len(tokens)):
+            if tokens[i].kind.startswith("CALL_FUNCTION"):
+                call_fn_tok = tokens[i]
+                break
+        if not call_fn_tok:
+            raise RuntimeError(
+                "build_class custom rule for %s needs to find CALL_FUNCTION" % opname
+            )
+
+        # customize build_class rule
+        # FIXME: What's the deal with the two rules? Different Python versions?
+        # Different situations? Note that the above rule is based on the CALL_FUNCTION
+        # token found, while this one doesn't.
+        # 3.6+ handling
+        call_function = call_fn_tok.kind
+        if call_function.startswith("CALL_FUNCTION_KW"):
+            self.addRule("classdef ::= build_class_kw store", nop_func)
+            rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s" % (
+                "expr " * (call_fn_tok.attr - 1),
+                call_function,
+            )
+        else:
+            call_function = self.call_fn_name(call_fn_tok)
+            rule = "build_class ::= LOAD_BUILD_CLASS mkfunc %s%s" % (
+                "expr " * (call_fn_tok.attr - 1),
+                call_function,
+            )
+        self.addRule(rule, nop_func)
+        return
+
+    # FIXME FIXME FIXME: The below is an utter mess. Come up with a better
+    # organization for this. For example, arrange organize by opcode base?
+
+    def customize_grammar_rules37(self, tokens, customize):
+        is_pypy = False
+
+        # For a rough break out on the first word. This may
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "CONTINUE",
+                "DELETE",
+                "FORMAT",
+                "GET",
+                "JUMP",
+                "LOAD",
+                "LOOKUP",
+                "MAKE",
+                "RETURN",
+                "RAISE",
+                "SETUP",
+                "UNPACK",
+                "WITH",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get added
+        # unconditionally and the rules are constant. So they need to be done
+        # only once and if we see the opcode a second we don't have to consider
+        # adding more rules.
+        #
+        # Note: BUILD_TUPLE_UNPACK_WITH_CALL gets considered by
+        # default because it starts with BUILD. So we'll set to ignore it from
+        # the start.
+        custom_ops_processed = {"BUILD_TUPLE_UNPACK_WITH_CALL"}
+
+        # A set of instruction operation names that exist in the token stream.
+        # We use this customize the grammar that we create.
+        # 2.6-compatible set comprehensions
+        self.seen_ops = frozenset([t.kind for t in tokens])
+        self.seen_op_basenames = frozenset(
+            [opname[: opname.rfind("_")] for opname in self.seen_ops]
+        )
+
+        # Loop over instructions adding custom grammar rules based on
+        # a specific instruction seen.
+
+        if "PyPy" in customize:
+            is_pypy = True
+            self.addRule(
+                """
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy       ::= expr expr expr store store store
+              assign2_pypy       ::= expr expr store store
+              """,
+                nop_func,
+            )
+
+        n = len(tokens)
+
+        # Determine if we have an iteration CALL_FUNCTION_1.
+        has_get_iter_call_function1 = False
+        for i, token in enumerate(tokens):
+            if (
+                token == "GET_ITER"
+                and i < n - 2
+                and self.call_fn_name(tokens[i + 1]) == "CALL_FUNCTION_1"
+            ):
+                has_get_iter_call_function1 = True
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # The order of opname listed is roughly sorted below
+
+            if opname == "LOAD_ASSERT" and "PyPy" in customize:
+                rules_str = """
+                stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "BEFORE_ASYNC_WITH":
+                rules_str = """
+                   stmt            ::= async_with_stmt
+                   stmt            ::= async_with_as_stmt
+                   c_stmt          ::= c_async_with_stmt
+                """
+
+                if self.version < (3, 8):
+                    rules_str += """
+                      stmt                 ::= async_with_stmt SETUP_ASYNC_WITH
+                      c_stmt               ::= c_async_with_stmt SETUP_ASYNC_WITH
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              c_suite_stmts_opt
+                                              POP_BLOCK LOAD_CONST
+                                              async_with_post
+                      async_with_as_stmt   ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                    """
+                else:
+                    rules_str += """
+                      async_with_pre       ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
+                      async_with_post      ::= BEGIN_FINALLY COME_FROM_ASYNC_WITH
+                                               WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               WITH_CLEANUP_FINISH END_FINALLY
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts
+                                               POP_BLOCK
+                                               BEGIN_FINALLY
+                                               WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               WITH_CLEANUP_FINISH POP_FINALLY LOAD_CONST RETURN_VALUE
+                                               COME_FROM_ASYNC_WITH
+                                               WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               WITH_CLEANUP_FINISH END_FINALLY
+                      c_async_with_stmt   ::= async_with_stmt
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                    """
+                self.addRule(rules_str, nop_func)
+
+            elif opname_base == "BUILD_CONST_KEY_MAP":
+                kvlist_n = "expr " * (token.attr)
+                rule = """
+                   expr ::= dict
+                   dict ::= %sLOAD_CONST %s
+                """ % (
+                    kvlist_n,
+                    opname,
+                )
+                self.addRule(rule, nop_func)
+
+            elif opname.startswith("BUILD_LIST_UNPACK"):
+                v = token.attr
+                rule = "build_list_unpack ::= %s%s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+                rule = "expr ::= build_list_unpack"
+                self.addRule(rule, nop_func)
+
+            elif opname_base in ("BUILD_MAP", "BUILD_MAP_UNPACK"):
+
+                if opname == "BUILD_MAP_UNPACK":
+                    self.addRule(
+                        """
+                        expr        ::= dict_unpack
+                        dict_unpack ::= expr BUILD_MAP_UNPACK
+                        """,
+                        nop_func,
+                    )
+                    pass
+                elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
+                    v = token.attr
+                    rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
+                    self.addRule(rule, nop_func)
+
+                kvlist_n = "kvlist_%s" % token.attr
+                if opname == "BUILD_MAP_n":
+                    # PyPy sometimes has no count. Sigh.
+                    rule = (
+                        "dict_comp_func ::= BUILD_MAP_n LOAD_ARG for_iter store "
+                        "comp_iter JUMP_LOOP RETURN_VALUE RETURN_LAST"
+                    )
+                    self.add_unique_rule(rule, "dict_comp_func", 1, customize)
+
+                    kvlist_n = "kvlist_n"
+                    rule = "kvlist_n ::=  kvlist_n kv3"
+                    self.add_unique_rule(rule, "kvlist_n", 0, customize)
+                    rule = "kvlist_n ::="
+                    self.add_unique_rule(rule, "kvlist_n", 1, customize)
+                    rule = """
+                       expr ::= dict
+                       dict ::=  BUILD_MAP_n kvlist_n
+                    """
+
+                if not opname.startswith("BUILD_MAP_WITH_CALL"):
+                    # FIXME: Use the attr
+                    # so this doesn't run into exponential parsing time.
+                    if opname.startswith("BUILD_MAP_UNPACK"):
+                        # FIXME: start here. The LHS should be dict_unpack, not dict.
+                        # FIXME: really we need a combination of dict_entry-like things.
+                        # It just so happens the most common case is not to mix
+                        # dictionary comphensions with dictionary, elements
+                        if "LOAD_DICTCOMP" in self.seen_ops:
+                            rule = """
+                               expr ::= dict_comp
+                               expr ::= dict
+                               dict ::= %s%s
+                            """ % (
+                                "dict_comp " * token.attr,
+                                opname,
+                            )
+                            self.addRule(rule, nop_func)
+                        rule = """
+                         expr        ::= dict_unpack
+                         dict_unpack ::= %s%s
+                         """ % (
+                            "expr " * token.attr,
+                            opname,
+                        )
+                    else:
+                        rule = "%s ::= %s %s" % (
+                            kvlist_n,
+                            "expr " * (token.attr * 2),
+                            opname,
+                        )
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = (
+                            """
+                        expr ::= dict
+                        dict ::=  %s
+                        """
+                            % kvlist_n
+                        )
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
+                v = token.attr
+                rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+
+            elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
+                v = token.attr
+                rule = (
+                    "build_tuple_unpack_with_call ::= "
+                    + "expr1024 " * int(v // 1024)
+                    + "expr32 " * int((v // 32) % 32)
+                    + "expr " * (v % 32)
+                    + opname
+                )
+                self.addRule(rule, nop_func)
+                rule = "starred ::= %s %s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+
+            elif opname_base in (
+                "BUILD_LIST",
+                "BUILD_SET",
+                "BUILD_TUPLE",
+                "BUILD_TUPLE_UNPACK",
+            ):
+                collection_size = token.attr
+
+                is_LOAD_CLOSURE = False
+                if opname_base == "BUILD_TUPLE":
+                    # If is part of a "load_closure", then it is not part of a
+                    # "list".
+                    is_LOAD_CLOSURE = True
+                    for j in range(collection_size):
+                        if tokens[i - j - 1].kind != "LOAD_CLOSURE":
+                            is_LOAD_CLOSURE = False
+                            break
+                    if is_LOAD_CLOSURE:
+                        rule = "load_closure ::= %s%s" % (
+                            ("LOAD_CLOSURE " * collection_size),
+                            opname,
+                        )
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                if not is_LOAD_CLOSURE or collection_size == 0:
+                    # We do this complicated test to speed up parsing of
+                    # pathelogically long literals, especially those over 1024.
+                    thousands = collection_size // 1024
+                    thirty32s = (collection_size // 32) % 32
+                    if thirty32s > 0:
+                        rule = "expr32 ::=%s" % (" expr" * 32)
+                        self.add_unique_rule(
+                            rule, opname_base, collection_size, customize
+                        )
+                        pass
+                    if thousands > 0:
+                        self.add_unique_rule(
+                            "expr1024 ::=%s" % (" expr32" * 32),
+                            opname_base,
+                            collection_size,
+                            customize,
+                        )
+                        pass
+                    collection = opname_base[opname_base.find("_") + 1 :].lower()
+                    rule = (
+                        ("%s ::= " % collection)
+                        + "expr1024 " * thousands
+                        + "expr32 " * thirty32s
+                        + "expr " * (collection_size % 32)
+                        + opname
+                    )
+                    self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
+                    continue
+                continue
+            elif opname_base == "BUILD_SLICE":
+                if token.attr == 2:
+                    self.add_unique_rules(
+                        [
+                            "expr ::= slice2",
+                            "slice2 ::= expr expr BUILD_SLICE_2",
+                        ],
+                        customize,
+                    )
+                else:
+                    assert token.attr == 3, (
+                        "BUILD_SLICE value must be 2 or 3; is %s" % token.attr
+                    )
+                    self.add_unique_rules(
+                        [
+                            "expr   ::= slice3",
+                            "slice3 ::= expr expr expr BUILD_SLICE_3",
+                        ],
+                        customize,
+                    )
+
+            elif opname.startswith("BUILD_STRING"):
+                v = token.attr
+                rules_str = """
+                    expr                 ::= joined_str
+                    joined_str           ::= %sBUILD_STRING_%d
+                """ % (
+                    "expr " * v,
+                    v,
+                )
+                self.add_unique_doc_rules(rules_str, customize)
+                if "FORMAT_VALUE_ATTR" in self.seen_ops:
+                    rules_str = """
+                      formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
+                      expr                 ::= formatted_value_attr
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname in frozenset(
+                (
+                    "CALL_FUNCTION",
+                    "CALL_FUNCTION_EX",
+                    "CALL_FUNCTION_EX_KW",
+                    "CALL_FUNCTION_VAR",
+                    "CALL_FUNCTION_VAR_KW",
+                )
+            ) or opname.startswith("CALL_FUNCTION_KW"):
+
+                if opname == "CALL_FUNCTION" and token.attr == 1:
+                    rule = """
+                     expr         ::= dict_comp
+                     dict_comp    ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr
+                                      GET_ITER CALL_FUNCTION_1
+                    classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
+                    classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
+                    """
+                    self.addRule(rule, nop_func)
+
+                self.custom_classfunc_rule(opname, token, customize, tokens[i + 1])
+                # Note: don't add to custom_ops_processed.
+
+            elif opname_base == "CALL_METHOD":
+                # PyPy and Python 3.7+ only - DRY with parse2
+
+                if opname == "CALL_METHOD_KW":
+                    args_kw = token.attr
+                    rules_str = """
+                         expr ::= call_kw_pypy37
+                         pypy_kw_keys ::= LOAD_CONST
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+                    rule = (
+                        "call_kw_pypy37 ::= expr "
+                        + ("expr " * args_kw)
+                        + " pypy_kw_keys "
+                        + opname
+                    )
+                else:
+                    args_pos, args_kw = self.get_pos_kw(token)
+                    # number of apply equiv arguments:
+                    nak = (len(opname_base) - len("CALL_METHOD")) // 3
+                    rule = (
+                        "call ::= expr "
+                        + ("expr " * args_pos)
+                        + ("kwarg " * args_kw)
+                        + "expr " * nak
+                        + opname
+                    )
+
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "CONTINUE":
+                self.addRule("continue ::= CONTINUE", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "CONTINUE_LOOP":
+                self.addRule("continue ::= CONTINUE_LOOP", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_ATTR":
+                self.addRule("delete ::= expr DELETE_ATTR", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_DEREF":
+                self.addRule(
+                    """
+                   stmt           ::= del_deref_stmt
+                   del_deref_stmt ::= DELETE_DEREF
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_SUBSCR":
+                self.addRule(
+                    """
+                    delete ::= delete_subscript
+                    delete_subscript ::= expr expr DELETE_SUBSCR
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "FORMAT_VALUE":
+                rules_str = """
+                    expr              ::= formatted_value1
+                    formatted_value1  ::= expr FORMAT_VALUE
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "FORMAT_VALUE_ATTR":
+                rules_str = """
+                expr              ::= formatted_value2
+                formatted_value2  ::= expr expr FORMAT_VALUE_ATTR
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "GET_AITER":
+                self.addRule(
+                    """
+                    stmt                ::= genexpr_func_async
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "JUMP_IF_NOT_DEBUG":
+                v = token.attr
+                self.addRule(
+                    """
+                    stmt        ::= assert_pypy
+                    stmt        ::= assert2_pypy", nop_func)
+                    assert_pypy ::=  JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM,
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "LOAD_BUILD_CLASS":
+                self.custom_build_class_rule(opname, i, token, tokens, customize)
+                # Note: don't add to custom_ops_processed.
+            elif opname == "LOAD_CLASSDEREF":
+                # Python 3.4+
+                self.addRule("expr ::= LOAD_CLASSDEREF", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "LOAD_CLASSNAME":
+                self.addRule("expr ::= LOAD_CLASSNAME", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "LOAD_DICTCOMP":
+                if has_get_iter_call_function1:
+                    rule_pat = (
+                        "dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
+                        "GET_ITER CALL_FUNCTION_1"
+                    )
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    pass
+                custom_ops_processed.add(opname)
+            elif opname == "LOAD_LISTCOMP":
+                self.add_unique_rule(
+                    "expr ::= list_comp", opname, token.attr, customize
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "LOAD_NAME":
+                if (
+                    token.attr == "__annotations__"
+                    and "SETUP_ANNOTATIONS" in self.seen_ops
+                ):
+                    token.kind = "LOAD_ANNOTATION"
+                    self.addRule(
+                        """
+                        stmt       ::= SETUP_ANNOTATIONS
+                        stmt       ::= ann_assign
+                        ann_assign ::= expr LOAD_ANNOTATION LOAD_STR STORE_SUBSCR
+                        """,
+                        nop_func,
+                    )
+                    pass
+            elif opname == "LOAD_SETCOMP":
+                # Should this be generalized and put under MAKE_FUNCTION?
+                if has_get_iter_call_function1:
+                    self.addRule("expr ::= set_comp", nop_func)
+                    rule_pat = (
+                        "set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr "
+                        "GET_ITER CALL_FUNCTION_1"
+                    )
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    pass
+                custom_ops_processed.add(opname)
+            elif opname == "LOOKUP_METHOD":
+                # A PyPy speciality - DRY with parse3
+                self.addRule(
+                    """
+                             expr      ::= attribute
+                             attribute ::= expr LOOKUP_METHOD
+                             """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname.startswith("MAKE_CLOSURE"):
+                # DRY with MAKE_FUNCTION
+                # Note: this probably doesn't handle kwargs proprerly
+
+                if opname == "MAKE_CLOSURE_0" and "LOAD_DICTCOMP" in self.seen_ops:
+                    # Is there something general going on here?
+                    # Note that 3.6+ doesn't do this, but we'll remove
+                    # this rule in parse36.py
+                    rule = """
+                        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
+                                      MAKE_CLOSURE_0 expr
+                                      GET_ITER CALL_FUNCTION_1
+                    """
+                    self.addRule(rule, nop_func)
+
+                args_pos, args_kw, annotate_args = token.attr
+
+                # FIXME: Fold test  into add_make_function_rule
+                j = 2
+                if is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"):
+                    rule_pat = """
+                                expr        ::= lambda_body
+                                lambda_body ::= %sload_closure LOAD_LAMBDA %%s%s
+                               """ % (
+                        "expr " * args_pos,
+                        opname,
+                    )
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+
+                if has_get_iter_call_function1:
+                    rule_pat = (
+                        "generator_exp ::= %sload_closure load_genexpr %%s%s expr "
+                        "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
+                    )
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+
+                    if has_get_iter_call_function1:
+                        if is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"):
+                            # In the tokens we saw:
+                            #   LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or
+                            #   LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or
+                            #   and have GET_ITER CALL_FUNCTION_1
+                            # Todo: For Pypy we need to modify this slightly
+                            rule_pat = (
+                                "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
+                                "GET_ITER CALL_FUNCTION_1"
+                                % ("expr " * args_pos, opname)
+                            )
+                            self.add_make_function_rule(
+                                rule_pat, opname, token.attr, customize
+                            )
+                        if is_pypy or (i >= j and tokens[i - j] == "LOAD_SETCOMP"):
+                            rule_pat = (
+                                "set_comp ::= %sload_closure LOAD_SETCOMP %%s%s expr "
+                                "GET_ITER CALL_FUNCTION_1"
+                                % ("expr " * args_pos, opname)
+                            )
+                            self.add_make_function_rule(
+                                rule_pat, opname, token.attr, customize
+                            )
+                        if is_pypy or (i >= j and tokens[i - j] == "LOAD_DICTCOMP"):
+                            self.add_unique_rule(
+                                "dict_comp ::= %sload_closure LOAD_DICTCOMP %s "
+                                "expr GET_ITER CALL_FUNCTION_1"
+                                % ("expr " * args_pos, opname),
+                                opname,
+                                token.attr,
+                                customize,
+                            )
+
+                if args_kw > 0:
+                    kwargs_str = "kwargs "
+                else:
+                    kwargs_str = ""
+
+                rule = "mkfunc ::= %s%s%s load_closure LOAD_CODE LOAD_STR %s" % (
+                    "expr " * args_pos,
+                    kwargs_str,
+                    "expr " * annotate_args,
+                    opname,
+                )
+
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+                if args_kw == 0:
+                    rule = "mkfunc ::= %sload_closure load_genexpr %s" % (
+                        "expr " * args_pos,
+                        opname,
+                    )
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+
+                pass
+            elif opname_base.startswith("MAKE_FUNCTION"):
+                args_pos, args_kw, annotate_args, closure = token.attr
+                stack_count = args_pos + args_kw + annotate_args
+                if closure:
+                    if args_pos:
+                        rule = """
+                             expr        ::= lambda_body
+                             lambda_body ::= %s%s%s%s
+                             """ % (
+                            "expr " * stack_count,
+                            "load_closure " * closure,
+                            "BUILD_TUPLE_1 LOAD_LAMBDA LOAD_STR ",
+                            opname,
+                        )
+                    else:
+                        rule = """
+                             expr        ::= lambda_body
+                             lambda_body ::= %s%s%s""" % (
+                            "load_closure " * closure,
+                            "LOAD_LAMBDA LOAD_STR ",
+                            opname,
+                        )
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+
+                else:
+                    rule = """
+                         expr        ::= lambda_body
+                         lambda_body ::= %sLOAD_LAMBDA LOAD_STR %s""" % (
+                        ("expr " * stack_count),
+                        opname,
+                    )
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+
+                rule = "mkfunc ::= %s%s%s%s" % (
+                    "expr " * stack_count,
+                    "load_closure " * closure,
+                    "LOAD_CODE LOAD_STR ",
+                    opname,
+                )
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+                if has_get_iter_call_function1:
+                    rule_pat = (
+                        "generator_exp ::= %sload_genexpr %%s%s expr "
+                        "GET_ITER CALL_FUNCTION_1"
+                    ) % ("expr " * args_pos, opname)
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    rule_pat = """
+                           expr          ::= generator_exp
+                           generator_exp ::= %sload_closure load_genexpr %%s%s expr
+                           GET_ITER CALL_FUNCTION_1""" % (
+                        "expr " * args_pos,
+                        opname,
+                    )
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    if is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LISTCOMP"):
+                        # 3.6+ sometimes bundles all of the
+                        # 'exprs' in the rule above into a
+                        # tuple.
+                        rule_pat = (
+                            "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
+                            "expr GET_ITER CALL_FUNCTION_1" % (opname,)
+                        )
+                        self.add_make_function_rule(
+                            rule_pat, opname, token.attr, customize
+                        )
+                        rule_pat = (
+                            "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
+                        )
+                        self.add_make_function_rule(
+                            rule_pat, opname, token.attr, customize
+                        )
+
+                if is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LAMBDA"):
+                    rule_pat = """
+                        expr        ::= lambda_body
+                        lambda_body ::= %s%sLOAD_LAMBDA %%s%s
+                        """ % (
+                        ("expr " * args_pos),
+                        ("kwarg " * args_kw),
+                        opname,
+                    )
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                continue
+
+                args_pos, args_kw, annotate_args, closure = token.attr
+
+                j = 2
+
+                if has_get_iter_call_function1:
+                    rule_pat = (
+                        "generator_exp ::= %sload_genexpr %%s%s expr "
+                        "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
+                    )
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+
+                    if is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"):
+                        # In the tokens we saw:
+                        #   LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or
+                        #   LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or
+                        #   and have GET_ITER CALL_FUNCTION_1
+                        # Todo: For Pypy we need to modify this slightly
+                        rule_pat = (
+                            "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
+                        )
+                        self.add_make_function_rule(
+                            rule_pat, opname, token.attr, customize
+                        )
+
+                # FIXME: Fold test  into add_make_function_rule
+                if is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"):
+                    rule_pat = """
+                        expr        ::= lambda_body
+                        lambda_body ::= %s%sLOAD_LAMBDA %%s%s
+                        """ % (
+                        ("expr " * args_pos),
+                        ("kwarg " * args_kw),
+                        opname,
+                    )
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+
+                if args_kw == 0:
+                    kwargs = "no_kwargs"
+                    self.add_unique_rule("no_kwargs ::=", opname, token.attr, customize)
+                else:
+                    kwargs = "kwargs"
+
+                # positional args before keyword args
+                rule = "mkfunc ::= %s%s %s%s" % (
+                    "expr " * args_pos,
+                    kwargs,
+                    "LOAD_CODE LOAD_STR ",
+                    opname,
+                )
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "RETURN_VALUE_LAMBDA":
+                self.addRule(
+                    """
+                    return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_0":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt0
+                    last_stmt  ::= raise_stmt0
+                    raise_stmt0 ::= RAISE_VARARGS_0
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_1":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt1
+                    last_stmt  ::= raise_stmt1
+                    raise_stmt1 ::= expr RAISE_VARARGS_1
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_2":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt2
+                    last_stmt  ::= raise_stmt2
+                    raise_stmt2 ::= expr expr RAISE_VARARGS_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "SETUP_EXCEPT":
+                self.addRule(
+                    """
+                    try_except     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler opt_come_from_except
+                    c_try_except   ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler opt_come_from_except
+                    stmt           ::= tryelsestmt3
+                    tryelsestmt3   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler COME_FROM else_suite
+                                       opt_come_from_except
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_from_except_clauses
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_froms
+
+                    c_stmt         ::= c_tryelsestmt
+                    c_tryelsestmt  ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler
+                                       come_any_froms else_suitec
+                                       come_from_except_clauses
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "WITH_CLEANUP_START":
+                rules_str = """
+                  stmt        ::= with_null
+                  with_null   ::= with_suffix
+                  with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                self.addRule(rules_str, nop_func)
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                  stmt        ::= with
+                  stmt        ::= with_as_pass
+                  stmt        ::= with_as
+                  c_stmt      ::= c_with
+
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr SETUP_WITH POP_TOP
+                                  suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+
+                  with_as  ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                  with_as  ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                  with_as  ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                    with      ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   with_suffix
+                    """
+                else:
+                    rules_str += """
+                     # A return at the end of a withas stmt can be this.
+                     # FIXME: should this be a different kind of return?
+                     return      ::= return_expr POP_BLOCK
+                                     ROT_TWO
+                                     BEGIN_FINALLY
+                                     WITH_CLEANUP_START
+                                     WITH_CLEANUP_FINISH
+                                     POP_FINALLY
+                                     RETURN_VALUE
+
+                      with       ::= expr
+                                     SETUP_WITH POP_TOP suite_stmts_opt
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                     with_suffix
+
+
+                      with_as ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+
+                      with_as ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK BEGIN_FINALLY COME_FROM_WITH
+                                     with_suffix
+
+                      # with_as ::= expr SETUP_WITH store suite_stmts
+                      #                COME_FROM expr COME_FROM POP_BLOCK ROT_TWO
+                      #                BEGIN_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                      #                POP_FINALLY RETURN_VALUE COME_FROM_WITH
+                      #                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+                      with         ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                       BEGIN_FINALLY COME_FROM_WITH
+                                       with_suffix
+                    """
+                self.addRule(rules_str, nop_func)
+
+            elif opname_base in ("UNPACK_EX",):
+                before_count, after_count = token.attr
+                rule = (
+                    """
+                        store  ::= unpack
+                        unpack ::= """
+                    + opname
+                    + " store" * (before_count + after_count + 1)
+                )
+                self.addRule(rule, nop_func)
+
+            elif opname_base == "UNPACK_SEQUENCE":
+                rule = (
+                    """
+                    store  ::= unpack
+                    unpack ::= """
+                    + opname
+                    + " store" * token.attr
+                )
+                self.addRule(rule, nop_func)
+            pass
+
+        self.reduce_check_table = {
+            "ifstmts_jump": ifstmts_jump_invalid,
+            "and": and_invalid,
+            "and_cond": and_cond_check,
+            "and_not": and_not_check,
+            "if_and_stmt": if_and_stmt,
+            "if_and_elsestmtc": if_and_elsestmt,
+            "ifelsestmt": ifelsestmt,
+            "ifelsestmtc": ifelsestmt,
+            "iflaststmt": iflaststmt,
+            "iflaststmtc": iflaststmt,
+            "if_not_stmtc": ifstmt,
+            "ifstmt": ifstmt,
+            "ifstmtc": ifstmt,
+            "lastc_stmt": lastc_stmt,
+            "list_if_not": list_if_not,
+            "not_or": not_or_check,
+            "or": or_check37_invalid,
+            "or_cond": or_cond_check_invalid,
+            "testtrue": testtrue,
+            "testfalsec": testtrue,
+            "while1elsestmt": while1elsestmt,
+            "while1stmt": while1stmt,
+            "whilestmt": whilestmt,
+            "c_tryelsestmt": c_tryelsestmt,
+            "c_try_except": tryexcept,
+        }
+
+        self.check_reduce["and"] = "AST"
+        self.check_reduce["and_cond"] = "AST"
+        self.check_reduce["and_not"] = "AST"
+        self.check_reduce["annotate_tuple"] = "tokens"
+        self.check_reduce["aug_assign1"] = "AST"
+        self.check_reduce["aug_assign2"] = "AST"
+        self.check_reduce["c_try_except"] = "AST"
+        self.check_reduce["c_tryelsestmt"] = "AST"
+        self.check_reduce["if_and_stmt"] = "AST"
+        self.check_reduce["if_and_elsestmtc"] = "AST"
+        self.check_reduce["if_not_stmtc"] = "AST"
+        self.check_reduce["ifelsestmt"] = "AST"
+        self.check_reduce["ifelsestmtc"] = "AST"
+        self.check_reduce["iflaststmt"] = "AST"
+        self.check_reduce["iflaststmtc"] = "AST"
+        self.check_reduce["ifstmt"] = "AST"
+        self.check_reduce["ifstmtc"] = "AST"
+        self.check_reduce["ifstmts_jump"] = "AST"
+        self.check_reduce["ifstmts_jumpc"] = "AST"
+        self.check_reduce["import_as37"] = "tokens"
+        self.check_reduce["import_from37"] = "AST"
+        self.check_reduce["import_from_as37"] = "tokens"
+        self.check_reduce["lastc_stmt"] = "tokens"
+        self.check_reduce["list_if_not"] = "AST"
+        self.check_reduce["while1elsestmt"] = "tokens"
+        self.check_reduce["while1stmt"] = "tokens"
+        self.check_reduce["whilestmt"] = "tokens"
+        self.check_reduce["not_or"] = "AST"
+        self.check_reduce["or"] = "AST"
+        self.check_reduce["or_cond"] = "tokens"
+        self.check_reduce["testtrue"] = "tokens"
+        self.check_reduce["testfalsec"] = "tokens"
+        return
+
+    def custom_classfunc_rule(self, opname, token, customize, next_token):
+        """
+        call ::= expr {expr}^n CALL_FUNCTION_n
+        call ::= expr {expr}^n CALL_FUNCTION_VAR_n
+        call ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n
+        call ::= expr {expr}^n CALL_FUNCTION_KW_n
+
+        classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc {expr}^n-1 CALL_FUNCTION_n
+        """
+        args_pos, args_kw = self.get_pos_kw(token)
+
+        # Additional exprs for * and ** args:
+        #  0 if neither
+        #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
+        #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
+        # Yes, this computation based on instruction name is a little bit hoaky.
+        nak = (len(opname) - len("CALL_FUNCTION")) // 3
+        uniq_param = args_kw + args_pos
+
+        if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
+            rule = (
+                "async_call ::= expr "
+                + ("expr " * args_pos)
+                + ("kwarg " * args_kw)
+                + "expr " * nak
+                + token.kind
+                + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
+            )
+            self.add_unique_rule(rule, token.kind, uniq_param, customize)
+            self.add_unique_rule(
+                "expr ::= async_call", token.kind, uniq_param, customize
+            )
+
+        if opname.startswith("CALL_FUNCTION_VAR"):
+            token.kind = self.call_fn_name(token)
+            if opname.endswith("KW"):
+                kw = "expr "
+            else:
+                kw = ""
+            rule = (
+                "call ::= expr expr "
+                + ("expr " * args_pos)
+                + ("kwarg " * args_kw)
+                + kw
+                + token.kind
+            )
+
+            # Note: semantic actions make use of the fact of whether "args_pos"
+            # zero or not in creating a template rule.
+            self.add_unique_rule(rule, token.kind, args_pos, customize)
+        else:
+            token.kind = self.call_fn_name(token)
+            uniq_param = args_kw + args_pos
+
+            # Note: 3.5+ have subclassed this method; so we don't handle
+            # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here.
+            rule = (
+                "call ::= expr "
+                + ("expr " * args_pos)
+                + ("kwarg " * args_kw)
+                + "expr " * nak
+                + token.kind
+            )
+
+            self.add_unique_rule(rule, token.kind, uniq_param, customize)
+
+            if "LOAD_BUILD_CLASS" in self.seen_ops:
+                if (
+                    next_token == "CALL_FUNCTION"
+                    and next_token.attr == 1
+                    and args_pos > 1
+                ):
+                    rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % (
+                        ("expr " * (args_pos - 1)),
+                        opname,
+                        args_pos,
+                    )
+                    self.add_unique_rule(rule, token.kind, uniq_param, customize)
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        lhs = rule[0]
+        n = len(tokens)
+        last = min(last, n - 1)
+        fn = self.reduce_check_table.get(lhs, None)
+        try:
+            if fn:
+                return fn(self, lhs, n, rule, ast, tokens, first, last)
+        except Exception:
+            import sys
+            import traceback
+
+            print(
+                f"Exception in {fn.__name__} {sys.exc_info()[1]}\n"
+                + f"rule: {rule2str(rule)}\n"
+                + f"offsets {tokens[first].offset} .. {tokens[last].offset}"
+            )
+            print(traceback.print_tb(sys.exc_info()[2], -1))
+            raise ParserError(tokens[last], tokens[last].off2int(), self.debug["rules"])
+
+        if lhs in ("aug_assign1", "aug_assign2") and ast[0][0] == "and":
+            return True
+        elif lhs == "annotate_tuple":
+            return not isinstance(tokens[first].attr, tuple)
+        elif lhs == "import_from37":
+            importlist37 = ast[3]
+            alias37 = importlist37[0]
+            if importlist37 == "importlist37" and alias37 == "alias37":
+                store = alias37[1]
+                assert store == "store"
+                return alias37[0].attr != store[0].attr
+            return False
+        elif lhs == "import_as37":
+            return tokens[first + 1].pattr is not None
+        elif lhs == "import_from_as37":
+            return tokens[first + 1].pattr is None
+
+        return False

python/py/Lib/site-packages/decompyle3/parsers/p37/full.py

@@ -0,0 +1,989 @@
+#  Copyright (c) 2020-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+spark grammar for Python 3.7
+"""
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from decompyle3.parsers.p37.lambda_expr import Python37LambdaParser
+
+
+class Python37Parser(Python37LambdaParser):
+    def __init__(self, start_symbol: str = "stmts", debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python37Parser, self).__init__(start_symbol, debug_parser)
+        self.customized = {}
+
+    ###############################################
+    #  Python 3.7 grammar rules with statements
+    ###############################################
+    def p_start37(self, args):
+        """
+        # The start or goal symbol
+        stmts ::= sstmt+
+        """
+
+    def p_eval_mode(self, args):
+        """
+        # eval-mode compilation.  Single-mode interactive compilation
+        # adds another rule.
+        expr_stmt ::= expr POP_TOP
+        """
+
+    def p_stmt_loop(self, args):
+        """
+        #########################################################
+        # Higher-level rules for statements in some sort of loop.
+        #
+        # Loops allow "continue" and "break" at the Python level.
+        # At the bytecode level, there are backward jumps.
+        #
+        # Productions that can appear outside of
+        # loop should be derivable from inside a loop, but
+        # not necessarily vice versa, such as for "BREAK"
+        # and "CONTINUE" (pseudo or real) instructions.
+        #
+        # Nonterminal names that start "c_" or end in "c", indicates
+        # rule that can only to appear in a loop.
+        # (The "c" stands for "continue". It is
+        # a little bit historical. "l" was considered but can
+        # be confused with "last".)
+        #
+        #########################################################
+        c_stmts ::= _stmts
+        c_stmts ::= _stmts lastc_stmt
+        c_stmts ::= lastc_stmt
+        c_stmts ::= continues
+        c_stmts ::= c_stmt+
+        c_stmts ::= c_returns
+
+        # Additional statements that *must* be in a loop
+        c_stmt  ::= break
+        c_stmt  ::= continue
+
+        # If statement inside a loop. The RHS may have looping jumps in them.
+        c_stmt  ::= ifstmtc
+        c_stmt  ::= if_not_stmtc
+        c_stmt  ::= if_and_elsestmtc
+        c_stmt  ::= ifelsestmtc
+        c_stmt  ::= c_tryfinallystmt
+
+        c_stmt  ::= c_try_except
+        c_stmt  ::= c_try_except36
+        c_stmt  ::= stmt
+
+        c_stmts_opt ::= c_stmts
+        c_stmts_opt ::= pass
+
+        else_suitec ::= c_stmts
+        else_suitec ::= c_returns
+        else_suitec ::= suite_stmts
+
+        c_suite_stmts     ::= c_stmts
+        c_suite_stmts     ::= suite_stmts
+        c_suite_stmts_opt ::= c_suite_stmts
+        c_suite_stmts_opt ::= suite_stmts_opt
+
+        c_returns         ::= c_stmts return
+        c_returns         ::= returns
+
+        c_except  ::=  POP_TOP POP_TOP POP_TOP c_stmts_opt POP_EXCEPT jump
+        c_except  ::=  POP_TOP POP_TOP POP_TOP c_returns
+
+        # FIXME regularize name c_last_stmt, etc.
+        # Do we really need these?
+        lastc_stmt ::= forelselaststmtc
+        lastc_stmt ::= iflaststmtc
+
+        # FIXME: Do we need these?
+        lastc_stmt ::= ifelsestmtc
+        lastc_stmt ::= tryelsestmtc
+        """
+
+    def p_stmt(self, args):
+        """
+        pass ::=
+
+        stmts_opt ::= stmts
+        stmts_opt ::= pass
+
+        stmts  ::= stmt+
+        stmts  ::= stmts last_stmt
+        _stmts ::= stmts
+
+        suite_stmts ::= _stmts
+        suite_stmts ::= returns
+
+        suite_stmts_opt ::= suite_stmts
+
+        # passtmt is needed for semantic actions to add "pass"
+        suite_stmts_opt ::= pass
+
+        else_suite_opt ::= else_suite
+        else_suite_opt ::= pass
+
+        else_suite ::= suite_stmts
+        else_suite ::= returns
+
+        stmt ::= classdef
+        stmt ::= expr_stmt
+
+        stmt ::= ifstmt
+        stmt ::= if_or_stmt
+        stmt ::= if_and_stmt
+        stmt ::= ifelsestmt
+        stmt ::= if_or_not_elsestmt
+
+        stmt ::= whilestmt
+        stmt ::= while1stmt
+        stmt ::= whileelsestmt
+        stmt ::= while1elsestmt
+        stmt ::= for
+        stmt ::= forelsestmt
+        stmt ::= try_except
+        stmt ::= tryelsestmt
+        stmt ::= tryfinallystmt
+        stmt ::= last_stmt
+
+        stmt ::= dict_comp_func
+
+        stmt ::= set_comp_func
+
+        # last_stmt is a Python statement for which
+        # end is a "return" or raise statement and
+        # therefore may not have a COME_FROM after
+        # it. It does *not* have to be the last stmt of
+        # a list of stmts or c_stmts
+        last_stmt  ::= forelselaststmt
+        last_stmt  ::= iflaststmt
+
+        stmt   ::= delete
+        delete ::= DELETE_FAST
+        delete ::= DELETE_NAME
+        delete ::= DELETE_GLOBAL
+
+        stmt   ::= return
+
+        # "returns" nonterminal is a sequence of statements that ends in a RETURN statement.
+        # In later Python versions with jump optimization, this can cause JUMPs
+        # that would normally appear to be omitted.
+
+        returns ::= return
+        returns ::= _stmts return
+
+        stmt ::= genexpr_func
+        """
+        pass
+
+    # # A "condition", in contrast to an "expr"ession ,is something that is is used in
+    # # tests and pops the condition after testing
+    # def p_if_conditions(self, args):
+    #     """
+    #     condition ::= and_or_cond
+    #     condition ::= nor_cond
+    #     condition ::= or_cond
+    #     stmt ::= if_cond_stmt
+    #     if_cond_stmt ::= condition stmt
+    #     if_cond_else_stmt ::= condition
+    #     """
+
+    def p_function_def(self, args):
+        """
+        stmt               ::= function_def
+        function_def       ::= mkfunc store
+        stmt               ::= function_def_deco
+        function_def_deco  ::= mkfuncdeco store
+        mkfuncdeco         ::= expr mkfuncdeco CALL_FUNCTION_1
+        mkfuncdeco         ::= expr mkfuncdeco0 CALL_FUNCTION_1
+        mkfuncdeco0        ::= mkfunc
+        load_closure       ::= load_closure LOAD_CLOSURE
+        load_closure       ::= LOAD_CLOSURE
+        """
+
+    def p_augmented_assign(self, args):
+        """
+        stmt ::= aug_assign1
+        stmt ::= aug_assign2
+
+        # This is odd in that other aug_assign1's have only 3 slots
+        # The store isn't used as that's supposed to be also
+        # indicated in the first expr
+        aug_assign1 ::= expr expr
+                        inplace_op store
+        aug_assign1 ::= expr expr
+                        inplace_op ROT_THREE STORE_SUBSCR
+        aug_assign2 ::= expr DUP_TOP LOAD_ATTR expr
+                        inplace_op ROT_TWO STORE_ATTR
+
+        inplace_op ::= INPLACE_ADD
+        inplace_op ::= INPLACE_SUBTRACT
+        inplace_op ::= INPLACE_MULTIPLY
+        inplace_op ::= INPLACE_TRUE_DIVIDE
+        inplace_op ::= INPLACE_FLOOR_DIVIDE
+        inplace_op ::= INPLACE_MODULO
+        inplace_op ::= INPLACE_POWER
+        inplace_op ::= INPLACE_LSHIFT
+        inplace_op ::= INPLACE_RSHIFT
+        inplace_op ::= INPLACE_AND
+        inplace_op ::= INPLACE_XOR
+        inplace_op ::= INPLACE_OR
+        """
+
+    def p_assign(self, args):
+        """
+        stmt ::= assign
+        assign ::= expr DUP_TOP designList
+        assign ::= expr store
+
+        stmt ::= assign2
+        stmt ::= assign3
+        assign2 ::= expr expr ROT_TWO store store
+        assign3 ::= expr expr expr ROT_THREE ROT_TWO store store store
+        """
+
+    def p_await(self, args):
+        # Python 3.5+ Await things
+        """
+        # Can move this after have a p37/full_custom.py
+        stmt       ::= await_stmt
+        await_stmt ::= await_expr POP_TOP
+        """
+
+    def p_for_loop(self, args):
+        """
+        setup_loop  ::= SETUP_LOOP _come_froms
+        for         ::= setup_loop expr get_for_iter store for_block
+                        POP_BLOCK
+        for         ::= setup_loop expr get_for_iter store for_block
+                        POP_BLOCK COME_FROM_LOOP
+
+        # FIXME: investigate - can code really produce a NOP?
+        for         ::= setup_loop expr get_for_iter store for_block POP_BLOCK NOP
+                        COME_FROM_LOOP
+
+
+        come_from_loops ::= COME_FROM_LOOP*
+
+        for_block   ::= c_stmts_opt COME_FROM_LOOP JUMP_LOOP
+        for_block   ::= c_stmts_opt _come_froms JUMP_LOOP
+        for_block   ::= c_stmts_opt come_from_loops JUMP_LOOP
+        for_block   ::= c_stmts
+        for_block   ::= c_stmts JUMP_LOOP
+
+        forelsestmt ::= SETUP_LOOP expr get_for_iter store
+                        for_block POP_BLOCK else_suite _come_froms
+
+        forelsestmt ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec
+        forelsestmt ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suite
+                        COME_FROM_LOOP
+
+
+        forelselaststmt ::= SETUP_LOOP expr get_for_iter store
+                           for_block POP_BLOCK else_suitec _come_froms
+
+        forelselaststmt  ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec
+                              COME_FROM_LOOP
+
+        forelselaststmtc ::= SETUP_LOOP expr get_for_iter store
+                for_block POP_BLOCK else_suitec _come_froms
+        """
+
+    def p_whilestmt(self, args):
+        """
+        while1elsestmt ::= setup_loop c_stmts JUMP_LOOP POP_BLOCK else_suite COME_FROM_LOOP
+        while1elsestmt ::= setup_loop c_stmts JUMP_LOOP _come_froms POP_BLOCK else_suitec COME_FROM_LOOP
+        while1elsestmt ::= setup_loop c_stmts JUMP_LOOP else_suite COME_FROM_LOOP
+        while1elsestmt ::= setup_loop c_stmts JUMP_LOOP else_suitec
+
+        # FIXME: Python 3.? starts adding branch optimization? Put this starting there.
+
+        while1stmt ::= setup_loop c_stmts _come_froms JUMP_LOOP
+                       COME_FROM_LOOP
+        while1stmt ::= setup_loop c_stmts _come_froms JUMP_LOOP POP_BLOCK
+                       COME_FROM_LOOP
+        while1stmt ::= setup_loop c_stmts COME_FROM_LOOP
+        while1stmt ::= setup_loop c_stmts COME_FROM_LOOP JUMP_LOOP POP_BLOCK
+                       COME_FROM_LOOP
+        while1stmt ::= setup_loop c_stmts POP_BLOCK
+                       COME_FROM_LOOP
+
+        whileTruestmt ::= SETUP_LOOP c_stmts_opt JUMP_LOOP COME_FROM_LOOP
+        whileTruestmt ::= setup_loop c_stmts_opt JUMP_LOOP POP_BLOCK _come_froms
+
+        # FIXME the below masks a bug in not detecting COME_FROM_LOOP
+        # grammar rules with COME_FROM -> COME_FROM_LOOP already exist
+        whileelsestmt     ::= setup_loop testexpr c_stmts_opt
+                              JUMP_LOOP POP_BLOCK
+                              else_suite COME_FROM
+
+        whileelsestmt     ::= setup_loop testexpr c_stmts_opt
+                              JUMP_LOOP POP_BLOCK
+                              else_suite COME_FROM_LOOP
+
+        # There is no JUMP_LOOP here because c_stmts contineus, returns, or breaks
+        whileelsestmt     ::= setup_loop testexpr
+                              c_stmts come_froms POP_BLOCK
+                              else_suite COME_FROM_LOOP
+
+        whilestmt ::= setup_loop testexprc c_stmts_opt COME_FROM JUMP_LOOP POP_BLOCK COME_FROM_LOOP
+        whilestmt ::= setup_loop testexprc c_stmts_opt JUMP_LOOP POP_BLOCK COME_FROM_LOOP
+
+        # We can be missing a COME_FROM_LOOP if the "while" statement is nested inside an if/else
+        # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
+        # This is undoubtedly some sort of JUMP optimization going on.
+        # We have a reduction check for this peculiar case.
+
+        whilestmt ::= setup_loop testexpr c_stmts_opt JUMP_LOOP come_froms POP_BLOCK
+
+        whilestmt ::= setup_loop testexpr c_stmts_opt JUMP_LOOP come_froms POP_BLOCK COME_FROM_LOOP
+        whilestmt ::= setup_loop testexpr c_stmts_opt come_froms JUMP_LOOP come_froms POP_BLOCK COME_FROM_LOOP
+        whilestmt ::= setup_loop testexpr c_stmts_opt come_froms POP_BLOCK COME_FROM_LOOP
+        whilestmt ::= setup_loop testexpr returns POP_BLOCK COME_FROM_LOOP
+        whilestmt ::= setup_loop testexpr returns come_froms POP_BLOCK COME_FROM_LOOP
+        """
+
+    def p_import20(self, args):
+        """
+        stmt ::= import
+        stmt ::= import_from
+        stmt ::= import_from_star
+        stmt ::= importmultiple
+
+        importlist ::= importlist alias
+        importlist ::= alias
+        alias      ::= IMPORT_NAME store
+        alias      ::= IMPORT_FROM store
+        alias      ::= IMPORT_NAME attributes store
+
+        import           ::= LOAD_CONST LOAD_CONST alias
+        import_from_star ::= LOAD_CONST LOAD_CONST IMPORT_NAME IMPORT_STAR
+        import_from_star ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR IMPORT_STAR
+        import_from      ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist POP_TOP
+        importmultiple   ::= LOAD_CONST LOAD_CONST alias imports_cont
+
+        imports_cont ::= import_cont+
+        import_cont  ::= LOAD_CONST LOAD_CONST alias
+
+        attributes   ::= LOAD_ATTR+
+        """
+
+    def p_import37(self, args):
+        """
+        # The 3.7base scanner adds IMPORT_NAME_ATTR
+        alias            ::= IMPORT_NAME_ATTR attributes store
+        alias            ::= IMPORT_NAME_ATTR store
+
+        alias37          ::= IMPORT_NAME store
+        alias37          ::= IMPORT_FROM store
+
+        import_as37      ::= LOAD_CONST LOAD_CONST importlist37 store POP_TOP
+        import_from      ::= LOAD_CONST LOAD_CONST importlist POP_TOP
+        import_from37    ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist37 POP_TOP
+        import_from_as37 ::= LOAD_CONST LOAD_CONST import_from_attr37 store POP_TOP
+
+        # A single entry in a dotted import a.b.c.d
+        import_one       ::= importlists ROT_TWO IMPORT_FROM
+        import_one       ::= importlists ROT_TWO POP_TOP IMPORT_FROM
+
+        # Semantic checks distinguish importattr37 from import_from_attr37
+        # in the former the "from" slot in a prior LOAD_CONST is null.
+
+        # Used in: import .. as ..
+        importattr37      ::= IMPORT_NAME_ATTR IMPORT_FROM
+
+        # Used in: from xx import .. as ..
+        import_from_attr37 ::= IMPORT_NAME_ATTR IMPORT_FROM
+
+        importlist37  ::= import_one
+        importlist37  ::= importattr37
+        importlist37  ::= alias37+
+
+        importlists   ::= importlist37+
+
+        stmt          ::= import_as37
+        stmt          ::= import_from37
+        stmt          ::= import_from_as37
+        """
+
+    def p_32on(self, args):
+        """
+        # Python 3.5+ has jump optimization to remove the redundant
+        # jump_excepts. But in 3.3 we need them added
+
+        except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                           END_FINALLY
+
+        tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                           except_handler else_suite
+                           jump_excepts come_from_except_clauses
+
+        jump_excepts   ::= jump_except+
+
+        kv3       ::= expr expr STORE_MAP
+        """
+        return
+
+    def p_35_on(self, args):
+        """
+        inplace_op       ::= INPLACE_MATRIX_MULTIPLY
+
+        # FIXME: do we need these?
+        return_expr ::= expr
+        return_if_stmt ::= return_expr RETURN_END_IF POP_BLOCK
+
+        jb_cf     ::= JUMP_LOOP COME_FROM
+        ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
+
+        # We want to keep the positions of the "then" and
+        # "else" statements in "ifelstmtl" similar to others of this ilk.
+        testexpr_cf ::= testexpr come_froms
+
+        ifelsestmtc ::= testexpr_cf c_stmts_opt jb_cf else_suitec
+        iflaststmt  ::= testexpr stmts_opt JUMP_FORWARD
+        """
+
+    def p_37_async(self, args):
+        """
+        stmt     ::= async_for_stmt37
+        stmt     ::= async_for_stmt
+        stmt     ::= async_for_stmt2
+        stmt     ::= async_forelse_stmt
+
+        # FIXME: DRY this with rules.
+        async_for_stmt     ::= setup_loop expr
+                               GET_AITER
+                               _come_froms
+                               SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               store
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                               END_FINALLY COME_FROM
+                               for_block
+                               COME_FROM
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
+                               opt_come_from_loop
+
+        async_for_stmt2    ::= setup_loop expr
+                               GET_AITER
+                               _come_froms
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               store
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK
+                               opt_come_from_loop
+
+        # Order of LOAD_CONST YIELD_FROM is switched from 3.6 to save a LOAD_CONST
+        async_for_stmt37   ::= setup_loop expr
+                               GET_AITER
+                               _come_froms
+                               SETUP_EXCEPT GET_ANEXT
+                               LOAD_CONST YIELD_FROM
+                               store
+                               POP_BLOCK JUMP_LOOP COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                               END_FINALLY
+                               for_block COME_FROM
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT
+                               POP_TOP POP_BLOCK
+                               COME_FROM_LOOP
+
+        async_forelse_stmt ::= setup_loop expr
+                               GET_AITER
+                               _come_froms
+                               SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               store
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                               END_FINALLY COME_FROM
+                               for_block
+                               COME_FROM
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
+                               else_suite COME_FROM_LOOP
+
+        """
+
+    def p_grammar(self, args):
+        """sstmt ::= stmt
+        sstmt ::= ifelsestmtr
+        sstmt ::= return RETURN_LAST
+
+        return_if_stmts ::= return_if_stmt come_from_opt
+        return_if_stmts ::= _stmts return_if_stmt _come_froms
+        return_if_stmt  ::= return_expr RETURN_END_IF
+        returns         ::= _stmts return_if_stmt
+
+
+        break     ::= BREAK_LOOP
+        continue  ::= CONTINUE
+        continues ::= _stmts lastc_stmt continue
+        continues ::= lastc_stmt continue
+        continues ::= continue
+
+
+        kwarg      ::= LOAD_STR expr
+        kwargs     ::= kwarg+
+
+        classdef ::= build_class store
+
+        # FIXME: we need to add these because don't detect this properly
+        # in custom rules. Specifically if one of the exprs is CALL_FUNCTION
+        # then we'll mistake that for the final CALL_FUNCTION.
+        # We can fix by triggering on the CALL_FUNCTION op
+        # Python3 introduced LOAD_BUILD_CLASS
+        # Other definitions are in a custom rule
+        build_class ::= LOAD_BUILD_CLASS mkfunc expr call CALL_FUNCTION_3
+        build_class ::= LOAD_BUILD_CLASS mkfunc expr call expr CALL_FUNCTION_4
+
+        stmt ::= classdefdeco
+        classdefdeco ::= classdefdeco1 store
+
+        # Some LOAD_GLOBALs we don't convert to LOAD_ASSERT because
+        # of the intevening "expr CALL_FUNCTION1" which can be an arbitrary number
+        # of instructions
+        assert2     ::= expr
+                        POP_JUMP_IF_TRUE LOAD_GLOBAL expr CALL_FUNCTION_1 RAISE_VARARGS_1
+
+        assert2_not ::= expr
+                        POP_JUMP_IF_FALSE LOAD_GLOBAL expr CALL_FUNCTION_1 RAISE_VARARGS_1
+
+        # "assert_invert" tests on the negative of the condition given
+        stmt          ::= assert_invert
+        assert_invert ::= testtrue LOAD_GLOBAL RAISE_VARARGS_1
+
+        expr    ::= LOAD_ASSERT
+
+        pop_jump    ::= POP_JUMP_IF_TRUE
+        pop_jump    ::= POP_JUMP_IF_FALSE
+
+        # These rules need reduce checks on the "_come_froms".
+        # When the come_from is empty the end of the "then"
+        # can't fall through. And when the "_come_froms" aren't
+        # empty they have to be reasonable, e.g. testexpr has to
+        # jump to one of the COME_FROMS
+        ifstmt      ::= testexpr stmts _come_froms
+        ifstmt      ::= bool_op stmts _come_froms
+        ifstmt      ::= testexpr ifstmts_jump _come_froms
+
+        stmt        ::= ifstmt_bool
+        ifstmt_bool ::= or_and_not stmts come_froms
+        ifstmt_bool ::= or_and1 stmts come_froms
+        ifstmt_bool ::= not_and_not stmts come_froms
+
+        if_or_stmt  ::= expr POP_JUMP_IF_TRUE expr pop_jump come_froms
+                        stmts COME_FROM
+        if_and_stmt ::= expr_pjif expr COME_FROM
+                        stmts _come_froms
+
+        if_and_elsestmtc    ::= expr_pjif
+                                expr_pjif
+                                c_stmts jb_cfs else_suitec opt_come_from_except
+        if_or_not_elsestmt  ::= expr POP_JUMP_IF_TRUE
+                                come_from_opt expr POP_JUMP_IF_TRUE come_froms
+                                stmts jf_cfs else_suite opt_come_from_except
+
+        testexpr   ::= testfalse
+        testexpr   ::= testtrue
+        testexpr   ::= or_and_not
+
+        testfalse  ::= expr_pjif
+        testfalsec ::= expr POP_JUMP_IF_TRUE_LOOP
+        testfalsec ::= c_compare_chained_middleb_false_37
+
+        testtrue   ::= expr_pjit
+        testtruec  ::= expr POP_JUMP_IF_FALSE_LOOP
+        # Do we have to check the c_compare_chained37 ends in a POP_JUMP_IF_FALSE_LOOP?
+        testtruec  ::= c_compare_chained37_false
+        testtruec  ::= c_compare_chained37
+        testtruec  ::= c_nand
+
+        testtrue   ::= compare_chained37
+        testtrue   ::= compare_chained_and
+
+        testtrue   ::= nor_cond
+
+        testfalse  ::= and_not
+        testfalse  ::= not_or
+        testfalse  ::= compare_chained37_false
+        testfalse  ::= or_cond
+        testfalse  ::= or_cond1
+        testfalse  ::= and_or_cond
+
+        ifstmts_jump ::= return_if_stmts
+        ifstmts_jump ::= stmts_opt come_froms
+        ifstmts_jump ::= COME_FROM stmts COME_FROM
+
+        # Python 3.4+ optimizes the trailing two JUMPS away
+        ifstmts_jump ::= stmts_opt JUMP_FORWARD JUMP_FORWARD _come_froms
+
+        # For "iflaststmt" there is a rule check for the below that the end of
+        # "stmts" doesn't fall through.
+        iflaststmt  ::= testexpr stmts
+        iflaststmt  ::= testexpr returns
+        iflaststmt  ::= testexpr stmts JUMP_FORWARD
+
+        iflaststmtc ::= testexpr c_stmts
+        iflaststmtc ::= testexpr c_stmts JUMP_LOOP
+        iflaststmtc ::= testexpr c_stmts JUMP_LOOP COME_FROM_LOOP
+        iflaststmtc ::= testexpr c_stmts JUMP_LOOP POP_BLOCK
+
+        # c_stmts might terminate, or have "continue" so no JUMP_LOOP.
+        # But if that's true, the "testexpr" needs still to jump to the "COME_FROM'
+        iflaststmtc ::= testexpr c_stmts come_froms
+
+        # Note: in if/else kinds of statements, we err on the side
+        # of missing "else" clauses. Therefore we include grammar
+        # rules with and without ELSE.
+
+        ifelsestmt    ::= testexpr
+                          stmts_opt jf_cfs else_suite_opt opt_come_from_except
+        ifelsestmt    ::= bool_op
+                          stmts_opt jf_cfs else_suite_opt opt_come_from_except
+
+
+        ifelsestmtc ::= testexpr
+                        c_stmts_opt jump_forward_else
+                        else_suitec opt_come_from_except
+        ifelsestmtc ::= testexpr
+                        c_stmts_opt cf_jump_back
+                        else_suitec
+
+        # This handles the case where a "JUMP_ABSOLUTE" is part
+        # of an inner if in c_stmts_opt
+        ifelsestmtc ::= testexpr c_stmts come_froms
+                        else_suite
+
+
+        ifelsestmtr ::= testexpr return_if_stmts returns
+
+
+        cf_jump_back ::= COME_FROM JUMP_LOOP
+
+        # This is nested inside a try_except
+        tryfinallystmt   ::= SETUP_FINALLY suite_stmts_opt
+                             POP_BLOCK LOAD_CONST
+                             COME_FROM_FINALLY suite_stmts_opt END_FINALLY
+
+        c_tryfinallystmt ::= SETUP_FINALLY c_suite_stmts_opt
+                             POP_BLOCK LOAD_CONST COME_FROM_FINALLY
+                             c_suite_stmts_opt END_FINALLY
+
+        # This a funny kind of try finally inside a try_except in a loop
+        c_except_suite     ::= SETUP_FINALLY c_suite_stmts
+                               POP_BLOCK LOAD_CONST
+                               COME_FROM_FINALLY LOAD_CONST STORE_FAST DELETE_FAST
+                               END_FINALLY
+                               POP_EXCEPT JUMP_LOOP COME_FROM
+
+        c_except_suite     ::= except_suite
+        c_except_suite     ::= c_stmts POP_EXCEPT JUMP_LOOP
+        c_except_handler36 ::= COME_FROM_EXCEPT c_except_stmts END_FINALLY
+        c_try_except36     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                               c_except_handler36 come_from_opt
+        c_try_except36     ::= SETUP_EXCEPT returns
+                               c_except_handler36 come_from_opt
+
+
+        except_handler ::= jmp_abs COME_FROM except_stmts
+                           _come_froms END_FINALLY
+        except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
+                           _come_froms END_FINALLY
+
+        c_except_handler ::= jmp_abs COME_FROM c_except_stmts
+                           _come_froms END_FINALLY
+        c_except_handler ::= jmp_abs COME_FROM_EXCEPT c_except_stmts
+                           _come_froms END_FINALLY
+        c_except_handler ::= jmp_abs COME_FROM_EXCEPT c_except_stmts
+
+        try_except   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                         except_handler
+                         jump_excepts come_from_except_clauses
+
+        c_try_except ::= SETUP_EXCEPT c_suite_stmts_opt POP_BLOCK
+                         c_except_handler
+                         jump_excepts come_from_except_clauses
+
+        # FIXME: remove this
+        except_handler ::= JUMP_FORWARD COME_FROM except_stmts
+                           come_froms END_FINALLY come_from_opt
+
+        except_stmts   ::= except_stmt+
+
+        except_stmt    ::= except_cond1 except_suite come_from_opt
+        except_stmt    ::= except_cond2 except_suite come_from_opt
+        except_stmt    ::= except_cond2 except_suite_finalize
+        except_stmt    ::= except
+        except_stmt    ::= stmt
+
+        c_except_stmts ::= except_stmts
+        c_except_stmts ::= c_except_stmt+
+        c_except_stmt  ::= c_stmt
+        c_except_stmt  ::= c_except
+        c_except_stmt  ::= except_cond1 c_except_suite come_from_opt
+        c_except_stmt  ::= except_cond2 c_except_suite come_from_opt
+        c_except_stmt  ::= stmt
+
+        ## FIXME: what's except_pop_except?
+        except_stmt    ::= except_pop_except
+
+        # Python3 introduced POP_EXCEPT
+        except_suite ::= c_stmts_opt POP_EXCEPT jump_except
+        jump_except ::= JUMP_ABSOLUTE
+        jump_except ::= JUMP_LOOP
+        jump_except ::= JUMP_FORWARD
+        jump_except ::= CONTINUE
+
+        # This is used in Python 3 in
+        # "except ... as e" to remove 'e' after the c_stmts_opt finishes
+        except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize
+                                  END_FINALLY jump
+
+        except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize
+                                  END_FINALLY POP_EXCEPT jump
+
+        except_var_finalize ::= POP_BLOCK POP_EXCEPT LOAD_CONST COME_FROM_FINALLY
+                                LOAD_CONST store delete
+        except_var_finalize ::= POP_BLOCK            LOAD_CONST COME_FROM_FINALLY
+                                LOAD_CONST store delete
+
+        except_suite   ::= returns
+        c_except_suite ::= c_returns
+
+        # except XXX:
+        except_cond1 ::= DUP_TOP expr COMPARE_OP
+                         POP_JUMP_IF_FALSE POP_TOP POP_TOP POP_TOP
+
+        # except XXX as var:
+        except_cond2 ::= DUP_TOP expr COMPARE_OP
+                         POP_JUMP_IF_FALSE POP_TOP store POP_TOP come_from_opt
+
+        except  ::=  POP_TOP POP_TOP POP_TOP c_stmts_opt POP_EXCEPT JUMP_FORWARD
+        except  ::=  POP_TOP POP_TOP POP_TOP returns
+
+        except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+                           come_froms END_FINALLY
+        jmp_abs ::= JUMP_ABSOLUTE
+        jmp_abs ::= JUMP_LOOP
+        jmp_abs ::= JUMP_FORWARD
+
+        stmt    ::= assert2
+        stmt    ::= assert2_not
+        """
+
+    def p_come_from3(self, args):
+        """
+        # In 3.7+ a SETUP_LOOP to a JUMP_FORWARD can
+        # get replaced by the JUMP_FORWARD addressed. Therefore COME_FROMs may
+        # appear out of nesting order. For example:
+        #   if x
+        #     for ... jump forward endif (1)
+        #        ...
+        #        break - jump forward endif (2)
+        #     end for
+        #     optional jump forward endif (1)
+        #   else:
+        #       ...
+        #   endif
+        #   come from loop 2 - note not strictly nested
+        #   come from if-then 1
+
+        come_any_froms ::= come_any_froms come_any_from
+        come_any_froms ::= come_any_from
+        come_any_from  ::= COME_FROM_LOOP
+        come_any_from  ::= COME_FROM_EXCEPT
+        come_any_from  ::= COME_FROM
+
+        opt_come_from_except ::= come_any_froms?
+        opt_come_from_loop   ::= COME_FROM_LOOP?
+
+        come_from_except_clauses ::= COME_FROM_EXCEPT_CLAUSE*
+        """
+
+    def p_jump3(self, args):
+        """
+        # FIXME: simplify this
+        return_expr_or_cond ::= if_exp_ret
+        return_expr_or_cond ::= return_expr
+
+        if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM return_expr_or_cond
+
+        testfalse ::= or POP_JUMP_IF_FALSE COME_FROM
+        testfalse ::= nand
+        testfalse ::= and
+
+        testexprc   ::= testexpr
+        testexprc   ::= testfalsec
+        testexprc   ::= testtruec
+        iflaststmtc ::= testexprc c_stmts
+        iflaststmtc ::= testexprc c_stmts JUMP_LOOP COME_FROM_LOOP
+        iflaststmtc ::= testexprc c_stmts JUMP_LOOP opt_pop_block
+
+        opt_pop_block ::= POP_BLOCK?
+
+        """
+
+    def p_stmt3(self, args):
+        """
+        if_exp_lambda      ::= expr_pjif expr return_if_lambda
+                               return_stmt_lambda
+        if_exp_not_lambda
+                           ::= expr POP_JUMP_IF_TRUE expr return_if_lambda
+                               return_stmt_lambda
+        return_stmt_lambda ::= return_expr RETURN_VALUE_LAMBDA
+
+        stmt               ::= return_closure
+        return_closure     ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST
+
+        stmt               ::= whileTruestmt
+        ifelsestmt         ::= testexpr stmts_opt JUMP_FORWARD else_suite _come_froms
+        ifelsestmtc        ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_froms
+
+        ifstmtc            ::= testexpr ifstmts_jumpc
+
+        # We need a reduction rule to disambiguate ifstmtc from if_not_stmtc
+        # The difference is that when ifstmts_jumpc geos back to to a loop
+        # and testexprc is testtruec, then we have if_not_stmtc.
+
+        ifstmtc            ::= testexprc ifstmts_jumpc _come_froms
+        if_not_stmtc       ::= testexprc ifstmts_jumpc _come_froms
+
+        ifstmts_jumpc             ::= ifstmts_jump
+        ifstmts_jumpc             ::= c_stmts_opt come_froms
+        ifstmts_jumpc             ::= COME_FROM c_stmts come_froms
+        ifstmts_jumpc             ::= c_stmts
+        ifstmts_jumpc             ::= c_stmts JUMP_LOOP
+
+        ifstmts_jump              ::= stmts come_froms
+        ifstmts_jump              ::= COME_FROM stmts come_froms
+
+
+        # The following can happen when the jump offset is large and
+        # Python is looking to do a small jump to a larger jump to get
+        # around the problem that the offset can't be represented in
+        # the size allowed for the jump offset. This is more likely to
+        # happen in wordcode Python since the offset range has been
+        # reduced.  FIXME: We should add a reduction check that the
+        # final jump goes to another jump.
+
+        ifstmts_jumpc     ::= COME_FROM c_stmts JUMP_LOOP
+        ifstmts_jumpc     ::= COME_FROM c_stmts JUMP_FORWARD
+
+        """
+
+    def p_3try_except(self, args):
+        """
+        # In 3.6+, A sequence of statements ending in a RETURN can cause
+        # JUMP_FORWARD END_FINALLY to be omitted from try middle
+
+        except_return    ::= POP_TOP POP_TOP POP_TOP returns
+        except_handler   ::= JUMP_FORWARD COME_FROM_EXCEPT except_return
+
+        # Try middle following a returns
+        except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY
+
+        stmt             ::= try_except36
+        try_except36     ::= SETUP_EXCEPT returns except_handler36
+                             opt_come_from_except
+        try_except36     ::= SETUP_EXCEPT suite_stmts
+        try_except36     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                             except_handler36 come_from_opt
+
+        # 3.6 omits END_FINALLY sometimes
+        except_handler36 ::= COME_FROM_EXCEPT except_stmts
+        except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
+        except_handler   ::= jmp_abs COME_FROM_EXCEPT except_stmts
+
+        stmt             ::= tryfinally36
+        tryfinally36     ::= SETUP_FINALLY returns
+                             COME_FROM_FINALLY suite_stmts
+        tryfinally36     ::= SETUP_FINALLY returns
+                             COME_FROM_FINALLY suite_stmts_opt END_FINALLY
+        except_suite_finalize ::= SETUP_FINALLY returns
+                                  COME_FROM_FINALLY suite_stmts_opt END_FINALLY jump
+
+        stmt ::= tryfinally_return_stmt1
+        stmt ::= tryfinally_return_stmt2
+        tryfinally_return_stmt1 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
+                                    COME_FROM_FINALLY returns
+        tryfinally_return_stmt2 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
+                                    COME_FROM_FINALLY
+
+        """
+
+    def p_36misc(self, args):
+        """
+        sstmt ::= sstmt RETURN_LAST
+
+        # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
+        return_closure   ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
+
+        # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
+        # RETURN_VALUE is meant. Specifically this can happen in
+        # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
+        return ::= return_expr RETURN_END_IF
+        return ::= return_expr RETURN_VALUE
+
+        jf_cf        ::= JUMP_FORWARD COME_FROM
+
+        if_exp       ::= expr_pjif expr jf_cf expr COME_FROM
+
+        except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM
+
+        jb_cfs      ::= come_from_opt JUMP_LOOP come_froms
+        ifelsestmtc ::= testexpr c_stmts_opt jb_cfs else_suitec
+        """
+
+
+def info(args):
+    # Check grammar
+    import sys
+
+    p = Python37Parser()
+    if len(args) > 0:
+        arg = args[0]
+        if arg == "3.7":
+            from decompyle3.parser.parse37 import Python37Parser
+
+            p = Python37Parser()
+        elif arg == "3.8":
+            from decompyle3.parser.parse38 import Python38Parser
+
+            p = Python38Parser()
+        else:
+            raise RuntimeError("Only 3.7 and 3.8 supported")
+    p.check_grammar()
+    if len(sys.argv) > 1 and sys.argv[1] == "dump":
+        print("-" * 50)
+        p.dump_grammar()
+
+
+if __name__ == "__main__":
+    # Check grammar
+    from decompyle3.parsers.dump import dump_and_check
+
+    p = Python37Parser()
+    modified_tokens = set(
+        """JUMP_LOOP CONTINUE RETURN_END_IF COME_FROM
+           LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+           LAMBDA_MARKER RETURN_LAST
+        """.split()
+    )
+
+    dump_and_check(p, (3, 7), modified_tokens)

python/py/Lib/site-packages/decompyle3/parsers/p37/heads.py

@@ -0,0 +1,68 @@
+"""
+All of the specific kinds of canned parsers for Python 3.7
+
+These are derived from "compile-modes" but we have others that
+can be used to parse common part of a larger grammar.
+
+For example:
+* a basic-block expression (no branching)
+* an unadorned expression (no POP_TOP needed afterwards)
+* A non-compound statement
+"""
+from decompyle3.parsers.p37.full import Python37Parser
+from decompyle3.parsers.p37.lambda_expr import Python37LambdaParser
+from decompyle3.parsers.parse_heads import (
+    PythonParserEval,
+    PythonParserExec,
+    PythonParserExpr,
+    PythonParserLambda,
+    PythonParserSingle,
+    # FIXME: add
+    # PythonParserSimpleStmt
+    # PythonParserStmt
+)
+from decompyle3.scanners.tok import Token
+
+
+# Make sure to list Python37... classes first so we prefer to inherit methods from that first.
+# In particular methods like reduce_is_invalid() need to come from there rather than
+# a more generic place.
+
+
+class Python37ParserEval(Python37LambdaParser, PythonParserEval):
+    def __init__(self, debug_parser):
+        PythonParserEval.__init__(self, debug_parser)
+
+
+class Python37ParserExec(Python37Parser, PythonParserExec):
+    def __init__(self, debug_parser):
+        PythonParserExec.__init__(self, debug_parser)
+
+
+class Python37ParserExpr(Python37Parser, PythonParserExpr):
+    def __init__(self, debug_parser):
+        PythonParserExpr.__init__(self, debug_parser)
+
+
+class Python37ParserLambda(Python37LambdaParser, PythonParserLambda):
+    def __init__(self, debug_parser):
+        PythonParserLambda.__init__(self, debug_parser)
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        if rule[0] == "call_kw":
+            # Make sure we don't derive call_kw
+            nt = ast[0]
+            while not isinstance(nt, Token):
+                if nt[0] == "call_kw":
+                    return True
+                nt = nt[0]
+                pass
+            pass
+        return False
+
+
+# These classes are here just to get parser doc-strings for the
+# various classes inherited properly and start_symbols set pproperly
+class Python37ParserSingle(Python37Parser, PythonParserSingle):
+    def __init__(self, debug_parser):
+        PythonParserSingle.__init__(self, debug_parser)

python/py/Lib/site-packages/decompyle3/parsers/p37/lambda_custom.py

@@ -0,0 +1,693 @@
+#  Copyright (c) 2020-2022 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Grammar Customization rules for Python 3.7's Lambda expression grammar.
+"""
+
+from decompyle3.parsers.p37.base import Python37BaseParser
+from decompyle3.parsers.parse_heads import nop_func
+
+
+class Python37LambdaCustom(Python37BaseParser):
+    def __init__(self):
+        self.new_rules = set()
+        self.customized = {}
+
+    def customize_grammar_rules_lambda37(self, tokens, customize):
+        Python37BaseParser.customize_grammar_rules37(self, tokens, customize)
+        self.check_reduce["call_kw"] = "AST"
+
+        # For a rough break out on the first word. This may
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "DICT",
+                "GET",
+                "FORMAT",
+                "LIST",
+                "LOAD",
+                "MAKE",
+                "SETUP",
+                "UNPACK",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get added
+        # unconditionally and the rules are constant. So they need to be done
+        # only once and if we see the opcode a second we don't have to consider
+        # adding more rules.
+        #
+        # Note: BUILD_TUPLE_UNPACK_WITH_CALL gets considered by
+        # default because it starts with BUILD. So we'll set to ignore it from
+        # the start.
+        custom_ops_processed = {"BUILD_TUPLE_UNPACK_WITH_CALL"}
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            if opname == "LOAD_ASSERT":
+                if "PyPy" in customize:
+                    rules_str = """
+                    stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+            elif opname == "FORMAT_VALUE":
+                rules_str = """
+                    expr              ::= formatted_value1
+                    formatted_value1  ::= expr FORMAT_VALUE
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+            elif opname == "FORMAT_VALUE_ATTR":
+                rules_str = """
+                expr              ::= formatted_value2
+                formatted_value2  ::= expr expr FORMAT_VALUE_ATTR
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+            elif opname == "MAKE_FUNCTION_CLOSURE":
+                if "LOAD_DICTCOMP" in self.seen_ops:
+                    # Is there something general going on here?
+                    rule = """
+                       dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
+                                     MAKE_FUNCTION_CLOSURE expr
+                                     GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+                elif "LOAD_SETCOMP" in self.seen_ops:
+                    rule = """
+                       set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
+                                    MAKE_FUNCTION_CLOSURE expr
+                                    GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+
+            elif opname == "BEFORE_ASYNC_WITH":
+                rules_str = """
+                  stmt               ::= async_with_stmt SETUP_ASYNC_WITH
+                  async_with_pre     ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
+                  async_with_post    ::= COME_FROM_ASYNC_WITH
+                                         WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                         WITH_CLEANUP_FINISH END_FINALLY
+
+                  stmt               ::= async_with_as_stmt
+                  async_with_as_stmt ::= expr
+                                         async_with_pre
+                                         store
+                                         suite_stmts_opt
+                                         POP_BLOCK LOAD_CONST
+                                         async_with_post
+
+                  async_with_stmt     ::= expr
+                                          async_with_pre
+                                          POP_TOP
+                                          c_suite_stmts
+                                          POP_BLOCK
+                                          BEGIN_FINALLY
+                                          WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                          WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
+                                          POP_BLOCK
+                                          BEGIN_FINALLY
+                                          COME_FROM_ASYNC_WITH
+                                          WITH_CLEANUP_START
+                                          GET_AWAITABLE
+                                          LOAD_CONST
+                                          YIELD_FROM
+                                          WITH_CLEANUP_FINISH
+                                          END_FINALLY
+
+
+                 async_with_stmt     ::= expr
+                                         async_with_pre
+                                         POP_TOP
+                                         suite_stmts_opt
+                                         POP_BLOCK LOAD_CONST
+                                         async_with_post
+                 async_with_stmt     ::= expr
+                                         async_with_pre
+                                         POP_TOP
+                                         suite_stmts_opt
+                                         async_with_post
+                """
+                self.addRule(rules_str, nop_func)
+
+            elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
+                if opname == "BUILD_CONST_DICT":
+                    rule = f"""
+                           add_consts          ::= ADD_VALUE*
+                           const_list          ::= COLLECTION_START add_consts {opname}
+                           dict                ::= const_list
+                           expr                ::= dict
+                           """
+                else:
+                    rule = f"""
+                           add_consts          ::= ADD_VALUE*
+                           const_list          ::= COLLECTION_START add_consts {opname}
+                           expr                ::= const_list
+                           """
+                self.addRule(rule, nop_func)
+            elif opname_base in (
+                "BUILD_LIST",
+                "BUILD_SET",
+                "BUILD_SET_UNPACK",
+                "BUILD_TUPLE",
+                "BUILD_TUPLE_UNPACK",
+            ):
+                v = token.attr
+
+                is_LOAD_CLOSURE = False
+                if opname_base == "BUILD_TUPLE":
+                    # If is part of a "load_closure", then it is not part of a
+                    # "list".
+                    is_LOAD_CLOSURE = True
+                    for j in range(v):
+                        if tokens[i - j - 1].kind != "LOAD_CLOSURE":
+                            is_LOAD_CLOSURE = False
+                            break
+                    if is_LOAD_CLOSURE:
+                        rule = "load_closure ::= %s%s" % (("LOAD_CLOSURE " * v), opname)
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+
+                elif opname_base == "BUILD_LIST":
+                    v = token.attr
+                    if v == 0:
+                        rule_str = """
+                           list        ::= BUILD_LIST_0
+                           list_unpack ::= BUILD_LIST_0 expr LIST_EXTEND
+                           list        ::= list_unpack
+                        """
+                        self.add_unique_doc_rules(rule_str, customize)
+
+                elif opname == "BUILD_TUPLE_UNPACK_WITH_CALL":
+                    # FIXME: should this be parameterized by EX value?
+                    self.addRule(
+                        """expr        ::= call_ex_kw3
+                           call_ex_kw3 ::= expr
+                                           build_tuple_unpack_with_call
+                                           expr
+                                           CALL_FUNCTION_EX_KW
+                        """,
+                        nop_func,
+                    )
+
+                if not is_LOAD_CLOSURE or v == 0:
+                    # We do this complicated test to speed up parsing of
+                    # pathelogically long literals, especially those over 1024.
+                    build_count = token.attr
+                    thousands = build_count // 1024
+                    thirty32s = (build_count // 32) % 32
+                    if thirty32s > 0:
+                        rule = "expr32 ::=%s" % (" expr" * 32)
+                        self.add_unique_rule(rule, opname_base, build_count, customize)
+                        pass
+                    if thousands > 0:
+                        self.add_unique_rule(
+                            "expr1024 ::=%s" % (" expr32" * 32),
+                            opname_base,
+                            build_count,
+                            customize,
+                        )
+                        pass
+                    collection = opname_base[opname_base.find("_") + 1 :].lower()
+                    rule = (
+                        ("%s ::= " % collection)
+                        + "expr1024 " * thousands
+                        + "expr32 " * thirty32s
+                        + "expr " * (build_count % 32)
+                        + opname
+                    )
+                    self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
+                    continue
+                continue
+
+            elif opname.startswith("BUILD_STRING"):
+                v = token.attr
+                rules_str = """
+                    expr                 ::= joined_str
+                    joined_str           ::= %sBUILD_STRING_%d
+                """ % (
+                    "expr " * v,
+                    v,
+                )
+                self.add_unique_doc_rules(rules_str, customize)
+                if "FORMAT_VALUE_ATTR" in self.seen_ops:
+                    rules_str = """
+                      formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
+                      expr                 ::= formatted_value_attr
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+            elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
+                v = token.attr
+                rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+            elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
+                v = token.attr
+                rule = (
+                    "build_tuple_unpack_with_call ::= "
+                    + "expr1024 " * int(v // 1024)
+                    + "expr32 " * int((v // 32) % 32)
+                    + "expr " * (v % 32)
+                    + opname
+                )
+                self.addRule(rule, nop_func)
+                rule = "starred ::= %s %s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+
+            elif opname == "FORMAT_VALUE":
+                rules_str = """
+                    expr              ::= formatted_value1
+                    formatted_value1  ::= expr FORMAT_VALUE
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+            elif opname == "FORMAT_VALUE_ATTR":
+                rules_str = """
+                expr              ::= formatted_value2
+                formatted_value2  ::= expr expr FORMAT_VALUE_ATTR
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "GET_AITER":
+                self.add_unique_doc_rules("get_aiter ::= expr GET_AITER", customize)
+                self.addRule(
+                    """
+                    expr                ::= dict_comp_async
+                    expr                ::= generator_exp_async
+                    expr                ::= list_comp_async
+
+                    dict_comp_async     ::= LOAD_DICTCOMP
+                                            LOAD_STR
+                                            MAKE_FUNCTION_0
+                                            get_aiter
+                                            CALL_FUNCTION_1
+
+                    dict_comp_async     ::= BUILD_MAP_0 LOAD_ARG
+                                            dict_comp_async
+
+                    func_async_middle   ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
+                                            DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                                            END_FINALLY COME_FROM
+
+                    generator_exp_async ::= load_genexpr LOAD_STR MAKE_FUNCTION_0
+                                            get_aiter CALL_FUNCTION_1
+
+                    # FIXME this is a workaround for probably some bug in the Earley parser
+                    # if we use get_aiter, then list_comp_async doesn't match, and I don't
+                    # understand why.
+                    expr_get_aiter      ::= expr GET_AITER
+
+                    list_afor           ::= get_aiter list_afor2
+
+                    list_afor2          ::= func_async_prefix
+                                            store func_async_middle list_iter
+                                            JUMP_LOOP COME_FROM
+                                            POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
+
+                    list_comp_async     ::= BUILD_LIST_0 LOAD_ARG list_afor2
+                    list_comp_async     ::= LOAD_LISTCOMP LOAD_STR MAKE_FUNCTION_0
+                                            expr_get_aiter CALL_FUNCTION_1
+                                            GET_AWAITABLE LOAD_CONST
+                                            YIELD_FROM
+
+                    list_iter           ::= list_afor
+
+                    set_comp_async       ::= LOAD_SETCOMP
+                                             LOAD_STR
+                                             MAKE_FUNCTION_0
+                                             get_aiter
+                                             CALL_FUNCTION_1
+
+                    set_comp_async       ::= LOAD_CLOSURE
+                                             BUILD_TUPLE_1
+                                             LOAD_SETCOMP
+                                             LOAD_STR MAKE_FUNCTION_CLOSURE
+                                             get_aiter CALL_FUNCTION_1
+                                             await
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+                self.addRule(
+                    """
+                    dict_comp_async      ::= BUILD_MAP_0 LOAD_ARG
+                                             dict_comp_async
+
+                    expr                 ::= dict_comp_async
+                    expr                 ::= generator_exp_async
+                    expr                 ::= list_comp_async
+                    expr                 ::= set_comp_async
+
+                    func_async_middle   ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
+                                            DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                                            END_FINALLY _come_froms
+
+                    # async_iter         ::= block_break SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM
+
+                    get_aiter            ::= expr GET_AITER
+
+                    list_afor            ::= get_aiter list_afor2
+
+                    list_comp_async      ::= BUILD_LIST_0 LOAD_ARG list_afor2
+                    list_iter            ::= list_afor
+
+
+                    set_afor             ::= get_aiter set_afor2
+                    set_iter             ::= set_afor
+
+                    set_comp_async       ::= BUILD_SET_0 LOAD_ARG
+                                             set_comp_async
+
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_ANEXT":
+                self.addRule(
+                    """
+                    expr                 ::= genexpr_func_async
+                    expr                 ::= BUILD_MAP_0 genexpr_func_async
+                    expr                 ::= list_comp_async
+
+                    dict_comp_async      ::= BUILD_MAP_0 genexpr_func_async
+
+                    async_iter           ::= _come_froms
+                                             SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM
+
+                    store_async_iter_end ::= store
+                                             POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
+                                             DUP_TOP LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+                                             END_FINALLY COME_FROM
+
+                    func_async_prefix    ::= _come_froms SETUP_EXCEPT GET_ANEXT LOAD_CONST YIELD_FROM
+
+                    # We use store_async_iter_end to make comp_iter come out in the right position,
+                    # (after the logical "store")
+                    genexpr_func_async   ::= LOAD_ARG async_iter
+                                             store_async_iter_end
+                                             comp_iter
+                                             JUMP_LOOP COME_FROM
+                                             POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
+
+                    genexpr_func_async   ::= LOAD_ARG func_async_prefix
+                                             store func_async_middle comp_iter
+                                             JUMP_LOOP COME_FROM
+                                             POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
+
+                    list_afor2           ::= async_iter
+                                             store
+                                             list_iter
+                                             JUMP_LOOP
+                                             COME_FROM_FINALLY
+                                             END_ASYNC_FOR
+
+                    list_comp_async      ::= BUILD_LIST_0 LOAD_ARG list_afor2
+
+                    set_afor2            ::= async_iter
+                                             store
+                                             func_async_middle
+                                             set_iter
+                                             JUMP_LOOP COME_FROM
+                                             POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
+
+                    set_afor2            ::= expr_or_arg
+                                             set_iter_async
+
+                    set_comp_async       ::= BUILD_SET_0 set_afor2
+
+                    set_iter_async       ::= async_iter
+                                             store
+                                             set_iter
+                                             JUMP_LOOP
+                                             _come_froms
+                                             END_ASYNC_FOR
+
+                    return_expr_lambda   ::= genexpr_func_async
+                                             LOAD_CONST RETURN_VALUE
+                                             RETURN_VALUE_LAMBDA
+
+                    return_expr_lambda   ::= BUILD_SET_0 genexpr_func_async
+                                             RETURN_VALUE_LAMBDA
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_AWAITABLE":
+                rule_str = """
+                    await_expr ::= expr GET_AWAITABLE LOAD_CONST YIELD_FROM
+                    expr       ::= await_expr
+                """
+                self.add_unique_doc_rules(rule_str, customize)
+
+            elif opname == "GET_ITER":
+                self.addRule(
+                    """
+                    expr      ::= get_iter
+                    get_iter  ::= expr GET_ITER
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_ASSERT":
+                if "PyPy" in customize:
+                    rules_str = """
+                    stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "LOAD_ATTR":
+                self.addRule(
+                    """
+                  expr      ::= attribute
+                  attribute ::= expr LOAD_ATTR
+                  """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "MAKE_FUNCTION_CLOSURE":
+                if "LOAD_DICTCOMP" in self.seen_ops:
+                    # Is there something general going on here?
+                    rule = """
+                       dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
+                                     MAKE_FUNCTION_CLOSURE expr
+                                     GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+                elif "LOAD_SETCOMP" in self.seen_ops:
+                    rule = """
+                       set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
+                                    MAKE_FUNCTION_CLOSURE expr
+                                    GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+
+            elif opname == "MAKE_FUNCTION_CLOSURE_POS":
+                args_pos, args_kw, annotate_args, closure = token.attr
+                stack_count = args_pos + args_kw + annotate_args
+                if args_pos:
+                    # This was seen ion line 447 of Python 3.8
+                    # This is needed for Python 3.8 line 447 of site-packages/nltk/tgrep.py
+                    # line 447:
+                    #    lambda i: lambda n, m=None, l=None: ...
+                    # which has
+                    #  L. 447         0  LOAD_CONST               (None, None)
+                    #                 2  LOAD_CLOSURE             'i'
+                    #                 4  LOAD_CLOSURE             'predicate'
+                    #                 6  BUILD_TUPLE_2         2
+                    #                 8  LOAD_LAMBDA              '<code_object <lambda>>'
+                    #                10  LOAD_STR                 '_tgrep_relation_action.<locals>.<lambda>.<locals>.<lambda>'
+                    #                12  MAKE_FUNCTION_CLOSURE_POS    'default, closure'
+                    # FIXME: Possibly we need to generalize for more nested lambda's of lambda's?
+                    rule = """
+                         expr        ::= lambda_body
+                         lambda_body ::= %s%s%s%s
+                         """ % (
+                        "expr " * stack_count,
+                        "load_closure " * closure,
+                        "LOAD_LAMBDA LOAD_STR ",
+                        opname,
+                    )
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
+                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+                # Removes POP_BLOCK LOAD_CONST from 3.6-
+                with_as    ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                    with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                    """
+                else:
+                    rules_str += """
+                    with        ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   BEGIN_FINALLY COME_FROM_WITH
+                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                                   END_FINALLY
+                    """
+                self.addRule(rules_str, nop_func)
+                pass
+            pass
+
+    def custom_classfunc_rule(self, opname, token, customize, next_token):
+
+        args_pos, args_kw = self.get_pos_kw(token)
+
+        # Additional exprs for * and ** args:
+        #  0 if neither
+        #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
+        #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
+        # Yes, this computation based on instruction name is a little bit hoaky.
+        nak = (len(opname) - len("CALL_FUNCTION")) // 3
+        uniq_param = args_kw + args_pos
+
+        if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
+            rule_str = """
+                await      ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
+                await_expr ::= expr await
+                expr       ::= await_expr
+            """
+            self.add_unique_doc_rules(rule_str, customize)
+            rule = (
+                "async_call ::= expr "
+                + ("expr " * args_pos)
+                + ("kwarg " * args_kw)
+                + "expr " * nak
+                + token.kind
+                + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
+            )
+            self.add_unique_rule(rule, token.kind, uniq_param, customize)
+            self.add_unique_rule(
+                "expr ::= async_call", token.kind, uniq_param, customize
+            )
+
+        if opname.startswith("CALL_FUNCTION_KW"):
+            self.addRule("expr ::= call_kw36", nop_func)
+            values = "expr " * token.attr
+            rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
+            self.add_unique_rule(rule, token.kind, token.attr, customize)
+        elif opname == "CALL_FUNCTION_EX_KW":
+            # Note that we don't add to customize token.kind here. Instead, the non-terminal
+            # names call_ex_kw... are is in semantic actions.
+            self.addRule(
+                """expr        ::= call_ex_kw4
+                                   call_ex_kw4 ::= expr
+                                   expr
+                                   expr
+                                   CALL_FUNCTION_EX_KW
+                """,
+                nop_func,
+            )
+            if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                self.addRule(
+                    """expr        ::= call_ex_kw
+                       call_ex_kw  ::= expr expr build_map_unpack_with_call
+                                       CALL_FUNCTION_EX_KW
+                    """,
+                    nop_func,
+                )
+            if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                # FIXME: should this be parameterized by EX value?
+                self.addRule(
+                    """expr        ::= call_ex_kw3
+                                       call_ex_kw3 ::= expr
+                                       build_tuple_unpack_with_call
+                                       expr
+                                       CALL_FUNCTION_EX_KW
+                    """,
+                    nop_func,
+                )
+                if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                    # FIXME: should this be parameterized by EX value?
+                    self.addRule(
+                        """expr        ::= call_ex_kw2
+                                           call_ex_kw2 ::= expr
+                                           build_tuple_unpack_with_call
+                                           build_map_unpack_with_call
+                                           CALL_FUNCTION_EX_KW
+                        """,
+                        nop_func,
+                    )
+
+        elif opname == "CALL_FUNCTION_EX":
+            self.addRule(
+                """
+                expr        ::= call_ex
+                starred     ::= expr
+                call_ex     ::= expr starred CALL_FUNCTION_EX
+                """,
+                nop_func,
+            )
+            if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
+                self.addRule(
+                    """
+                        expr        ::= call_ex_kw
+                        call_ex_kw  ::= expr expr
+                                        build_map_unpack_with_call CALL_FUNCTION_EX
+                        """,
+                    nop_func,
+                )
+            if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
+                self.addRule(
+                    """
+                        expr        ::= call_ex_kw3
+                        call_ex_kw3 ::= expr
+                                        build_tuple_unpack_with_call
+                                        %s
+                                        CALL_FUNCTION_EX
+                        """
+                    % "expr "
+                    * token.attr,
+                    nop_func,
+                )
+                pass
+
+            # FIXME: Is this right?
+            self.addRule(
+                """
+                        expr        ::= call_ex_kw4
+                        call_ex_kw4 ::= expr
+                                        expr
+                                        expr
+                                        CALL_FUNCTION_EX
+                        """,
+                nop_func,
+            )
+            pass
+        else:
+            Python37BaseParser.custom_classfunc_rule(
+                self, opname, token, customize, next_token
+            )

python/py/Lib/site-packages/decompyle3/parsers/p37/lambda_expr.py

@@ -0,0 +1,716 @@
+#  Copyright (c) 2017-2023 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Python 3.7 lambda grammar for the spark Earley-algorithm parser.
+"""
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from decompyle3.parsers.p37.lambda_custom import Python37LambdaCustom
+from decompyle3.parsers.parse_heads import PythonBaseParser, PythonParserLambda
+
+
+class Python37LambdaParser(Python37LambdaCustom, PythonParserLambda):
+    def __init__(
+        self,
+        start_symbol: str = "lambda_start",
+        debug_parser: dict = PARSER_DEFAULT_DEBUG,
+    ):
+        PythonParserLambda.__init__(
+            self, debug_parser=debug_parser, start_symbol=start_symbol
+        )
+        PythonBaseParser.__init__(
+            self, start_symbol=start_symbol, debug_parser=debug_parser
+        )
+        Python37LambdaCustom.__init__(self)
+
+    def customize_grammar_rules(self, tokens, customize):
+        self.customize_grammar_rules_lambda37(tokens, customize)
+
+    ###################################################
+    #  Python 3.7 grammar rules for lambda expressions
+    ###################################################
+    pass
+
+    def p_lambda(self, args):
+        """
+        lambda_start       ::= return_expr_lambda LAMBDA_MARKER
+
+        return_expr_lambda ::= expr RETURN_VALUE_LAMBDA
+        return_expr_lambda ::= genexpr_func LOAD_CONST RETURN_VALUE_LAMBDA
+        return_expr_lambda ::= if_exp_lambda
+        return_expr_lambda ::= if_exp_lambda2
+        return_expr_lambda ::= if_exp_not_lambda
+        return_expr_lambda ::= if_exp_not_lambda2
+        return_expr_lambda ::= if_exp_dead_code
+        return_expr_lambda ::= dict_comp_func
+
+        ## FIXME: add rules for these
+        # return_expr_lambda ::= generator_exp
+        # return_expr_lambda ::= list_comp_func
+
+        return_expr_lambda ::= set_comp_func
+
+
+        return_if_lambda   ::= RETURN_END_IF_LAMBDA COME_FROM
+        return_if_lambda   ::= RETURN_END_IF_LAMBDA
+
+        if_exp_lambda      ::= expr_pjif expr return_if_lambda
+                               return_expr_lambda LAMBDA_MARKER
+        if_exp_lambda2     ::= and_parts return_expr_lambda come_froms
+                               return_expr_lambda opt_lambda_marker
+        if_exp_not_lambda  ::= expr POP_JUMP_IF_TRUE expr return_if_lambda
+                               return_expr_lambda LAMBDA_MARKER
+        if_exp_not_lambda2 ::= expr POP_JUMP_IF_TRUE expr
+                               RETURN_VALUE_LAMBDA COME_FROM return_expr_lambda
+        if_exp_dead_code   ::= return_expr_lambda return_expr_lambda
+        opt_lambda_marker  ::= LAMBDA_MARKER?
+        """
+
+    def p_and_or_not(self, args):
+        """
+        # Note: reduction-rule checks are needed for many of the below;
+        # the rules in of themselves are not sufficient.
+
+        # Nonterminals that end in "_cond" are used in "conditions":
+        # used for testing in control structures where the test is important and
+        # the value popped. Conditions also generally have non-local COME_FROMs
+        # that often need to be checked in the control structure. This is for example
+        # how we determine the difference between some "if not (not a or b) versus
+        # "if a and b".
+
+        # FIXME: this is some sort of bool_not or not_cond. Figure out how to have
+        # it not appear in arbitrary expr's
+        not        ::= expr_pjit
+
+        and_parts  ::= expr_pjif+
+
+        # Note: "and" like "nor" might not have a trailing "come_from".
+        #       "nand" and "or", in contrast, *must* have at least one "come_from".
+        not_or       ::= and_parts expr_pjif _come_froms
+
+        and_cond     ::= and_parts expr_pjif _come_froms
+        and_cond     ::= testfalse expr_pjif _come_froms
+        and_not_cond ::= and_not
+
+        # FIXME: Investigate - We don't do the below because these rules prevent the
+        # "and_cond" from triggering.
+
+        # and      ::= and_parts expr
+        # and      ::= not expr
+
+        nand       ::= and_parts expr_pjit  come_froms
+        c_nand     ::= and_parts expr_pjitt come_froms
+
+        or_parts  ::= expr_pjit+
+
+        # Note: "nor" like "and" might not have a trailing "come_from".
+        #       "nand" and "or_cond", in contrast, *must* have at least one "come_from".
+        or_cond     ::= or_parts expr_pjif come_froms
+        or_cond     ::= not_and_not expr_pjif come_froms
+        or_cond1    ::= and POP_JUMP_IF_TRUE come_froms expr_pjif come_from_opt
+
+        nor_cond    ::= or_parts expr_pjif
+
+        # When we alternating and/or's such as:
+        #    a and (b or c) and d
+        # instead of POP_JUMP_IF_TRUE, JUMP_IF_FALSE_OR_POP is sometimes be used
+        # The semantic rules for "and" require expr-like things in positions 0 and 1,
+        # thus the use of expr_jifop_cfs below.
+
+        expr_jifop_cfs ::= expr JUMP_IF_FALSE_OR_POP _come_froms
+        and            ::= expr_jifop_cfs expr _come_froms
+
+        or_and         ::= expr_jitop expr come_from_opt JUMP_IF_FALSE_OR_POP expr
+                           _come_froms
+        or_and1        ::= or_parts and_parts come_froms
+        and_or         ::= expr_jifop expr come_from_opt JUMP_IF_TRUE_OR_POP expr
+                          _come_froms
+
+        ## A COME_FROM is dropped off because of JUMP-to-JUMP optimization
+        # and       ::= expr_pjif expr
+
+        ## Note that "POP_JUMP_IF_FALSE" is what we check on in the "and" reduce rule.
+        # and       ::= expr_pjif expr COME_FROM
+
+        jump_if_false_cf ::= POP_JUMP_IF_FALSE COME_FROM
+        and_or_cond      ::= and_parts expr POP_JUMP_IF_TRUE come_froms expr_pjif
+                             _come_froms
+
+        # For "or", keep index 0 and 1 be the two expressions.
+
+        or        ::= or_parts   expr
+        or        ::= expr_pjit  expr COME_FROM
+        or        ::= expr_pjit  expr jump_if_false_cf
+
+        # Note: in the "or below", if "come_from_opt" becomes
+        # _come_froms, then we will need to write a check to make sure
+        # *all* of the COME_FROMs are associated with the
+        # "or".
+        #
+        # Otherwise, in 3.8 we may turn:
+        #     i and j or k # i == i and (j or k)
+        #  erroneously into:
+        #     i and (j or k)
+
+        or        ::= expr_jitop expr come_from_opt
+        or_expr   ::= expr JUMP_IF_TRUE expr COME_FROM
+
+        jitop_come_from_expr ::= JUMP_IF_TRUE_OR_POP _come_froms expr
+        and_or_expr  ::= and_parts expr jitop_come_from_expr COME_FROM
+        """
+
+    def p_come_froms(self, args):
+        """
+        # Zero or one COME_FROM
+        # And/or expressions have this
+        come_from_opt ::= COME_FROM?
+
+        # One or more COME_FROMs - joins of tryelse's have this
+        come_froms    ::= COME_FROM+
+
+        # Zero or more COME_FROMs - loops can have this
+        _come_froms   ::= COME_FROM*
+        _come_froms   ::= COME_FROM_LOOP
+        """
+
+    def p_jump(self, args):
+        """
+        jump               ::= JUMP_FORWARD
+        jump               ::= JUMP_LOOP
+        jump_or_break      ::= jump
+        jump_or_break      ::= BREAK_LOOP
+
+        # These are used to keep parse tree indices the same
+        # in "if"/"else" like rules.
+        jump_forward_else  ::= JUMP_FORWARD _come_froms
+        jump_forward_else  ::= come_froms jump COME_FROM
+
+        pjump_ift          ::= POP_JUMP_IF_TRUE
+        pjump_ift          ::= POP_JUMP_IF_TRUE_LOOP
+
+        pjump_iff          ::= POP_JUMP_IF_FALSE
+        pjump_iff          ::= POP_JUMP_IF_FALSE_LOOP
+
+        # pjump              ::= pjump_iff
+        # pjump              ::= pjump_ift
+        """
+
+    def p_37chained(self, args):
+        """
+        # A compare_chained is two comparisons like x <= y <= z
+        compare_chained     ::= expr compare_chained_middle ROT_TWO POP_TOP _come_froms
+        compare_chained     ::= compare_chained37
+        compare_chained     ::= compare_chained37_false
+
+        compare_chained_and ::= expr chained_parts
+                                compare_chained_righta_false_37
+                                come_froms
+                                POP_TOP JUMP_FORWARD COME_FROM
+                                negated_testtrue
+                                come_froms
+
+        # We don't use testtrue directly because we need to tell the semantic
+        # action to negate the testtrue
+        negated_testtrue ::= testtrue
+
+
+        c_compare_chained   ::= c_compare_chained37_false
+
+        compare_chained37   ::= expr chained_parts
+        compare_chained37   ::= expr compare_chained_middlea_37
+        compare_chained37   ::= expr compare_chained_middlec_37
+        c_compare_chained37   ::= expr c_compare_chained_middlea_37
+        # c_compare_chained37   ::= expr c_compare_chained_middlec_37
+
+        compare_chained37_false   ::= expr compare_chained_middle_false_37
+        compare_chained37_false   ::= expr compare_chained_middleb_false_37
+        compare_chained37_false   ::= expr compare_chained_right_false_37
+
+        c_compare_chained37_false ::= expr c_compare_chained_right_false_37
+        c_compare_chained37_false ::= expr c_compare_chained_middleb_false_37
+        c_compare_chained37_false ::= compare_chained37_false
+
+        compare_chained_middle     ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                       JUMP_IF_FALSE_OR_POP compare_chained_middle
+                                       COME_FROM
+        compare_chained_middle     ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                       JUMP_IF_FALSE_OR_POP compare_chained_right
+                                       COME_FROM
+
+        chained_parts              ::= chained_part+
+
+        chained_part               ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt
+                                       POP_JUMP_IF_FALSE
+
+        chained_part               ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt
+
+        # c_chained_parts            ::= c_chained_part+
+        # c_chained_part             ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt
+                                         POP_JUMP_IF_FALSE_LOOP
+        # c_chained_parts            ::= chained_parts
+
+
+        compare_chained_middlea_37       ::= chained_parts
+                                       compare_chained_righta_37 COME_FROM
+                                       POP_TOP come_from_opt
+        c_compare_chained_middlea_37     ::= chained_parts
+                                       c_compare_chained_righta_37 COME_FROM
+                                       POP_TOP come_from_opt
+
+        compare_chained_middleb_false_37 ::= chained_parts
+                                       compare_chained_rightb_false_37
+                                       POP_TOP jump _come_froms
+
+        c_compare_chained_middleb_false_37 ::= chained_parts
+                                         c_compare_chained_rightb_false_37 POP_TOP jump
+                                         _come_froms
+        c_compare_chained_middleb_false_37 ::= chained_parts
+                                         compare_chained_rightb_false_37 POP_TOP jump
+                                         _come_froms
+
+        compare_chained_middlec_37       ::= chained_parts
+                                             compare_chained_righta_37 POP_TOP
+
+        compare_chained_middle_false_37  ::= chained_parts
+                                             compare_chained_rightc_37 POP_TOP JUMP_FORWARD
+                                             come_from_opt
+        compare_chained_middle_false_37  ::= chained_parts
+                                             compare_chained_rightb_false_37 POP_TOP jump
+                                             COME_FROM
+
+        compare_chained_right           ::= expr COMPARE_OP JUMP_FORWARD
+        compare_chained_right           ::= expr COMPARE_OP RETURN_VALUE
+        compare_chained_right           ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
+
+        compare_chained_right_false_37  ::= chained_parts
+                                            compare_chained_righta_false_37 POP_TOP
+                                            JUMP_LOOP COME_FROM
+        c_compare_chained_right_false_37 ::= chained_parts
+                                             c_compare_chained_righta_false_37 POP_TOP
+                                             JUMP_LOOP COME_FROM
+
+        compare_chained_righta_37       ::= expr COMPARE_OP come_from_opt
+                                            POP_JUMP_IF_TRUE JUMP_FORWARD
+        c_compare_chained_righta_37     ::= expr COMPARE_OP come_from_opt
+                                            POP_JUMP_IF_TRUE_LOOP JUMP_FORWARD
+
+
+        compare_chained_righta_37       ::= expr COMPARE_OP come_from_opt
+                                            POP_JUMP_IF_TRUE JUMP_LOOP
+        compare_chained_righta_false_37 ::= expr COMPARE_OP come_from_opt
+                                            POP_JUMP_IF_FALSE jf_cfs
+
+
+        compare_chained_rightb_false_37   ::= expr COMPARE_OP come_from_opt
+                                              POP_JUMP_IF_FALSE
+                                              jump_or_break COME_FROM
+        c_compare_chained_rightb_false_37 ::= expr COMPARE_OP come_from_opt
+                                              POP_JUMP_IF_FALSE_LOOP jump_or_break
+                                              COME_FROM
+        c_compare_chained_righta_false_37 ::= expr COMPARE_OP come_from_opt
+                                              POP_JUMP_IF_FALSE_LOOP jf_cfs
+        c_compare_chained_righta_false_37 ::= expr COMPARE_OP come_from_opt
+                                              POP_JUMP_IF_FALSE_LOOP
+        c_compare_chained_rightb_false_37 ::= expr COMPARE_OP come_from_opt
+                                              JUMP_FORWARD COME_FROM
+
+
+        compare_chained_rightc_37          ::= chained_parts
+                                               compare_chained_righta_false_37
+        """
+
+    def p_expr(self, args):
+        """
+        expr ::= LOAD_CODE
+        expr ::= LOAD_CONST
+        expr ::= LOAD_DEREF
+        expr ::= LOAD_FAST
+        expr ::= LOAD_GLOBAL
+        expr ::= LOAD_NAME
+        expr ::= LOAD_STR
+
+        expr ::= and
+        expr ::= and_or
+        expr ::= and_or_expr
+        expr ::= attribute37
+        expr ::= bin_op
+        expr ::= call
+        expr ::= compare
+        expr ::= genexpr_func
+        expr ::= if_exp
+        expr ::= if_exp_loop
+        expr ::= list_comp
+        expr ::= not
+        expr ::= or
+        expr ::= or_and
+        expr ::= or_expr
+        expr ::= set_comp
+        expr ::= subscript
+        expr ::= subscript2
+        expr ::= unary_not
+        expr ::= unary_op
+        expr ::= yield
+
+        # Python 3.3+ adds yield from.
+        expr          ::= yield_from
+        yield_from    ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
+
+        attribute37       ::= expr LOAD_METHOD
+
+        # bin_op (formerly "binary_expr") is the Python AST BinOp
+        bin_op            ::= expr expr binary_operator
+
+        binary_operator   ::= BINARY_ADD
+        binary_operator   ::= BINARY_AND
+        binary_operator   ::= BINARY_FLOOR_DIVIDE
+        binary_operator   ::= BINARY_LSHIFT
+        binary_operator   ::= BINARY_MATRIX_MULTIPLY
+        binary_operator   ::= BINARY_MODULO
+        binary_operator   ::= BINARY_MULTIPLY
+        binary_operator   ::= BINARY_OR
+        binary_operator   ::= BINARY_POWER
+        binary_operator   ::= BINARY_RSHIFT
+        binary_operator   ::= BINARY_SUBTRACT
+        binary_operator   ::= BINARY_TRUE_DIVIDE
+        binary_operator   ::= BINARY_XOR
+
+        # FIXME: the below is to work around test_grammar expecting a "call" to be
+        # on the LHS because it is also somewhere on in a rule.
+        call              ::= expr CALL_METHOD_0
+
+        compare           ::= compare_chained
+        compare           ::= compare_single
+        compare_single    ::= expr expr COMPARE_OP
+        c_compare         ::= c_compare_chained
+
+        genexpr_func      ::= LOAD_ARG _come_froms FOR_ITER store comp_iter
+                              _come_froms JUMP_LOOP _come_froms
+
+        load_genexpr      ::= LOAD_GENEXPR
+        load_genexpr      ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR
+
+        subscript         ::= expr expr BINARY_SUBSCR
+        subscript2        ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
+
+        # unary_op (formerly "unary_expr") is the Python AST UnaryOp
+        unary_op          ::= expr unary_operator
+
+        unary_operator    ::= UNARY_POSITIVE
+        unary_operator    ::= UNARY_NEGATIVE
+        unary_operator    ::= UNARY_INVERT
+
+        unary_not         ::= expr UNARY_NOT
+
+        yield             ::= expr YIELD_VALUE
+        """
+
+    def p_comprehension_list(self, args):
+        """
+        lc_body         ::= expr LIST_APPEND
+        list_comp       ::= BUILD_LIST_0 list_iter
+
+        list_iter       ::= list_for
+        list_iter       ::= list_if
+        list_iter       ::= list_if_not
+        list_iter       ::= list_if_or_not
+        list_iter       ::= lc_body
+
+        set_iter        ::= set_for
+        set_iter        ::= list_if
+        # set_iter        ::= list_if_and_or
+        # set_iter        ::= list_if_chained
+        set_iter        ::= list_if_not
+        set_iter        ::= set_comp_body
+
+        list_for  ::= expr_or_arg
+                      for_iter
+                      store list_iter
+                      jb_or_c _come_froms
+
+        set_for   ::= expr_or_arg
+                      for_iter
+                      store set_iter
+                      jb_or_c _come_froms
+
+        list_if_not_end ::= pjump_ift _come_froms
+        list_if_not ::= expr list_if_not_end list_iter come_from_opt
+
+        list_if     ::= expr pjump_iff list_iter come_from_opt
+        list_if     ::= expr jump_if_false_cf   list_iter
+        list_if_or_not ::= expr_pjit expr_pjit COME_FROM list_iter
+
+        list_if_end ::= pjump_iff _come_froms
+        list_if     ::= expr list_if_end list_iter come_from_opt
+
+        jb_or_c ::= JUMP_LOOP
+        jb_or_c ::= CONTINUE
+
+
+        """
+
+    def p_37conditionals(self, args):
+        """
+        expr                       ::= if_exp_compare
+        bool_op                    ::= and_cond
+        bool_op                    ::= and_not_cond
+        bool_op                    ::= and POP_JUMP_IF_TRUE expr
+
+        expr_pjif                  ::= expr POP_JUMP_IF_FALSE
+        expr_pjit                  ::= expr POP_JUMP_IF_TRUE
+        expr_pjitt                 ::= expr pjump_ift
+        expr_jifop                 ::= expr JUMP_IF_FALSE_OR_POP
+        expr_jitop                 ::= expr JUMP_IF_TRUE_OR_POP
+        expr_pjiff                 ::= expr pjump_iff
+        expr_pjift                 ::= expr pjump_ift
+
+        if_exp                     ::= expr_pjif expr jump_forward_else expr come_froms
+
+        if_exp_compare             ::= expr expr jf_cfs expr COME_FROM
+        if_exp_compare             ::= bool_op expr jf_cfs expr COME_FROM
+
+        if_exp_loop                ::= expr_pjif
+                                       expr
+                                       POP_JUMP_IF_FALSE_LOOP
+                                       JUMP_FORWARD
+                                       come_froms
+                                       expr
+
+        jf_cfs                     ::= JUMP_FORWARD _come_froms
+        list_iter                  ::= list_if37
+        list_iter                  ::= list_if37_not
+        list_if37                  ::= c_compare_chained37_false list_iter
+        list_if37_not              ::= compare_chained37 list_iter
+
+        # A reduction check distinguishes between "and" and "and_not"
+        # based on whether the POP_IF_JUMP location matches the location of the
+        # POP_JUMP_IF_FALSE.
+        and_not                    ::= expr_pjif expr_pjit
+        or_and_not                 ::= expr_pjit and_not COME_FROM
+
+        not_and_not                ::= not expr_pjif COME_FROM
+
+        expr                       ::= if_exp_37a
+        expr                       ::= if_exp_37b
+        if_exp_37a                 ::= and_not expr JUMP_FORWARD come_froms expr COME_FROM
+        if_exp_37b                 ::= expr_pjif expr_pjif jump_forward_else expr
+        """
+
+    def p_comprehension(self, args):
+        """
+        # Python3 scanner adds LOAD_LISTCOMP. Python3 does list comprehension like
+        # other comprehensions (set, dictionary).
+
+        comp_body      ::= dict_comp_body
+        comp_body      ::= gen_comp_body
+        # FIXME: decompile-cfg has this. We are missing a LHS rule?
+        # comp_body      ::= list_comp_body
+        comp_body      ::= set_comp_body
+
+        # Our "continue" heuristic -  in two successive JUMP_LOOPS, the first
+        # one may be a continue - sometimes classifies a JUMP_LOOP
+        # as a CONTINUE. The two are kind of the same in a comprehension.
+
+        comp_for       ::= expr get_for_iter store comp_iter
+                           CONTINUE
+                           _come_froms
+
+        comp_for       ::= expr get_for_iter store comp_iter
+                           JUMP_LOOP
+                           _come_froms
+
+        get_for_iter   ::= GET_ITER _come_froms FOR_ITER
+
+        dict_comp_body ::= expr expr MAP_ADD
+        set_comp_body  ::= expr SET_ADD
+
+        # See also common Python p_list_comprehension
+
+        comp_if        ::= expr_pjif comp_iter
+        comp_if         ::= expr_pjiff comp_iter
+        comp_if         ::= c_compare comp_iter
+        comp_if         ::= or_jump_if_false_cf comp_iter
+        comp_if         ::= or_jump_if_false_loop_cf comp_iter
+
+        # We need to have a reduction rule to disambiguate
+        # these "comp_if_not" and "comp_if". The difference is buried in the
+        # sense of the jump in
+        #     comp_iter -> comp_if_or -> or_parts_false_loop
+        # vs.:
+        #    comp_iter -> comp_if_or -> or_parts_true_loop
+        #
+        # If "true_loop then that goes with "comp_if_not"
+        # if "false_loop"  then that goes with comp_if"
+        #
+        # We might be able to do this in the grammar but it is a bit
+        # too pervasive and involved.
+
+        # We have a bunch of these comp_if_<logic expression>
+        # because the logic operation bleeds into the
+        # "if" of the comprehension. Note thet specific position of
+        # POP_JUMP_IF_xxx_LOOP stays the same.
+        comp_if_or      ::= or_parts
+                            expr POP_JUMP_IF_FALSE_LOOP
+                            come_froms
+                            comp_iter
+        # comp_if_or      ::= or_parts_true_loop
+        #                     expr POP_JUMP_IF_FALSE_LOOP
+        #                     come_froms
+        #                     comp_iter
+
+        # comp_if_or      ::= or_parts_false_loop
+        #                     expr POP_JUMP_IF_FALSE_LOOP
+        #                     come_froms
+        #                     comp_iter
+
+        # Here, the "or" is melded a little into the "comp_if" test
+        comp_if_or2     ::= compare compare_chained37_false comp_iter
+
+        comp_if_or_not  ::= or_parts
+                            expr POP_JUMP_IF_TRUE_LOOP
+                            come_froms
+                            comp_iter
+        ## FIXME: we add this, per comment above later.
+        ## comp_if         ::= expr pjump_ift comp_iter
+        comp_if_not     ::= expr pjump_ift comp_iter
+
+
+        comp_if_not_and ::= expr_pjif
+                            expr POP_JUMP_IF_TRUE_LOOP
+                            come_froms
+                            comp_iter
+        comp_if_not_or  ::= expr_pjif
+                            expr POP_JUMP_IF_FALSE_LOOP
+                            come_from_opt
+                            comp_iter
+
+        comp_iter     ::= dict_comp_body
+        comp_iter     ::= comp_body
+        comp_iter     ::= comp_if
+        comp_iter     ::= comp_if_not
+        comp_iter     ::= comp_if_not_and
+        comp_iter     ::= comp_if_not_or
+        comp_iter     ::= comp_if_or
+        comp_iter     ::= comp_if_or_not
+        comp_iter     ::= comp_if_or2
+
+        or_jump_if_false_cf      ::= or POP_JUMP_IF_FALSE COME_FROM
+        or_jump_if_false_loop_cf ::= or_loop POP_JUMP_IF_FALSE_LOOP COME_FROM
+
+        or_loop       ::= or
+        or_loop       ::= or_parts_loop expr
+        or_parts_loop ::= expr_pjift+
+
+        # Semantic rules require "comp_if" to have index 0 be some
+        # sort of "expr" and index 1 to be some sort of "comp_iter"
+        c_compare     ::= compare
+
+        expr_or_arg     ::= LOAD_ARG
+        expr_or_arg     ::= expr
+
+        ending_return  ::= RETURN_VALUE RETURN_LAST
+        ending_return  ::= RETURN_VALUE_LAMBDA LAMBDA_MARKER
+
+        for_iter       ::= _come_froms FOR_ITER
+        dict_comp_func ::= BUILD_MAP_0 LOAD_ARG for_iter store
+                           comp_iter JUMP_LOOP _come_froms
+                           ending_return
+
+        set_comp_func   ::= BUILD_SET_0
+                            expr_or_arg
+                            for_iter store comp_iter
+                            JUMP_LOOP
+                            _come_froms
+                            ending_return
+
+        set_comp_func   ::= BUILD_SET_0
+                            expr_or_arg
+                            for_iter store comp_iter
+                            COME_FROM
+                            JUMP_LOOP
+                            _come_froms
+                            ending_return
+
+        await_expr       ::= expr GET_AWAITABLE LOAD_CONST YIELD_FROM
+        set_comp_func    ::= BUILD_SET_0
+                             expr_or_arg
+                             for_iter store await_expr
+                             SET_ADD
+                             JUMP_LOOP
+                             _come_froms
+                             ending_return
+        """
+
+    def p_expr3(self, args):
+        """
+        expr               ::= if_exp_not
+        if_exp_not         ::= expr POP_JUMP_IF_TRUE expr jump_forward_else expr COME_FROM
+
+        # a JUMP_FORWARD to another JUMP_FORWARD can get turned into
+        # a JUMP_ABSOLUTE with no COME_FROM
+        if_exp             ::= expr_pjif expr jump_forward_else expr
+
+        # if_exp_true are are IfExp which always evaluate true, e.g.:
+        #      x = a if 1 else b
+        # There is dead or non-optional remnants of the condition code though,
+        # and we use that to match on to reconstruct the source more accurately
+        expr           ::= if_exp_true
+        if_exp_true    ::= expr JUMP_FORWARD expr COME_FROM
+
+        """
+
+    def p_set_comp(self, args):
+        """
+        comp_iter     ::= comp_for
+        gen_comp_body ::= expr YIELD_VALUE POP_TOP
+        set_comp      ::= BUILD_SET_0 set_iter
+        """
+
+    def p_store(self, args):
+        """
+        # Note. The below is right-recursive:
+        designList ::= store store
+        designList ::= store DUP_TOP designList
+
+        ## Can we replace with left-recursive, and redo with:
+        ##
+        ##   designList  ::= designLists store store
+        ##   designLists ::= designLists store DUP_TOP
+        ##   designLists ::=
+        ## Will need to redo semantic actiion
+
+        store           ::= STORE_FAST
+        store           ::= STORE_NAME
+        store           ::= STORE_GLOBAL
+        store           ::= STORE_DEREF
+        store           ::= expr STORE_ATTR
+        store           ::= store_subscript
+        store_subscript ::= expr expr STORE_SUBSCR
+        """
+
+
+if __name__ == "__main__":
+    # Check grammar
+    from decompyle3.parsers.dump import dump_and_check
+
+    p = Python37LambdaParser()
+    modified_tokens = set(
+        """JUMP_LOOP CONTINUE RETURN_END_IF_LAMBDA COME_FROM
+           LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+           LAMBDA_MARKER RETURN_VALUE_LAMBDA
+        """.split()
+    )
+
+    dump_and_check(p, (3, 7), modified_tokens)

python/py/Lib/site-packages/decompyle3/parsers/p38/__init__.py

@@ -0,0 +1,4 @@
+"""
+Here we have Python 3.8 grammars and associated customization
+for the both full language and the subset used in lambda expressions.
+"""

python/py/Lib/site-packages/decompyle3/parsers/p38/base.py

@@ -0,0 +1,54 @@
+#  Copyright (c) 2020-2022, 2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Python 3.8 base code. We keep non-custom-generated grammar rules out of this file.
+"""
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from decompyle3.parsers.parse_heads import PythonBaseParser
+from decompyle3.parsers.reduce_check import (
+    break_invalid,
+    for38_invalid,
+    forelse38_invalid,
+    pop_return_check,
+    whilestmt38_check,
+    whileTruestmt38_check,
+)
+
+
+class Python38BaseParser(PythonBaseParser):
+    def __init__(self, start_symbol, debug_parser: dict = PARSER_DEFAULT_DEBUG):
+        super(Python38BaseParser, self).__init__(
+            start_symbol=start_symbol, debug_parser=debug_parser
+        )
+
+    def customize_grammar_rules38(self, tokens, customize):
+        self.customize_grammar_rules37(tokens, customize)
+        self.check_reduce["break"] = "tokens"
+        self.check_reduce["for38"] = "tokens"
+        self.check_reduce["forelsestmt38"] = "AST"
+        self.check_reduce["pop_return"] = "tokens"
+        self.check_reduce["whileTruestmt38"] = "AST"
+        self.check_reduce["whilestmt38"] = "tokens"
+        self.check_reduce["try_elsestmtl38"] = "AST"
+
+        self.reduce_check_table["break"] = break_invalid
+        self.reduce_check_table["for38"] = for38_invalid
+        self.reduce_check_table["forelsestmt38"] = forelse38_invalid
+        self.reduce_check_table["pop_return"] = pop_return_check
+        self.reduce_check_table["whilestmt38"] = whilestmt38_check
+        self.reduce_check_table["whileTruestmt38"] = whileTruestmt38_check

python/py/Lib/site-packages/decompyle3/parsers/p38/full.py

@@ -0,0 +1,680 @@
+#  Copyright (c) 2017-2023 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+spark grammar differences over Python 3.7 for Python 3.8
+"""
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from spark_parser.spark import rule2str
+
+from decompyle3.parsers.p37.full import Python37Parser
+from decompyle3.parsers.p38.full_custom import Python38FullCustom
+from decompyle3.parsers.p38.lambda_expr import Python38LambdaParser
+from decompyle3.parsers.parse_heads import ParserError
+from decompyle3.scanners.tok import Token
+
+
+class Python38Parser(Python38LambdaParser, Python38FullCustom, Python37Parser):
+    def __init__(self, start_symbol: str = "stmts", debug_parser=PARSER_DEFAULT_DEBUG):
+        Python38LambdaParser.__init__(self, start_symbol, debug_parser)
+        self.customized = {}
+
+    def customize_grammar_rules(self, tokens, customize):
+        self.customize_grammar_rules_full38(tokens, customize)
+
+    ###############################################
+    #  Python 3.8 grammar rules with statements
+    ###############################################
+
+    def p_38_full_if_ifelse(self, args):
+        """
+        # cf_pt introduced to keep indices the same in ifelsestmtc
+        cf_pt              ::= COME_FROM POP_TOP
+        ifelsestmtc        ::= testexpr c_stmts cf_pt else_suite
+
+        # 3.8 can push a looping JUMP_LOOP into a JUMP_ from a statement that jumps to
+        # it
+        lastc_stmt         ::= ifpoplaststmtc
+        ifpoplaststmtc     ::= testexpr POP_TOP c_stmts_opt
+        ifelsestmtc        ::= testexpr c_stmts_opt jb_cfs else_suitec JUMP_LOOP
+                               come_froms
+
+        testtrue   ::= or_in_ifexp POP_JUMP_IF_TRUE
+
+
+        # The below ifelsetmtc is a really weird one for the inner if/else in:
+        #  if a:
+        #      while i:
+        #       if c:
+        #         j = j + 1
+        #                 # A JUMP_LOOP is here...
+        #       else:
+        #          break
+        #                 # but also a JUMP_LOOP is inserted here!
+        #  else:
+        #    j = 10
+
+        ifelsestmtc        ::= testexpr c_stmts_opt JUMP_LOOP else_suitec JUMP_LOOP
+        """
+
+    def p_38_full_stmt(self, args):
+        """
+        stmt               ::= async_with_stmt38
+        stmt               ::= for38
+        stmt               ::= forelselaststmt38
+        stmt               ::= forelselaststmtc38
+        stmt               ::= forelsestmt38
+        stmt               ::= try_elsestmtl38
+        stmt               ::= try_except38
+        stmt               ::= try_except38r
+        stmt               ::= try_except38r2
+        stmt               ::= try_except38r3
+        stmt               ::= try_except38r4
+        stmt               ::= try_except38r5
+        stmt               ::= try_except38r6
+        stmt               ::= try_except38r7
+        stmt               ::= try_except_as
+        stmt               ::= try_except_ret38
+        stmt               ::= try_except_ret38a
+        stmt               ::= tryfinallystmt_break
+        stmt               ::= tryfinally38astmt
+        stmt               ::= tryfinally38rstmt
+        stmt               ::= tryfinally38rstmt2
+        stmt               ::= tryfinally38rstmt3
+        stmt               ::= tryfinally38rstmt4
+        stmt               ::= tryfinally38rstmt5
+        stmt               ::= tryfinally38stmt
+        stmt               ::= tryfinally38_return
+        stmt               ::= tryfinally38a_return
+        stmt               ::= tryfinally38rstmt2
+        stmt               ::= whileTruestmt38
+        stmt               ::= whilestmt38
+
+        # FIXME: "break"" should be isolated to loops
+        stmt  ::= break
+
+        break ::= POP_BLOCK BREAK_LOOP
+        break ::= POP_BLOCK POP_TOP BREAK_LOOP
+        break ::= POP_TOP BREAK_LOOP
+        break ::= POP_EXCEPT BREAK_LOOP
+        break ::= POP_TOP CONTINUE JUMP_LOOP
+
+        # An except with nothing other than a single break
+        break_except ::= POP_EXCEPT POP_TOP BREAK_LOOP
+
+        # FIXME: this should be restricted to being inside a try block
+        stmt               ::= except_ret38
+        stmt               ::= except_ret38a
+
+        async_with_stmt38    ::= expr
+                                 BEFORE_ASYNC_WITH
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 SETUP_ASYNC_WITH
+                                 POP_TOP
+                                 c_stmts_opt
+                                 POP_BLOCK
+                                 BEGIN_FINALLY
+                                 COME_FROM_ASYNC_WITH
+                                 WITH_CLEANUP_START
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 WITH_CLEANUP_FINISH
+                                 END_FINALLY
+
+        async_with_stmt38    ::= expr
+                                 BEFORE_ASYNC_WITH
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 SETUP_ASYNC_WITH
+                                 POP_TOP
+                                 c_stmts_opt
+                                 POP_BLOCK
+                                 BEGIN_FINALLY
+                                 WITH_CLEANUP_START
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 WITH_CLEANUP_FINISH
+                                 POP_FINALLY
+
+        async_with_stmt38    ::= expr
+                                 BEFORE_ASYNC_WITH
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 SETUP_ASYNC_WITH
+                                 POP_TOP
+                                 c_stmts_opt
+                                 POP_BLOCK
+                                 BEGIN_FINALLY
+                                 WITH_CLEANUP_START
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 WITH_CLEANUP_FINISH
+                                 POP_FINALLY
+                                 JUMP_LOOP
+
+        # Seems to be used to discard values before a return in a "for" loop
+        discard_top        ::= ROT_TWO POP_TOP
+        discard_tops       ::= discard_top+
+        pop_tops           ::= POP_TOP+
+
+        return             ::= return_expr
+                               discard_tops RETURN_VALUE
+
+        return             ::= pop_return
+        return             ::= popb_return
+        return             ::= pop_ex_return
+        except_stmt        ::= except_with_break
+        except_stmt        ::= except_with_break2
+        except_stmt        ::= pop_ex_return
+        except_stmt        ::= pop3_except_return38
+        except_stmt        ::= pop3_rot4_except_return38
+        except_stmt        ::= except_cond_pop3_rot4_except_return38
+
+        except_stmts       ::= except_stmt+
+        except_stmts_opt   ::= except_stmt*
+
+        pop_return         ::= POP_TOP return_expr RETURN_VALUE
+        popb_return        ::= return_expr POP_BLOCK RETURN_VALUE
+
+        # Return from exception where value is on stack
+        pop_ex_return      ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
+
+        # Return from exception where value is no on stack but is computed
+        pop_ex_return2      ::= POP_EXCEPT expr RETURN_VALUE
+
+
+        # The below are 3.8 "except:" (no except condition)
+
+        pop3_except_return38       ::= POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                                       CALL_FINALLY return
+
+        except_return38            ::= POP_BLOCK
+                                       CALL_FINALLY POP_TOP return
+
+        pop3_rot4_except_return38  ::= POP_TOP POP_TOP POP_TOP
+                                       except_stmts_opt return_expr ROT_FOUR
+                                       POP_EXCEPT POP_BLOCK CALL_FINALLY RETURN_VALUE
+
+
+        pop3_rot4_except_return38  ::= POP_TOP POP_TOP POP_TOP
+                                       except_stmts_opt return_expr ROT_FOUR
+                                       POP_EXCEPT POP_BLOCK ROT_TWO POP_TOP
+                                       CALL_FINALLY RETURN_VALUE
+                                       END_FINALLY COME_FROM POP_BLOCK
+                                       BEGIN_FINALLY COME_FROM
+
+        # The above but with an except condition name e.g. "except Exception:"
+        except_cond_pop3_rot4_except_return38 ::= except_cond1
+                                       except_stmts_opt return_expr ROT_FOUR
+                                       POP_EXCEPT POP_BLOCK CALL_FINALLY RETURN_VALUE
+                                       COME_FROM
+
+        except_stmt        ::= except_cond1 except_suite come_from_opt
+        except_stmt        ::= except_cond2 except_ret38b
+
+        get_iter           ::= expr GET_ITER
+        for38              ::= expr get_iter store for_block JUMP_LOOP _come_froms
+        for38              ::= expr get_for_iter store for_block JUMP_LOOP _come_froms
+        for38              ::= expr get_for_iter store for_block JUMP_LOOP _come_froms
+                               POP_BLOCK
+        for38              ::= expr get_for_iter store for_block _come_froms
+
+        forelsestmt38      ::= expr get_for_iter store for_block POP_BLOCK else_suite
+                               _come_froms
+        forelsestmt38      ::= expr get_for_iter store for_block JUMP_LOOP _come_froms
+                               else_suite _come_froms
+
+        c_stmt             ::= c_forelsestmt38
+        c_stmt             ::= pop_tops return
+        c_forelsestmt38    ::= expr get_for_iter store for_block POP_BLOCK else_suitec
+        c_forelsestmt38    ::= expr get_for_iter store for_block JUMP_LOOP _come_froms
+                               else_suitec
+
+        # continue is a weird one. In 3.8, CONTINUE_LOOP was removed.
+        # Inside an loop we can have this, which can only appear in side a try/except
+        # And it can also appear at the end of the try except.
+        continue           ::= POP_EXCEPT JUMP_LOOP
+
+        forelselaststmt38    ::= expr get_for_iter store for_block else_suitec
+                                 _come_froms
+        forelselaststmtc38   ::= expr get_for_iter store for_block else_suitec
+                                 _come_froms
+        # forelselaststmt38  ::= expr get_for_iter store for_block POP_BLOCK else_suitec
+        # forelselaststmtc38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec
+
+        returns_in_except   ::= _stmts except_return_value
+        returns_in_except2   ::= _stmts except_return_value2
+
+        except_return_value ::= POP_BLOCK return
+        except_return_value ::= expr POP_BLOCK RETURN_VALUE
+        except_return_value2 ::= POP_BLOCK return
+
+        whilestmt38        ::= _come_froms testexpr  c_stmts_opt COME_FROM JUMP_LOOP
+                                POP_BLOCK
+        whilestmt38        ::= _come_froms testexpr  c_stmts_opt JUMP_LOOP POP_BLOCK
+        whilestmt38        ::= _come_froms testexpr  c_stmts_opt JUMP_LOOP come_froms
+        whilestmt38        ::= _come_froms testexprc c_stmts_opt come_froms JUMP_LOOP
+                               _come_froms
+        whilestmt38        ::= _come_froms testexpr  returns               POP_BLOCK
+        whilestmt38        ::= _come_froms testexpr  c_stmts     JUMP_LOOP _come_froms
+        whilestmt38        ::= _come_froms testexpr  c_stmts     come_froms
+        whilestmt38        ::= _come_froms bool_op   c_stmts     JUMP_LOOP _come_froms
+
+        # while1elsestmt   ::=  c_stmts JUMP_LOOP
+        whileTruestmt      ::= _come_froms c_stmts              JUMP_LOOP _come_froms
+                               POP_BLOCK
+        while1stmt         ::= _come_froms c_stmts COME_FROM_LOOP
+        while1stmt         ::= _come_froms c_stmts COME_FROM JUMP_LOOP COME_FROM_LOOP
+        whileTruestmt38    ::= _come_froms c_stmts JUMP_LOOP _come_froms
+        whileTruestmt38    ::= _come_froms c_stmts JUMP_LOOP COME_FROM_EXCEPT_CLAUSE
+        whileTruestmt38    ::= _come_froms pass JUMP_LOOP
+
+        for_block          ::= _come_froms c_stmts_opt come_from_loops JUMP_LOOP
+
+        # Note there is a 3.7 except_cond1 that doesn't have the final POP_EXCEPT
+        except_cond1       ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP
+                               POP_EXCEPT
+
+        except_suite       ::= c_stmts_opt
+                               POP_EXCEPT POP_TOP JUMP_FORWARD POP_EXCEPT
+                               jump_except
+
+        try_elsestmtl38    ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               except_handler38 COME_FROM
+                               else_suitec opt_come_from_except
+        try_except         ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               except_handler38
+        try_except         ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               except_handler38
+                               jump_excepts
+                               come_from_except_clauses
+
+        c_try_except       ::= SETUP_FINALLY c_suite_stmts POP_BLOCK
+                               except_handler38
+
+        c_stmt             ::= c_tryfinallystmt38
+        c_stmt             ::= c_tryfinallybstmt38
+
+        c_tryfinallystmt38 ::= SETUP_FINALLY c_suite_stmts_opt
+                               POP_BLOCK
+                               CALL_FINALLY
+                               POP_BLOCK
+                               POP_EXCEPT
+                               CALL_FINALLY
+                               JUMP_FORWARD
+                               POP_BLOCK BEGIN_FINALLY
+                               COME_FROM
+                               COME_FROM_FINALLY
+                               c_suite_stmts_opt END_FINALLY
+
+        # try:
+        #    ..
+        #    break
+        # finally:
+        c_tryfinallybstmt38 ::= SETUP_FINALLY c_suite_stmts_opt
+                               POP_BLOCK
+                               CALL_FINALLY
+                               POP_BLOCK
+                               POP_EXCEPT
+                               CALL_FINALLY
+                               BREAK_LOOP
+                               POP_BLOCK BEGIN_FINALLY
+                               COME_FROM
+                               COME_FROM_FINALLY
+                               c_suite_stmts_opt END_FINALLY
+
+        c_tryfinallystmt38 ::= SETUP_FINALLY c_suite_stmts_opt
+                               POP_BLOCK BEGIN_FINALLY COME_FROM COME_FROM_FINALLY
+                               c_suite_stmts_opt END_FINALLY
+
+        try_except38       ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
+                               except_handler38a
+
+        # suite_stmts has a return
+        try_except38       ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                               except_handler38b
+        try_except38r      ::= SETUP_FINALLY return_except
+                               except_handler38b
+        return_except      ::= stmts POP_BLOCK return
+
+
+        # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there
+        # is a final return in the "except" block.
+        # So we treat the "return" separate from the other statements
+        cond_except_stmt      ::= except_cond1 except_stmts
+        cond_except_stmts_opt ::= cond_except_stmt*
+
+        try_except38r2     ::= SETUP_FINALLY
+                               suite_stmts_opt
+                               POP_BLOCK JUMP_FORWARD
+                               COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               cond_except_stmts_opt
+                               POP_EXCEPT return
+                               END_FINALLY
+                               COME_FROM
+
+        try_except38r3     ::= SETUP_FINALLY
+                               suite_stmts_opt
+                               POP_BLOCK JUMP_FORWARD
+                               COME_FROM_FINALLY
+                               cond_except_stmts_opt
+                               POP_EXCEPT return
+                               COME_FROM
+                               END_FINALLY
+                               COME_FROM
+
+
+         # I think this can be combined with the r5
+        try_except38r4     ::= SETUP_FINALLY
+                               returns_in_except
+                               COME_FROM_FINALLY
+                               except_cond1
+                               return
+                               COME_FROM
+                               END_FINALLY
+
+        try_except38r5     ::= SETUP_FINALLY
+                               returns_in_except
+                               COME_FROM_FINALLY
+                               except_cond1
+                               except_ret38d
+                               COME_FROM
+                               END_FINALLY
+
+        try_except38r5     ::= SETUP_FINALLY
+                               returns_in_except
+                               COME_FROM_FINALLY
+                               except_cond1
+                               except_suite
+                               COME_FROM
+                               END_FINALLY
+                               COME_FROM
+
+        try_except38r5     ::= SETUP_FINALLY
+                               returns_in_except
+                               COME_FROM_FINALLY
+                               except_cond2
+                               except_ret38b
+                               END_FINALLY COME_FROM
+
+        try_except38r6     ::= SETUP_FINALLY
+                               returns_in_except2
+                               COME_FROM_FINALLY
+                               POP_TOP POP_TOP POP_TOP
+                               except_ret38d
+                               END_FINALLY
+
+
+        try_except38r7     ::= SETUP_FINALLY
+                               suite_stmts_opt
+                               POP_BLOCK JUMP_FORWARD
+                               COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               return_expr
+                               ROT_FOUR POP_EXCEPT POP_BLOCK ROT_TWO POP_TOP
+                               CALL_FINALLY RETURN_VALUE
+                               END_FINALLY
+                               COME_FROM POP_BLOCK
+                               BEGIN_FINALLY
+                               COME_FROM
+                               COME_FROM_FINALLY
+
+
+        try_except_as      ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                               except_handler_as END_FINALLY COME_FROM
+        try_except_as      ::= SETUP_FINALLY suite_stmts
+                               except_handler_as END_FINALLY COME_FROM
+
+
+        try_except_ret38   ::= SETUP_FINALLY returns except_ret38a
+        try_except_ret38a  ::= SETUP_FINALLY returns except_handler38c
+                               END_FINALLY come_from_opt
+
+        # Note: there is a suite_stmts_opt which seems
+        # to be bookkeeping which is not expressed in source code
+        except_ret38       ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT
+                               CALL_FINALLY RETURN_VALUE COME_FROM
+                               COME_FROM_FINALLY
+                               suite_stmts_opt END_FINALLY
+        except_ret38a      ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               expr ROT_FOUR
+                               POP_EXCEPT RETURN_VALUE END_FINALLY
+
+        except_ret38b      ::= SETUP_FINALLY suite_stmts expr
+                               ROT_FOUR POP_BLOCK POP_EXCEPT
+                               CALL_FINALLY RETURN_VALUE COME_FROM
+                               COME_FROM_FINALLY
+                               suite_stmts_opt END_FINALLY
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+
+        except_ret38c      ::= SETUP_FINALLY suite_stmts expr
+                               ROT_FOUR POP_BLOCK POP_EXCEPT
+                               CALL_FINALLY POP_BLOCK CALL_FINALLY RETURN_VALUE
+                               COME_FROM
+                               COME_FROM_FINALLY
+                               expr STORE_FAST DELETE_FAST END_FINALLY
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+                               END_FINALLY come_any_froms
+
+        except_ret38d      ::= suite_stmts_opt
+                               expr ROT_FOUR
+                               POP_EXCEPT RETURN_VALUE
+
+        except_handler38   ::= jump COME_FROM_FINALLY
+                               except_stmts
+                               END_FINALLY
+                               opt_come_from_except
+
+        except_handler38a  ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               POP_EXCEPT POP_TOP stmts END_FINALLY
+        except_handler38b  ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               POP_EXCEPT returns END_FINALLY
+        except_handler38c  ::= COME_FROM_FINALLY except_cond1 except_stmts
+                               COME_FROM
+        except_handler38c  ::= COME_FROM_FINALLY except_cond1 except_stmts
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+
+        except_handler_as  ::= COME_FROM_FINALLY except_cond2 tryfinallystmt
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+
+        except             ::= POP_TOP POP_TOP POP_TOP c_stmts_opt break
+                               POP_EXCEPT
+
+        # Except of a try inside a loop
+        except             ::= POP_TOP POP_TOP POP_TOP c_stmts_opt break
+                               POP_EXCEPT JUMP_LOOP
+
+        except             ::= POP_TOP POP_TOP POP_TOP c_stmts_opt
+                               POP_EXCEPT JUMP_LOOP
+
+        except_with_break  ::= POP_TOP POP_TOP POP_TOP c_stmts break_except
+                               POP_EXCEPT JUMP_LOOP
+
+        # Just except: break, no statements
+        except_with_break2 ::= POP_TOP POP_TOP POP_TOP break_except
+                               POP_EXCEPT JUMP_LOOP
+
+        except_with_return38 ::= POP_TOP POP_TOP POP_TOP stmts pop_ex_return2
+        except_with_return38 ::= POP_TOP POP_TOP POP_TOP pop_ex_return2
+
+        except_stmt         ::= except_with_return38
+
+
+        # In 3.8 any POP_EXCEPT comes before the "break" loop.
+        # We should add a rule to check that JUMP_FORWARD is indeed a "break".
+        break              ::=  POP_EXCEPT JUMP_FORWARD
+        break              ::=  POP_BLOCK POP_TOP JUMP_FORWARD
+
+        tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
+                               END_FINALLY
+
+        tryfinallystmt_break ::=
+                               SETUP_FINALLY suite_stmts_opt POP_BLOCK POP_EXCEPT
+                               CALL_FINALLY
+                               JUMP_FORWARD POP_BLOCK
+                               BEGIN_FINALLY COME_FROM COME_FROM_FINALLY suite_stmts_opt
+                               END_FINALLY
+
+
+        lc_setup_finally   ::= LOAD_CONST SETUP_FINALLY
+        call_finally_pt    ::= CALL_FINALLY POP_TOP
+        cf_cf_finally      ::= come_from_opt COME_FROM_FINALLY
+        pop_finally_pt     ::= POP_FINALLY POP_TOP
+        ss_end_finally     ::= suite_stmts END_FINALLY
+        sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns
+        sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK POP_EXCEPT CALL_FINALLY returns
+
+        suite_stmts_return ::= suite_stmts expr
+        suite_stmts_return ::= expr
+
+
+        # FIXME: DRY rules below
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally
+                               ss_end_finally
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally END_FINALLY
+                               suite_stmts
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally POP_FINALLY
+                               ss_end_finally
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               COME_FROM_FINALLY POP_FINALLY
+                               ss_end_finally
+
+        tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt
+                               returns
+                               cf_cf_finally pop_finally_pt
+                               ss_end_finally POP_TOP
+
+        tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE
+                               COME_FROM COME_FROM_FINALLY
+                               ss_end_finally
+
+        tryfinally38rstmt4 ::= lc_setup_finally suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM_FINALLY
+                               suite_stmts_return
+                               POP_FINALLY ROT_TWO POP_TOP
+                               RETURN_VALUE
+                               END_FINALLY POP_TOP
+
+
+        tryfinally38rstmt5 ::= lc_setup_finally try_except38r7 expr
+                               POP_FINALLY ROT_TWO POP_TOP
+                               RETURN_VALUE
+                               END_FINALLY POP_TOP
+
+        tryfinally38stmt   ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM_FINALLY
+                               POP_FINALLY suite_stmts_opt END_FINALLY
+
+        tryfinally38stmt   ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM
+                               COME_FROM_FINALLY
+                               suite_stmts_opt END_FINALLY
+
+        # try: .. finally: ending with return ...
+        tryfinally38_return ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               JUMP_FORWARD
+                               COME_FROM_FINALLY except_cond2 except_ret38c
+
+
+        tryfinally38a_return ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt except_return38
+                                 COME_FROM COME_FROM_FINALLY
+                                 suite_stmts_opt pop_finally_pt return
+                                 END_FINALLY POP_TOP
+
+
+        tryfinally38astmt  ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM_FINALLY
+                               POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP
+        """
+
+    def p_38_full_walrus(self, args):
+        """
+        # named_expr is also known as the "walrus op" :=
+        expr              ::= named_expr
+        named_expr        ::= expr DUP_TOP store
+        """
+
+    # FIXME: try this
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        lhs = rule[0]
+        if lhs == "call_kw":
+            # Make sure we don't derive call_kw
+            nt = ast[0]
+            while not isinstance(nt, Token):
+                if nt[0] == "call_kw":
+                    return True
+                nt = nt[0]
+                pass
+            pass
+        n = len(tokens)
+        last = min(last, n - 1)
+        fn = self.reduce_check_table.get(lhs, None)
+        try:
+            if fn:
+                return fn(self, lhs, n, rule, ast, tokens, first, last)
+        except Exception:
+            import sys
+            import traceback
+
+            print(
+                f"Exception in {fn.__name__} {sys.exc_info()[1]}\n"
+                + f"rule: {rule2str(rule)}\n"
+                + f"offsets {tokens[first].offset} .. {tokens[last].offset}"
+            )
+            print(traceback.print_tb(sys.exc_info()[2], -1))
+            raise ParserError(tokens[last], tokens[last].off2int(), self.debug["rules"])
+
+        if lhs in ("aug_assign1", "aug_assign2") and ast[0][0] == "and":
+            return True
+        elif lhs == "annotate_tuple":
+            return not isinstance(tokens[first].attr, tuple)
+        elif lhs == "import_from37":
+            importlist37 = ast[3]
+            alias37 = importlist37[0]
+            if importlist37 == "importlist37" and alias37 == "alias37":
+                store = alias37[1]
+                assert store == "store"
+                return alias37[0].attr != store[0].attr
+            return False
+
+        return False
+
+        return False
+
+
+if __name__ == "__main__":
+    # Check grammar
+    from decompyle3.parsers.dump import dump_and_check
+
+    p = Python38Parser()
+    modified_tokens = set(
+        """JUMP_LOOP CONTINUE RETURN_END_IF COME_FROM
+           LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+           LAMBDA_MARKER RETURN_LAST
+        """.split()
+    )
+
+    dump_and_check(p, (3, 8), modified_tokens)

python/py/Lib/site-packages/decompyle3/parsers/p38/full_custom.py

@@ -0,0 +1,1318 @@
+#  Copyright (c) 2021-2023 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# from decompyle3.parsers.reduce_check.import_from37 import import_from37_ok
+from decompyle3.parsers.p37.base import Python37BaseParser
+from decompyle3.parsers.p38.lambda_custom import Python38LambdaCustom
+from decompyle3.parsers.parse_heads import PythonBaseParser, nop_func
+from decompyle3.parsers.reduce_check import (  # joined_str_check,
+    break_invalid,
+    for38_invalid,
+    forelse38_invalid,
+    if_not_stmtc_invalid,
+    pop_return_check,
+    whilestmt38_check,
+    whileTruestmt38_check,
+)
+
+# from decompyle3.parsers.reduce_check.ifelsestmt_check import ifelsestmt_ok
+from decompyle3.parsers.reduce_check.ifstmt import ifstmt
+from decompyle3.parsers.reduce_check.or_cond_check import or_cond_check_invalid
+
+
+class Python38FullCustom(Python38LambdaCustom, PythonBaseParser):
+    def add_make_function_rule(self, rule, opname, attr, customize):
+        """Python 3.3 added an additional LOAD_STR before MAKE_FUNCTION and
+        this has an effect on many rules.
+        """
+        new_rule = rule % "LOAD_STR "
+        self.add_unique_rule(new_rule, opname, attr, customize)
+
+    @staticmethod
+    def call_fn_name(token):
+        """Customize CALL_FUNCTION to add the number of positional arguments"""
+        if token.attr is not None:
+            return f"{token.kind}_{token.attr}"
+        else:
+            return f"{token.kind}_0"
+
+    def remove_rules_38(self):
+        self.remove_rules(
+            """
+           stmt               ::= async_for_stmt37
+           stmt               ::= for
+           stmt               ::= forelsestmt
+           stmt               ::= try_except36
+           stmt               ::= async_forelse_stmt
+
+           # There is no SETUP_LOOP
+           setup_loop         ::= SETUP_LOOP _come_froms
+           forelselaststmt    ::= SETUP_LOOP expr get_for_iter store
+                                  for_block POP_BLOCK else_suitec _come_froms
+
+           forelsestmt        ::= SETUP_LOOP expr get_for_iter store
+           whileTruestmt      ::= SETUP_LOOP c_stmts_opt JUMP_LOOP COME_FROM_LOOP
+                                  for_block POP_BLOCK else_suite _come_froms
+
+           # async_for_stmt     ::= setup_loop expr
+           #                        GET_AITER
+           #                        SETUP_EXCEPT GET_ANEXT LOAD_CONST
+           #                        YIELD_FROM
+           #                        store
+           #                        POP_BLOCK JUMP_FORWARD bb_end_start DUP_TOP
+           #                        LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+           #                        END_FINALLY bb_end_start
+           #                        for_block
+           #                        COME_FROM
+           #                        POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
+           #                        COME_FROM_LOOP
+
+           # async_for_stmt37   ::= setup_loop expr
+           #                        GET_AITER
+           #                        SETUP_EXCEPT GET_ANEXT
+           #                        LOAD_CONST YIELD_FROM
+           #                        store
+           #                        POP_BLOCK JUMP_LOOP COME_FROM_EXCEPT DUP_TOP
+           #                        LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+           #                        END_FINALLY for_block COME_FROM
+           #                        POP_TOP POP_TOP POP_TOP POP_EXCEPT
+           #                        POP_TOP POP_BLOCK
+           #                        COME_FROM_LOOP
+
+           # async_forelse_stmt ::= setup_loop expr
+           #                        GET_AITER
+           #                        SETUP_EXCEPT GET_ANEXT LOAD_CONST
+           #                        YIELD_FROM
+           #                        store
+           #                        POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+           #                        LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+           #                        END_FINALLY COME_FROM
+           #                        for_block
+           #                        COME_FROM
+           #                        POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
+           #                        else_suite COME_FROM_LOOP
+
+           for                ::= setup_loop expr get_for_iter store for_block POP_BLOCK
+           for                ::= setup_loop expr get_for_iter store for_block POP_BLOCK NOP
+
+           for_block          ::= c_stmts_opt COME_FROM_LOOP JUMP_LOOP
+           forelsestmt        ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suite
+           forelselaststmt    ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec
+           forelselaststmtc   ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec
+
+           try_except         ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                  except_handler opt_come_from_except
+
+           tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                  LOAD_CONST COME_FROM_FINALLY suite_stmts_opt
+                                  END_FINALLY
+           tryfinally36       ::= SETUP_FINALLY returns
+                                  COME_FROM_FINALLY suite_stmts_opt END_FINALLY
+           tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                      LOAD_CONST COME_FROM_FINALLY
+        """
+        )
+
+    # def custom_classfunc_rule(self, opname, token, customize, next_token):
+    #     """
+    #     call ::= expr {expr}^n CALL_FUNCTION_n
+    #     call ::= expr {expr}^n CALL_FUNCTION_VAR_n
+    #     call ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n
+    #     call ::= expr {expr}^n CALL_FUNCTION_KW_n
+
+    #     classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc {expr}^n-1 CALL_FUNCTION_n
+    #     """
+    #     args_pos, args_kw = self.get_pos_kw(token)
+
+    #     # Additional exprs for * and ** args:
+    #     #  0 if neither
+    #     #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
+    #     #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
+    #     # Yes, this computation based on instruction name is a little bit hoaky.
+    #     nak = (len(opname) - len("CALL_FUNCTION")) // 3
+    #     uniq_param = args_kw + args_pos
+    #     if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
+    #         rule = (
+    #             "async_call ::= expr "
+    #             + ("expr " * args_pos)
+    #             + ("kwarg " * args_kw)
+    #             + "expr " * nak
+    #             + token.kind
+    #             + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
+    #         )
+    #         self.add_unique_rule(rule, token.kind, uniq_param, customize)
+    #         self.add_unique_rule(
+    #             "expr ::= async_call", token.kind, uniq_param, customize
+    #         )
+
+    #     if opname.startswith("CALL_FUNCTION_VAR"):
+    #         token.kind = self.call_fn_name(token)
+    #         if opname.endswith("KW"):
+    #             kw = "expr "
+    #         else:
+    #             kw = ""
+    #         rule = (
+    #             "call ::= expr expr "
+    #             + ("expr " * args_pos)
+    #             + ("kwarg " * args_kw)
+    #             + kw
+    #             + token.kind
+    #         )
+
+    #         # Note: semantic actions make use of the fact of whether "args_pos"
+    #         # zero or not in creating a template rule.
+    #         self.add_unique_rule(rule, token.kind, args_pos, customize)
+    #     else:
+    #         token.kind = self.call_fn_name(token)
+    #         uniq_param = args_kw + args_pos
+
+    #         # Note: 3.5+ have subclassed this method; so we don't handle
+    #         # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here.
+    #         rule = (
+    #             "call ::= expr "
+    #             + ("expr " * args_pos)
+    #             + ("kwarg " * args_kw)
+    #             + "expr " * nak
+    #             + token.kind
+    #         )
+
+    #         self.add_unique_rule(rule, token.kind, uniq_param, customize)
+
+    #         if "LOAD_BUILD_CLASS" in self.seen_ops:
+    #             if (
+    #                 next_token == "CALL_FUNCTION"
+    #                 and next_token.attr == 1
+    #                 and args_pos > 1
+    #             ):
+    #                 rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % (
+    #                     ("expr " * (args_pos - 1)),
+    #                     opname,
+    #                     args_pos,
+    #                 )
+    #                 self.add_unique_rule(rule, token.kind, uniq_param, customize)
+
+    def customize_grammar_rules_full38(self, tokens, customize):
+
+        self.customize_grammar_rules_lambda38(tokens, customize)
+        self.customize_reduce_checks_full38(tokens, customize)
+        self.remove_rules_38()
+
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "CONTINUE",
+                "DELETE",
+                "FORMAT",
+                "GET",
+                "JUMP",
+                "LOAD",
+                "LOOKUP",
+                "MAKE",
+                "RETURN",
+                "RAISE",
+                "SETUP",
+                "UNPACK",
+                "WITH",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get added
+        # unconditionally and the rules are constant. So they need to be done
+        # only once and if we see the opcode a second we don't have to consider
+        # adding more rules.
+        #
+        custom_ops_processed = set()
+
+        # A set of instruction operation names that exist in the token stream.
+        # We use this to customize the grammar that we create.
+        # 2.6-compatible set comprehensions
+
+        # The initial initialization is done in lambea_expr.py
+        self.seen_ops = frozenset([t.kind for t in tokens])
+        self.seen_op_basenames = frozenset(
+            [opname[: opname.rfind("_")] for opname in self.seen_ops]
+        )
+
+        # Loop over instructions adding custom grammar rules based on
+        # a specific instruction seen.
+
+        if "PyPy" in customize:
+            self.addRule(
+                """
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy       ::= expr expr expr store store store
+              assign2_pypy       ::= expr expr store store
+              """,
+                nop_func,
+            )
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # The order of opname listed is roughly sorted below
+
+            if opname == "LOAD_ASSERT" and "PyPy" in customize:
+                rules_str = """
+                stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "BEFORE_ASYNC_WITH":
+                rules_str = """
+                   stmt            ::= async_with_stmt
+                   stmt            ::= async_with_as_stmt
+                   c_stmt          ::= c_async_with_stmt
+                """
+
+                if self.version < (3, 8):
+                    rules_str += """
+                      stmt                 ::= async_with_stmt SETUP_ASYNC_WITH
+                      c_stmt               ::= c_async_with_stmt SETUP_ASYNC_WITH
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              c_suite_stmts_opt
+                                              POP_BLOCK LOAD_CONST
+                                              async_with_post
+                      async_with_as_stmt   ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                    """
+                else:
+                    rules_str += """
+                      async_with_pre       ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
+                      async_with_post      ::= BEGIN_FINALLY COME_FROM_ASYNC_WITH
+                                               WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               WITH_CLEANUP_FINISH END_FINALLY
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt   ::= async_with_stmt
+                      async_with_stmt     ::= expr
+                                              async_with_pre
+                                              POP_TOP
+                                              c_suite_stmts
+                                              POP_BLOCK
+                                              BEGIN_FINALLY
+                                              WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                              WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
+                                              POP_BLOCK
+                                              BEGIN_FINALLY
+                                              COME_FROM_ASYNC_WITH
+                                              WITH_AWAITABLE
+                                              LOAD_CONST
+                                              YEILD_FROM
+                                              WITH_CLEANUP_FINISH
+                                              END_FINALLY
+
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                    """
+                self.addRule(rules_str, nop_func)
+
+            elif opname == "BUILD_STRING_2":
+                self.addRule(
+                    """
+                      expr                  ::= formatted_value_debug
+                      formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
+                      formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "BUILD_STRING_3":
+                self.addRule(
+                    """
+                      expr                  ::= formatted_value_debug
+                      formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3
+                      formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname in frozenset(
+                (
+                    "CALL_FUNCTION",
+                    "CALL_FUNCTION_EX_KW",
+                    "CALL_FUNCTION_VAR_KW",
+                    "CALL_FUNCTION_VAR",
+                    "CALL_FUNCTION_VAR_KW",
+                )
+            ) or opname.startswith("CALL_FUNCTION_KW"):
+
+                if opname == "CALL_FUNCTION" and token.attr == 1:
+                    rule = """
+                    classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
+                    classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
+                    """
+                    self.addRule(rule, nop_func)
+
+                # self.custom_classfunc_rule(opname, token, customize, tokens[i + 1])
+                # Note: don't add to custom_ops_processed.
+
+            elif opname_base == "CALL_METHOD":
+                # PyPy and Python 3.7+ only - DRY with parse2
+
+                if opname == "CALL_METHOD_KW":
+                    args_kw = token.attr
+                    rules_str = """
+                         expr ::= call_kw_pypy37
+                         pypy_kw_keys ::= LOAD_CONST
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+                    rule = (
+                        "call_kw_pypy37 ::= expr "
+                        + ("expr " * args_kw)
+                        + " pypy_kw_keys "
+                        + opname
+                    )
+                else:
+                    args_pos, args_kw = self.get_pos_kw(token)
+                    # number of apply equiv arguments:
+                    nak = (len(opname_base) - len("CALL_METHOD")) // 3
+                    rule = (
+                        "call ::= expr "
+                        + ("expr " * args_pos)
+                        + ("kwarg " * args_kw)
+                        + "expr " * nak
+                        + opname
+                    )
+
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "CONTINUE":
+                self.addRule("continue ::= CONTINUE", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "CONTINUE_LOOP":
+                self.addRule("continue ::= CONTINUE_LOOP", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_ATTR":
+                self.addRule("delete ::= expr DELETE_ATTR", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_DEREF":
+                self.addRule(
+                    """
+                   stmt           ::= del_deref_stmt
+                   del_deref_stmt ::= DELETE_DEREF
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_SUBSCR":
+                self.addRule(
+                    """
+                    delete ::= delete_subscript
+                    delete_subscript ::= expr expr DELETE_SUBSCR
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "FORMAT_VALUE_ATTR":
+                self.addRule(
+                    """
+                      expr                  ::= formatted_value_debug
+                      formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
+                                                expr FORMAT_VALUE_ATTR
+                      formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
+                      formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_AITER":
+                self.addRule(
+                    """
+                    async_for          ::= GET_AITER _come_froms
+                                           SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
+
+                    async_for_stmt38   ::= expr async_for
+                                           store for_block
+                                           COME_FROM_FINALLY
+                                           END_ASYNC_FOR
+
+                    # FIXME: COME_FROMs after the else_suite or
+                    # END_ASYNC_FOR distinguish which of for / forelse
+                    # is used. Add COME_FROMs and check of add up
+                    # control-flow detection phase.
+                    # async_forelse_stmt38 ::= expr async_for store
+                    # for_block COME_FROM_FINALLY END_ASYNC_FOR
+                    # else_suite
+
+                    async_forelse_stmt38 ::= expr async_for
+                                             store for_block
+                                             COME_FROM_FINALLY
+                                             END_ASYNC_FOR
+                                             else_suite
+                                             POP_TOP COME_FROM
+
+                    stmt                 ::= async_for_stmt38
+                    stmt                 ::= async_forelse_stmt38
+                    stmt                 ::= generator_exp_async
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "GET_ANEXT":
+                self.addRule(
+                    """
+                    stmt ::= genexpr_func_async
+                    stmt ::= BUILD_SET_0 genexpr_func_async
+                             RETURN_VALUE
+                             _come_froms
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "JUMP_IF_NOT_DEBUG":
+                self.addRule(
+                    """
+                    stmt        ::= assert_pypy
+                    stmt        ::= assert2_pypy", nop_func)
+                    assert_pypy ::=  JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM,
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLASSDEREF":
+                # Python 3.4+
+                self.addRule("expr ::= LOAD_CLASSDEREF", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLASSNAME":
+                self.addRule("expr ::= LOAD_CLASSNAME", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "RAISE_VARARGS_0":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt0
+                    last_stmt  ::= raise_stmt0
+                    raise_stmt0 ::= RAISE_VARARGS_0
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_1":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt1
+                    last_stmt  ::= raise_stmt1
+                    raise_stmt1 ::= expr RAISE_VARARGS_1
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_2":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt2
+                    last_stmt  ::= raise_stmt2
+                    raise_stmt2 ::= expr expr RAISE_VARARGS_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "RETURN_VALUE_LAMBDA":
+                self.addRule(
+                    """
+                    return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "SETUP_EXCEPT":
+                self.addRule(
+                    """
+                    try_except     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler opt_come_from_except
+                    c_try_except   ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler opt_come_from_except
+                    stmt           ::= tryelsestmt3
+                    tryelsestmt3   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler COME_FROM else_suite
+                                       opt_come_from_except
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_from_except_clauses
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_froms
+
+                    c_stmt         ::= c_tryelsestmt
+                    c_tryelsestmt  ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler
+                                       come_any_froms else_suitec
+                                       come_from_except_clauses
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "WITH_CLEANUP_START":
+                rules_str = """
+                  stmt        ::= with_null
+                  with_null   ::= with_suffix
+                  with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                self.addRule(rules_str, nop_func)
+
+            # FIXME: reconcile with same code in lambda_custom.py
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                  stmt        ::= with
+                  stmt        ::= with_as_pass
+                  stmt        ::= with_as
+
+                  c_stmt      ::= c_with
+
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr SETUP_WITH POP_TOP
+                                  suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with_as     ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with_as     ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                                  with_suffix
+
+                  with_as     ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with_as_pass ::= expr
+                                  SETUP_WITH store pass
+                                  POP_BLOCK BEGIN_FINALLY COME_FROM_WITH
+                                  with_suffix
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                    with      ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   with_suffix
+                    """
+                else:
+                    rules_str += """
+                     # A return at the end of a withas stmt can be this.
+                     # FIXME: should this be a different kind of return?
+                     return      ::= return_expr POP_BLOCK
+                                     ROT_TWO
+                                     BEGIN_FINALLY
+                                     WITH_CLEANUP_START
+                                     WITH_CLEANUP_FINISH
+                                     POP_FINALLY
+                                     RETURN_VALUE
+
+                      with       ::= expr
+                                     SETUP_WITH POP_TOP suite_stmts_opt
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                     with_suffix
+
+
+                      with_as ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+
+                      with_as ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK BEGIN_FINALLY COME_FROM_WITH
+                                     with_suffix
+
+                      # with_as ::= expr SETUP_WITH store suite_stmts
+                      #                COME_FROM expr COME_FROM POP_BLOCK ROT_TWO
+                      #                BEGIN_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                      #                POP_FINALLY RETURN_VALUE COME_FROM_WITH
+                      #                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+                      with         ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                       BEGIN_FINALLY COME_FROM_WITH
+                                       with_suffix
+                    """
+                self.addRule(rules_str, nop_func)
+            pass
+
+        return
+
+    def customize_reduce_checks_full38(self, tokens, customize):
+        """
+        Extra tests when a reduction is made in the full grammar.
+
+        Reductions here are extended from those used in the lambda grammar
+        """
+        self.remove_rules_38()
+
+        self.check_reduce["and"] = "AST"
+        self.check_reduce["and_cond"] = "AST"
+        self.check_reduce["and_not"] = "AST"
+        self.check_reduce["annotate_tuple"] = "tokens"
+        self.check_reduce["aug_assign1"] = "AST"
+        self.check_reduce["aug_assign2"] = "AST"
+        self.check_reduce["c_forelsestmt38"] = "AST"
+        self.check_reduce["c_try_except"] = "AST"
+        self.check_reduce["c_tryelsestmt"] = "AST"
+        self.check_reduce["if_and_stmt"] = "AST"
+        self.check_reduce["if_and_elsestmtc"] = "AST"
+        self.check_reduce["if_not_stmtc"] = "AST"
+        self.check_reduce["ifelsestmt"] = "AST"
+        self.check_reduce["ifelsestmtc"] = "AST"
+        self.check_reduce["iflaststmt"] = "AST"
+        self.check_reduce["iflaststmtc"] = "AST"
+        self.check_reduce["ifstmt"] = "AST"
+        self.check_reduce["ifstmtc"] = "AST"
+        self.check_reduce["ifstmts_jump"] = "AST"
+        self.check_reduce["ifstmts_jumpc"] = "AST"
+        self.check_reduce["import_as37"] = "tokens"
+        self.check_reduce["import_from37"] = "AST"
+        self.check_reduce["import_from_as37"] = "tokens"
+        self.check_reduce["lastc_stmt"] = "tokens"
+        self.check_reduce["list_if_not"] = "AST"
+        self.check_reduce["while1elsestmt"] = "tokens"
+        self.check_reduce["while1stmt"] = "tokens"
+        self.check_reduce["whilestmt"] = "tokens"
+        self.check_reduce["not_or"] = "AST"
+        self.check_reduce["or"] = "AST"
+        self.check_reduce["or_cond"] = "tokens"
+        self.check_reduce["testtrue"] = "tokens"
+        self.check_reduce["testfalsec"] = "tokens"
+
+        self.check_reduce["break"] = "tokens"
+        self.check_reduce["forelselaststmt38"] = "AST"
+        self.check_reduce["forelselaststmtc38"] = "AST"
+        self.check_reduce["for38"] = "tokens"
+        self.check_reduce["ifstmt"] = "AST"
+        self.check_reduce["joined_str"] = "AST"
+        self.check_reduce["pop_return"] = "tokens"
+        self.check_reduce["whileTruestmt38"] = "AST"
+        self.check_reduce["whilestmt38"] = "tokens"
+        self.check_reduce["try_elsestmtl38"] = "AST"
+
+        self.reduce_check_table["break"] = break_invalid
+        self.reduce_check_table["if_not_stmtc"] = if_not_stmtc_invalid
+        self.reduce_check_table["for38"] = for38_invalid
+        self.reduce_check_table["c_forelsestmt38"] = forelse38_invalid
+        self.reduce_check_table["forelselaststmt38"] = forelse38_invalid
+        self.reduce_check_table["forelselaststmtc38"] = forelse38_invalid
+        # self.reduce_check_table["joined_str"] = joined_str_check.joined_str_invalid
+        self.reduce_check_table["or"] = or_cond_check_invalid
+        self.reduce_check_table["pop_return"] = pop_return_check
+        self.reduce_check_table["whilestmt38"] = whilestmt38_check
+        self.reduce_check_table["whileTruestmt38"] = whileTruestmt38_check
+
+        # Use update we don't destroy entries from lambda.
+        self.reduce_check_table.update(
+            {
+                # "ifelsestmt": ifelsestmt_ok,
+                "ifstmt": ifstmt,
+                # "import_from37": import_from37_ok,
+            }
+        )
+
+        self.check_reduce["ifelsestmt"] = "AST"
+        self.check_reduce["ifelsestmtc"] = "AST"
+        self.check_reduce["ifstmt"] = "AST"
+        # self.check_reduce["import_from37"] = "AST"
+
+    def customize_grammar_rules38(self, tokens, customize):
+        Python37BaseParser.customize_grammar_rules37(self, tokens, customize)
+        self.customize_reduce_checks_lambda38()
+        self.customize_reduce_checks_full38(tokens, customize)
+
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "CONTINUE",
+                "DELETE",
+                "FORMAT",
+                "GET",
+                "JUMP",
+                "LOAD",
+                "LOOKUP",
+                "MAKE",
+                "RETURN",
+                "RAISE",
+                "SETUP",
+                "UNPACK",
+                "WITH",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get added
+        # unconditionally and the rules are constant. So they need to be done
+        # only once and if we see the opcode a second we don't have to consider
+        # adding more rules.
+        #
+        custom_ops_processed = set()
+
+        # A set of instruction operation names that exist in the token stream.
+        # We use this to customize the grammar that we create.
+        # 2.6-compatible set comprehensions
+
+        # The initial initialization is done in lambda_expr.py
+        self.seen_ops = frozenset([t.kind for t in tokens])
+        self.seen_op_basenames = frozenset(
+            [opname[: opname.rfind("_")] for opname in self.seen_ops]
+        )
+
+        # Loop over instructions adding custom grammar rules based on
+        # a specific instruction seen.
+
+        if "PyPy" in customize:
+            self.addRule(
+                """
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy       ::= expr expr expr store store store
+              assign2_pypy       ::= expr expr store store
+              """,
+                nop_func,
+            )
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # The order of opname listed is roughly sorted below
+
+            if opname == "LOAD_ASSERT" and "PyPy" in customize:
+                rules_str = """
+                stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "BEFORE_ASYNC_WITH":
+                rules_str = """
+                   stmt            ::= async_with_stmt
+                   stmt            ::= async_with_as_stmt
+                   c_stmt          ::= c_async_with_stmt
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                      async_with_pre       ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               SETUP_ASYNC_WITH
+                      stmt                 ::= async_with_stmt SETUP_ASYNC_WITH
+                      c_stmt               ::= c_async_with_stmt SETUP_ASYNC_WITH
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              c_suite_stmts_opt
+                                              POP_BLOCK LOAD_CONST
+                                              async_with_post
+                      async_with_as_stmt   ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                    """
+                else:
+                    rules_str += """
+                      async_with_pre       ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
+                      async_with_post      ::= BEGIN_FINALLY COME_FROM_ASYNC_WITH
+                                               WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               WITH_CLEANUP_FINISH END_FINALLY
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt   ::= async_with_stmt
+                      async_with_stmt     ::= expr
+                                              async_with_pre
+                                              POP_TOP
+                                              c_suite_stmts
+                                              POP_BLOCK
+                                              BEGIN_FINALLY
+                                              WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                              WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
+                                              POP_BLOCK
+                                              BEGIN_FINALLY
+                                              COME_FROM_ASYNC_WITH
+                                              WITH_AWAITABLE
+                                              LOAD_CONST
+                                              YEILD_FROM
+                                              WITH_CLEANUP_FINISH
+                                              END_FINALLY
+
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                    """
+                self.addRule(rules_str, nop_func)
+
+            elif opname in frozenset(
+                (
+                    "CALL_FUNCTION",
+                    "CALL_FUNCTION_EX_KW",
+                    "CALL_FUNCTION_VAR_KW",
+                    "CALL_FUNCTION_VAR",
+                    "CALL_FUNCTION_VAR_KW",
+                )
+            ) or opname.startswith("CALL_FUNCTION_KW"):
+
+                if opname == "CALL_FUNCTION" and token.attr == 1:
+                    rule = """
+                    classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
+                    classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
+                    """
+                    self.addRule(rule, nop_func)
+
+                # self.custom_classfunc_rule(opname, token, customize, tokens[i + 1])
+                # Note: don't add to custom_ops_processed.
+
+            elif opname_base == "CALL_METHOD":
+                # PyPy and Python 3.7+ only - DRY with parse2
+
+                if opname == "CALL_METHOD_KW":
+                    args_kw = token.attr
+                    rules_str = """
+                         expr ::= call_kw_pypy37
+                         pypy_kw_keys ::= LOAD_CONST
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+                    rule = (
+                        "call_kw_pypy37 ::= expr "
+                        + ("expr " * args_kw)
+                        + " pypy_kw_keys "
+                        + opname
+                    )
+                else:
+                    args_pos, args_kw = self.get_pos_kw(token)
+                    # number of apply equiv arguments:
+                    nak = (len(opname_base) - len("CALL_METHOD")) // 3
+                    rule = (
+                        "call ::= expr "
+                        + ("expr " * args_pos)
+                        + ("kwarg " * args_kw)
+                        + "expr " * nak
+                        + opname
+                    )
+
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "CONTINUE":
+                self.addRule("continue ::= CONTINUE", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "CONTINUE_LOOP":
+                self.addRule("continue ::= CONTINUE_LOOP", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_ATTR":
+                self.addRule("delete ::= expr DELETE_ATTR", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_DEREF":
+                self.addRule(
+                    """
+                   stmt           ::= del_deref_stmt
+                   del_deref_stmt ::= DELETE_DEREF
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_SUBSCR":
+                self.addRule(
+                    """
+                    delete ::= delete_subscript
+                    delete_subscript ::= expr expr DELETE_SUBSCR
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_AITER":
+                self.addRule(
+                    """
+                    stmt ::= generator_exp_async
+                    stmt ::= genexpr_func_async
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "GET_ANEXT":
+                self.addRule(
+                    """
+                    stmt ::= BUILD_SET_0 genexpr_func_async
+                             RETURN_VALUE
+                             bb_doms_end_opt
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "JUMP_IF_NOT_DEBUG":
+                self.addRule(
+                    """
+                    stmt        ::= assert_pypy
+                    stmt        ::= assert2_pypy", nop_func)
+                    assert_pypy ::=  JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM,
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLASSDEREF":
+                # Python 3.4+
+                self.addRule("expr ::= LOAD_CLASSDEREF", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLASSNAME":
+                self.addRule("expr ::= LOAD_CLASSNAME", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "RAISE_VARARGS_0":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt0
+                    last_stmt  ::= raise_stmt0
+                    raise_stmt0 ::= RAISE_VARARGS_0
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_1":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt1
+                    last_stmt  ::= raise_stmt1
+                    raise_stmt1 ::= expr RAISE_VARARGS_1
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_2":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt2
+                    last_stmt  ::= raise_stmt2
+                    raise_stmt2 ::= expr expr RAISE_VARARGS_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "RETURN_VALUE_LAMBDA":
+                self.addRule(
+                    """
+                    return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "SETUP_EXCEPT":
+                self.addRule(
+                    """
+                    try_except     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler opt_come_from_except
+                    c_try_except   ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler opt_come_from_except
+                    stmt           ::= tryelsestmt3
+                    tryelsestmt3   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler COME_FROM else_suite
+                                       opt_come_from_except
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_from_except_clauses
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_froms
+
+                    c_stmt         ::= c_tryelsestmt
+                    c_tryelsestmt  ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler
+                                       come_any_froms else_suitec
+                                       come_from_except_clauses
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "WITH_CLEANUP_START":
+                rules_str = """
+                  stmt        ::= with_null
+                  with_null   ::= with_suffix
+                  with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                self.addRule(rules_str, nop_func)
+
+            # FIXME: reconcile with same code in lambda_custom.py
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                  stmt        ::= with
+                  stmt        ::= with_as
+                  c_stmt      ::= c_with
+
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr SETUP_WITH POP_TOP
+                                  suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+
+                  with_as  ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                  with_as  ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                  with_as  ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                    with      ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   with_suffix
+                    """
+                else:
+                    rules_str += """
+                     # A return at the end of a withas stmt can be this.
+                     # FIXME: should this be a different kind of return?
+                     return      ::= return_expr POP_BLOCK
+                                     ROT_TWO
+                                     BEGIN_FINALLY
+                                     WITH_CLEANUP_START
+                                     WITH_CLEANUP_FINISH
+                                     POP_FINALLY
+                                     RETURN_VALUE
+
+                      with       ::= expr
+                                     SETUP_WITH POP_TOP suite_stmts_opt
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                     with_suffix
+
+
+                      with_as ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+
+                      with_as ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK BEGIN_FINALLY COME_FROM_WITH
+                                     with_suffix
+
+                      # with_as ::= expr SETUP_WITH store suite_stmts
+                      #                COME_FROM expr COME_FROM POP_BLOCK ROT_TWO
+                      #                BEGIN_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                      #                POP_FINALLY RETURN_VALUE COME_FROM_WITH
+                      #                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+                      with         ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                       BEGIN_FINALLY COME_FROM_WITH
+                                       with_suffix
+                    """
+                self.addRule(rules_str, nop_func)
+            pass
+
+        return

python/py/Lib/site-packages/decompyle3/parsers/p38/heads.py

@@ -0,0 +1,56 @@
+"""
+All of the specific kinds of canned parsers for Python 3.8
+
+These are derived from "compile-modes" but we have others that
+can be used to parse common part of a larger grammar.
+
+For example:
+* a basic-block expression (no branching)
+* an unadorned expression (no POP_TOP needed afterwards)
+* A non-compound statement
+"""
+from decompyle3.parsers.p38.full import Python38Parser
+from decompyle3.parsers.p38.lambda_expr import Python38LambdaParser
+from decompyle3.parsers.parse_heads import (
+    PythonParserEval,
+    PythonParserExec,
+    PythonParserExpr,
+    PythonParserLambda,
+    PythonParserSingle,
+    # FIXME: add
+    # PythonParserSimpleStmt
+    # PythonParserStmt
+)
+
+# Make sure to list Python38... classes first so we prefer to inherit methods from that first.
+# In particular methods like reduce_is_invalid() need to come from there rather than
+# a more generic place.
+
+
+class Python38ParserEval(Python38LambdaParser, PythonParserEval):
+    def __init__(self, debug_parser):
+        PythonParserEval.__init__(self, debug_parser)
+
+
+class Python38ParserExec(Python38Parser, PythonParserExec):
+    def __init__(self, debug_parser):
+        PythonParserExec.__init__(self, debug_parser)
+
+
+class Python38ParserExpr(Python38Parser, PythonParserExpr):
+    def __init__(self, debug_parser):
+        PythonParserExpr.__init__(self, debug_parser)
+
+
+# Understand: Python38LambdaParser has to come before PythonParserLambda or we get a
+# MRO failure
+class Python38ParserLambda(Python38LambdaParser, PythonParserLambda):
+    def __init__(self, debug_parser):
+        PythonParserLambda.__init__(self, debug_parser)
+
+
+# These classes are here just to get parser doc-strings for the
+# various classes inherited properly and start_symbols set properly.
+class Python38ParserSingle(Python38Parser, PythonParserSingle):
+    def __init__(self, debug_parser):
+        PythonParserSingle.__init__(self, debug_parser)

python/py/Lib/site-packages/decompyle3/parsers/p38/lambda_custom.py

@@ -0,0 +1,775 @@
+#  Copyright (c) 2020-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Grammar Customization rules for Python 3.8's Lambda expression grammar.
+"""
+
+from decompyle3.parsers.p37.base import Python37BaseParser
+from decompyle3.parsers.p38.base import Python38BaseParser
+from decompyle3.parsers.parse_heads import nop_func
+
+
+class Python38LambdaCustom(Python38BaseParser):
+    def __init__(self):
+        self.new_rules = set()
+
+        # Special opcodes we see that trigger adding new grammar rules.
+        self.seen_ops = frozenset()
+
+        # Special opcodes we see that trigger adding new grammar rules.
+        self.seen_ops_basenames = frozenset()
+
+        # Customized grammar rules
+        self.customized = {}
+
+    def customize_grammar_rules_lambda38(self, tokens, customize):
+        Python38BaseParser.customize_grammar_rules38(self, tokens, customize)
+        self.check_reduce["call_kw"] = "AST"
+
+        # For a rough break out on the first word. This may
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "DICT",
+                "GET",
+                "FORMAT",
+                "LIST",
+                "LOAD",
+                "MAKE",
+                "SETUP",
+                "UNPACK",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get
+        # added unconditionally and the rules are constant. So they need to be
+        # done only once, and if we see the opcode a second time, we don't have
+        # to consider adding more rules.
+        #
+        custom_ops_processed = frozenset()
+
+        # A set of instruction operation names that exist in the token stream.
+        # We use this to customize the grammar that we create.
+        # 2.6-compatible set comprehensions
+        self.seen_ops = frozenset([t.kind for t in tokens])
+        self.seen_op_basenames = frozenset(
+            [opname[: opname.rfind("_")] for opname in self.seen_ops]
+        )
+
+        custom_ops_processed = {"DICT_MERGE"}
+
+        # Loop over instructions adding custom grammar rules based on
+        # a specific instruction seen.
+
+        if "PyPy" in customize:
+            self.addRule(
+                """
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy       ::= expr expr expr store store store
+              assign2_pypy       ::= expr expr store store
+              """,
+                nop_func,
+            )
+
+        n = len(tokens)
+
+        # Determine if we have an iteration CALL_FUNCTION_1.
+        has_get_iter_call_function1 = False
+        for i, token in enumerate(tokens):
+            if (
+                token == "GET_ITER"
+                and i < n - 2
+                and self.call_fn_name(tokens[i + 1]) == "CALL_FUNCTION_1"
+            ):
+                has_get_iter_call_function1 = True
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instructions in "if"/"elif".
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instructions in "if"/"elif".
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            if opname == "BEFORE_ASYNC_WITH":
+                rules_str = """
+                  async_with_post    ::= COME_FROM_ASYNC_WITH
+                                         WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                         WITH_CLEANUP_FINISH END_FINALLY
+
+                  stmt               ::= async_with_as_stmt
+                  async_with_as_stmt ::= expr
+                                         async_with_pre
+                                         store
+                                         suite_stmts_opt
+                                         POP_BLOCK LOAD_CONST
+                                         async_with_post
+
+                  async_with_stmt     ::= expr
+                                          async_with_pre
+                                          POP_TOP
+                                          c_suite_stmts
+                                          POP_BLOCK
+                                          BEGIN_FINALLY
+                                          WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                          WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
+                                          POP_BLOCK
+                                          BEGIN_FINALLY
+                                          COME_FROM_ASYNC_WITH
+                                          WITH_CLEANUP_START
+                                          GET_AWAITABLE
+                                          LOAD_CONST
+                                          YIELD_FROM
+                                          WITH_CLEANUP_FINISH
+                                          END_FINALLY
+
+
+                 async_with_stmt     ::= expr
+                                         async_with_pre
+                                         POP_TOP
+                                         suite_stmts_opt
+                                         POP_BLOCK LOAD_CONST
+                                         async_with_post
+                 async_with_stmt     ::= expr
+                                         async_with_pre
+                                         POP_TOP
+                                         suite_stmts_opt
+                                         async_with_post
+                """
+                self.addRule(rules_str, nop_func)
+
+            elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
+                if opname == "BUILD_CONST_DICT":
+                    rule = f"""
+                            add_consts          ::= ADD_VALUE*
+                            const_list          ::= COLLECTION_START add_consts {opname}
+                            dict                ::= const_list
+                            expr                ::= dict
+                        """
+                else:
+                    rule = f"""
+                            add_consts          ::= ADD_VALUE*
+                            const_list          ::= COLLECTION_START add_consts {opname}
+                            expr                ::= const_list
+                        """
+                self.addRule(rule, nop_func)
+            elif opname_base in (
+                "BUILD_LIST",
+                "BUILD_SET",
+                "BUILD_SET_UNPACK",
+                "BUILD_TUPLE",
+                "BUILD_TUPLE_UNPACK",
+            ):
+                v = token.attr
+
+                is_LOAD_CLOSURE = False
+                if opname_base == "BUILD_TUPLE":
+                    # If is part of a "load_closure", then it is not part of a
+                    # "list".
+                    is_LOAD_CLOSURE = True
+                    for j in range(v):
+                        if tokens[i - j - 1].kind != "LOAD_CLOSURE":
+                            is_LOAD_CLOSURE = False
+                            break
+                    if is_LOAD_CLOSURE:
+                        rule_str = "load_closure ::= %s%s" % (
+                            ("LOAD_CLOSURE " * v),
+                            opname,
+                        )
+                        self.add_unique_doc_rules(rule_str, customize)
+
+                elif opname_base == "BUILD_LIST":
+                    v = token.attr
+                    if v == 0:
+                        rule_str = """
+                           list        ::= BUILD_LIST_0
+                           list_unpack ::= BUILD_LIST_0 expr LIST_EXTEND
+                           list        ::= list_unpack
+                        """
+                        self.add_unique_doc_rules(rule_str, customize)
+
+                elif opname == "BUILD_TUPLE_UNPACK_WITH_CALL":
+                    # FIXME: should this be parameterized by EX value?
+                    self.addRule(
+                        """expr        ::= call_ex_kw3
+                           call_ex_kw3 ::= expr
+                                           build_tuple_unpack_with_call
+                                           expr
+                                           CALL_FUNCTION_EX_KW
+                        """,
+                        nop_func,
+                    )
+
+                if not is_LOAD_CLOSURE or v == 0:
+                    # We do this complicated test to speed up parsing of
+                    # pathologically long literals, especially those over 1024.
+                    build_count = token.attr
+                    thousands = build_count // 1024
+                    thirty32s = (build_count // 32) % 32
+                    if thirty32s > 0:
+                        rule = "expr32 ::=%s" % (" expr" * 32)
+                        self.add_unique_rule(rule, opname_base, build_count, customize)
+                        pass
+                    if thousands > 0:
+                        self.add_unique_rule(
+                            "expr1024 ::=%s" % (" expr32" * 32),
+                            opname_base,
+                            build_count,
+                            customize,
+                        )
+                        pass
+                    collection = opname_base[opname_base.find("_") + 1 :].lower()
+                    rule = (
+                        ("%s ::= " % collection)
+                        + "expr1024 " * thousands
+                        + "expr32 " * thirty32s
+                        + "expr " * (build_count % 32)
+                        + opname
+                    )
+                    self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
+                    continue
+                continue
+
+            elif opname.startswith("BUILD_STRING"):
+                v = token.attr
+                rules_str = """
+                    expr                 ::= joined_str
+                    joined_str           ::= %sBUILD_STRING_%d
+                """ % (
+                    "expr " * v,
+                    v,
+                )
+                self.add_unique_doc_rules(rules_str, customize)
+                if "FORMAT_VALUE_ATTR" in self.seen_ops:
+                    rules_str = """
+                      formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
+                      expr                 ::= formatted_value_attr
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+            elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
+                v = token.attr
+                rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+            elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
+                v = token.attr
+                rule = (
+                    "build_tuple_unpack_with_call ::= "
+                    + "expr1024 " * int(v // 1024)
+                    + "expr32 " * int((v // 32) % 32)
+                    + "expr " * (v % 32)
+                    + opname
+                )
+                self.addRule(rule, nop_func)
+                rule = "starred ::= %s %s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+
+            elif opname == "FORMAT_VALUE":
+                rules_str = """
+                    expr              ::= formatted_value1
+                    formatted_value1  ::= expr FORMAT_VALUE
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+            elif opname == "FORMAT_VALUE_ATTR":
+                rules_str = """
+                expr              ::= formatted_value2
+                formatted_value2  ::= expr expr FORMAT_VALUE_ATTR
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "GET_AITER":
+                self.add_unique_doc_rules("get_aiter ::= expr GET_AITER", customize)
+
+                if {"MAKE_FUNCTION_0", "MAKE_FUNCTION_CLOSURE"} not in self.seen_ops:
+                    self.addRule(
+                        """
+                        expr                ::= dict_comp_async
+                        expr                ::= generator_exp_async
+                        expr                ::= list_comp_async
+
+                        dict_comp_async     ::= LOAD_DICTCOMP
+                                                LOAD_STR
+                                                MAKE_FUNCTION_0
+                                                get_aiter
+                                                CALL_FUNCTION_1
+
+                        dict_comp_async     ::= BUILD_MAP_0 LOAD_ARG
+                                                dict_comp_async
+
+                        generator_exp_async ::= load_genexpr LOAD_STR MAKE_FUNCTION_0
+                                                get_aiter CALL_FUNCTION_1
+
+                        list_comp_async     ::= LOAD_LISTCOMP LOAD_STR MAKE_FUNCTION_0
+                                                get_aiter CALL_FUNCTION_1
+                                                await
+
+                        list_comp_async     ::= LOAD_CLOSURE
+                                                BUILD_TUPLE_1
+                                                LOAD_LISTCOMP
+                                                LOAD_STR MAKE_FUNCTION_CLOSURE
+                                                get_aiter CALL_FUNCTION_1
+                                                await
+
+                        set_comp_async       ::= LOAD_SETCOMP
+                                                 LOAD_STR
+                                                 MAKE_FUNCTION_0
+                                                 get_aiter
+                                                 CALL_FUNCTION_1
+
+                        set_comp_async       ::= LOAD_CLOSURE
+                                                 BUILD_TUPLE_1
+                                                 LOAD_SETCOMP
+                                                 LOAD_STR MAKE_FUNCTION_CLOSURE
+                                                 get_aiter CALL_FUNCTION_1
+                                                 await
+                       """,
+                        nop_func,
+                    )
+                    custom_ops_processed.add(opname)
+
+                self.addRule(
+                    """
+                    dict_comp_async      ::= BUILD_MAP_0 LOAD_ARG
+                                             dict_comp_async
+
+                    expr                 ::= dict_comp_async
+                    expr                 ::= generator_exp_async
+                    expr                 ::= list_comp_async
+                    expr                 ::= set_comp_async
+
+                    func_async_middle   ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
+                                            DUP_TOP LOAD_GLOBAL COMPARE_OP
+                                            POP_JUMP_IF_TRUE
+                                            END_FINALLY _come_froms
+
+                    # async_iter         ::= block_break SETUP_EXCEPT GET_ANEXT
+                                             LOAD_CONST YIELD_FROM
+
+                    get_aiter            ::= expr GET_AITER
+
+                    list_afor            ::= get_aiter list_afor2
+
+                    list_comp_async      ::= BUILD_LIST_0 LOAD_ARG list_afor2
+                    list_iter            ::= list_afor
+
+
+                    set_afor             ::= get_aiter set_afor2
+                    set_iter             ::= set_afor
+
+                    set_comp_async       ::= BUILD_SET_0 LOAD_ARG
+                                             set_comp_async
+
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_ANEXT":
+                self.addRule(
+                    """
+                    expr                 ::= genexpr_func_async
+                    expr                 ::= BUILD_MAP_0 genexpr_func_async
+                    expr                 ::= list_comp_async
+
+                    dict_comp_async      ::= BUILD_MAP_0 genexpr_func_async
+
+                    async_iter           ::= _come_froms
+                                             SETUP_FINALLY GET_ANEXT LOAD_CONST
+                                             YIELD_FROM POP_BLOCK
+
+                    func_async_prefix    ::= _come_froms SETUP_EXCEPT GET_ANEXT
+                                              LOAD_CONST YIELD_FROM
+
+                    genexpr_func_async   ::= LOAD_ARG async_iter
+                                             store
+                                             comp_iter
+                                             JUMP_LOOP
+                                             COME_FROM_FINALLY
+                                             END_ASYNC_FOR
+
+                    genexpr_func_async   ::= LOAD_ARG func_async_prefix
+                                             store func_async_middle comp_iter
+                                             JUMP_LOOP COME_FROM
+                                             POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
+
+                    list_afor2           ::= async_iter
+                                             store
+                                             list_iter
+                                             JUMP_LOOP
+                                             COME_FROM_FINALLY
+                                             END_ASYNC_FOR
+
+                    list_comp_async      ::= BUILD_LIST_0 LOAD_ARG list_afor2
+
+                    set_afor2            ::= async_iter
+                                             store
+                                             set_iter
+                                             JUMP_LOOP
+                                             COME_FROM_FINALLY
+                                             END_ASYNC_FOR
+
+                    set_afor2            ::= expr_or_arg
+                                             set_iter_async
+
+                    set_comp_async       ::= BUILD_SET_0 set_afor2
+
+                    set_iter_async       ::= async_iter
+                                             store
+                                             set_iter
+                                             JUMP_LOOP
+                                             _come_froms
+                                             END_ASYNC_FOR
+
+                    return_expr_lambda   ::= genexpr_func_async
+                                             LOAD_CONST RETURN_VALUE
+                                             RETURN_VALUE_LAMBDA
+
+                    return_expr_lambda   ::= BUILD_SET_0 genexpr_func_async
+                                             RETURN_VALUE_LAMBDA
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_AWAITABLE":
+                rule_str = """
+                    await      ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
+                    await_expr ::= expr await
+                    expr       ::= await_expr
+                """
+                self.add_unique_doc_rules(rule_str, customize)
+
+            elif opname == "GET_ITER":
+                self.addRule(
+                    """
+                    expr      ::= get_iter
+                    get_iter  ::= expr GET_ITER
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_ASSERT":
+                if "PyPy" in customize:
+                    rules_str = """
+                    stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "LOAD_ATTR":
+                self.addRule(
+                    """
+                  expr      ::= attribute
+                  attribute ::= expr LOAD_ATTR
+                  """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLOSURE":
+                self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
+
+            elif opname == "LOAD_DICTCOMP":
+                if has_get_iter_call_function1:
+                    rule_pat = "dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1"
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    pass
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_GENEXPR":
+                self.addRule("load_genexpr ::= LOAD_GENEXPR", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_LISTCOMP":
+                self.add_unique_rule(
+                    "expr ::= list_comp", opname, token.attr, customize
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_NAME":
+                if (
+                    token.attr == "__annotations__"
+                    and "SETUP_ANNOTATIONS" in self.seen_ops
+                ):
+                    token.kind = "LOAD_ANNOTATION"
+                    self.addRule(
+                        """
+                        stmt       ::= SETUP_ANNOTATIONS
+                        stmt       ::= ann_assign
+                        ann_assign ::= expr LOAD_ANNOTATION LOAD_STR STORE_SUBSCR
+                        """,
+                        nop_func,
+                    )
+                    pass
+            elif opname == "LOAD_SETCOMP":
+                # Should this be generalized and put under MAKE_FUNCTION?
+                if has_get_iter_call_function1:
+                    self.addRule("expr ::= set_comp", nop_func)
+                    rule_pat = "set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1"
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    pass
+                custom_ops_processed.add(opname)
+            elif opname == "LOOKUP_METHOD":
+                # A PyPy speciality - DRY with parse3
+                self.addRule(
+                    """
+                             expr      ::= attribute
+                             attribute ::= expr LOOKUP_METHOD
+                             """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "MAKE_FUNCTION_CLOSURE":
+                if "LOAD_DICTCOMP" in self.seen_ops:
+                    # Is there something general going on here?
+                    rule = """
+                       dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
+                                     MAKE_FUNCTION_CLOSURE expr
+                                     GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+                elif "LOAD_SETCOMP" in self.seen_ops:
+                    rule = """
+                       set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
+                                    MAKE_FUNCTION_CLOSURE expr
+                                    GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+
+            elif opname == "MAKE_FUNCTION_CLOSURE_POS":
+
+                args_pos, args_kw, annotate_args, closure = token.attr
+                stack_count = args_pos + args_kw + annotate_args
+
+                if closure:
+
+                    if args_pos:
+                        # This was seen ion line 447 of Python 3.8
+                        # This is needed for Python 3.8 line 447 of site-packages/nltk/tgrep.py
+                        # line 447:
+                        #    lambda i: lambda n, m=None, l=None: ...
+                        # which has
+                        #  L. 447 0  LOAD_CONST          (None, None)
+                        #         2  LOAD_CLOSURE        'i'
+                        #         4  LOAD_CLOSURE        'predicate'
+                        #         6  BUILD_TUPLE_2    2
+                        #         8  LOAD_LAMBDA         '<code_object <lambda>>'
+                        #        10  LOAD_STR            '_tgrep_relation_action.<locals>...'
+                        #        12  MAKE_FUNCTION_CLOSURE_POS   'default, closure'
+                        # FIXME: Possibly we need to generalize for more nested lambda's of lambda's?
+                        rule = """
+                             expr        ::= lambda_body
+                             lambda_body ::= %s%s%s%s
+                             """ % (
+                            "expr " * stack_count,
+                            "load_closure " * closure,
+                            "BUILD_TUPLE_2 LOAD_LAMBDA LOAD_STR ",
+                            opname,
+                        )
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = """
+                             expr        ::= lambda_body
+                             lambda_body ::= %s%s%s%s
+                             """ % (
+                            "expr " * stack_count,
+                            "load_closure " * closure,
+                            "LOAD_LAMBDA LOAD_STR ",
+                            opname,
+                        )
+
+                    else:
+                        rule = """
+                             expr        ::= lambda_body
+                             lambda_body ::= %s%s%s""" % (
+                            "load_closure " * closure,
+                            "LOAD_LAMBDA LOAD_STR ",
+                            opname,
+                        )
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
+                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+                # Removes POP_BLOCK LOAD_CONST from 3.6-
+                with_as    ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+a                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                    with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                    """
+                else:
+                    rules_str += """
+                    with        ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   BEGIN_FINALLY COME_FROM_WITH
+                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                                   END_FINALLY
+                    """
+                self.addRule(rules_str, nop_func)
+                pass
+            pass
+
+    def custom_classfunc_rule(self, opname, token, customize, next_token):
+
+        args_pos, args_kw = self.get_pos_kw(token)
+
+        # Additional exprs for * and ** args:
+        #  0 if neither
+        #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
+        #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
+        # Yes, this computation based on instruction name is a little bit hoaky.
+        nak = (len(opname) - len("CALL_FUNCTION")) // 3
+        uniq_param = args_kw + args_pos
+
+        if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
+            rule_str = """
+                await      ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
+                await_expr ::= expr await
+                expr       ::= await_expr
+            """
+            self.add_unique_doc_rules(rule_str, customize)
+            rule = (
+                "async_call ::= expr "
+                + ("expr " * args_pos)
+                + ("kwarg " * args_kw)
+                + "expr " * nak
+                + token.kind
+                + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
+            )
+            self.add_unique_rule(rule, token.kind, uniq_param, customize)
+            self.add_unique_rule(
+                "expr ::= async_call", token.kind, uniq_param, customize
+            )
+
+        if opname.startswith("CALL_FUNCTION_KW"):
+            self.addRule("expr ::= call_kw36", nop_func)
+            values = "expr " * token.attr
+            rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
+            self.add_unique_rule(rule, token.kind, token.attr, customize)
+        elif opname == "CALL_FUNCTION_EX_KW":
+            # Note that we don't add to customize token.kind here.
+            # Instead, the non-terminal names, "call_ex_kw"...
+            # are in semantic actions.
+            self.addRule(
+                """expr        ::= call_ex_kw4
+                                   call_ex_kw4 ::= expr
+                                   expr
+                                   expr
+                                   CALL_FUNCTION_EX_KW
+                """,
+                nop_func,
+            )
+            if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                self.addRule(
+                    """expr        ::= call_ex_kw
+                       call_ex_kw  ::= expr expr build_map_unpack_with_call
+                                       CALL_FUNCTION_EX_KW
+                    """,
+                    nop_func,
+                )
+            if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                # FIXME: should this be parameterized by EX value?
+                self.addRule(
+                    """expr        ::= call_ex_kw3
+                                       call_ex_kw3 ::= expr
+                                       build_tuple_unpack_with_call
+                                       expr
+                                       CALL_FUNCTION_EX_KW
+                    """,
+                    nop_func,
+                )
+                if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                    # FIXME: should this be parameterized by EX value?
+                    self.addRule(
+                        """expr        ::= call_ex_kw2
+                                           call_ex_kw2 ::= expr
+                                           build_tuple_unpack_with_call
+                                           build_map_unpack_with_call
+                                           CALL_FUNCTION_EX_KW
+                        """,
+                        nop_func,
+                    )
+
+        elif opname == "CALL_FUNCTION_EX":
+            self.addRule(
+                """
+                expr        ::= call_ex
+                starred     ::= expr
+                call_ex     ::= expr starred CALL_FUNCTION_EX
+                """,
+                nop_func,
+            )
+            if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
+                self.addRule(
+                    """
+                        expr        ::= call_ex_kw
+                        call_ex_kw  ::= expr expr
+                                        build_map_unpack_with_call CALL_FUNCTION_EX
+                        """,
+                    nop_func,
+                )
+            if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
+                self.addRule(
+                    """
+                        expr        ::= call_ex_kw3
+                        call_ex_kw3 ::= expr
+                                        build_tuple_unpack_with_call
+                                        %s
+                                        CALL_FUNCTION_EX
+                        """
+                    % "expr "
+                    * token.attr,
+                    nop_func,
+                )
+                pass
+
+            # FIXME: Is this right?
+            self.addRule(
+                """
+                        expr        ::= call_ex_kw4
+                        call_ex_kw4 ::= expr
+                                        expr
+                                        expr
+                                        CALL_FUNCTION_EX
+                        """,
+                nop_func,
+            )
+            pass
+        else:
+            Python37BaseParser.custom_classfunc_rule(
+                self, opname, token, customize, next_token
+            )

python/py/Lib/site-packages/decompyle3/parsers/p38/lambda_expr.py

@@ -0,0 +1,85 @@
+#  Copyright (c) 2020-2023 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Differences over Python 3.7 for Python 3.8 in the Earley-algorithm lambda grammar
+"""
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from decompyle3.parsers.p37.lambda_expr import Python37LambdaParser
+from decompyle3.parsers.p38.lambda_custom import Python38LambdaCustom
+from decompyle3.parsers.parse_heads import PythonBaseParser, PythonParserLambda
+
+
+class Python38LambdaParser(
+    Python38LambdaCustom, Python37LambdaParser, PythonParserLambda
+):
+    def p_38walrus(self, args):
+        """
+        # named_expr is also known as the "walrus op" :=
+        expr              ::= named_expr
+        named_expr        ::= expr DUP_TOP store
+        """
+
+    def p_lambda_start(self, args):
+        """
+        return_expr_lambda ::= genexpr_func LOAD_CONST RETURN_VALUE_LAMBDA
+
+        """
+
+    def p_expr38(self, args):
+        """
+        expr ::= if_exp_compare38
+
+        if_exp_compare38 ::= or_in_ifexp jump_if_false_cf expr jf_cfs expr come_froms
+
+        list_iter        ::= list_if_not38
+        list_if_not38    ::= expr pjump_ift expr pjump_ift _come_froms list_iter
+                             come_from_opt
+
+        or_in_ifexp      ::= expr_pjit expr
+        or_in_ifexp      ::= or_in_ifexp POP_JUMP_IF_TRUE expr
+        """
+
+    def __init__(
+        self,
+        start_symbol: str = "lambda_start",
+        debug_parser: dict = PARSER_DEFAULT_DEBUG,
+    ):
+        PythonParserLambda.__init__(
+            self, debug_parser=debug_parser, start_symbol=start_symbol
+        )
+        PythonBaseParser.__init__(
+            self, start_symbol=start_symbol, debug_parser=debug_parser
+        )
+        Python38LambdaCustom.__init__(self)
+
+    def customize_grammar_rules(self, tokens, customize):
+        self.customize_grammar_rules_lambda38(tokens, customize)
+
+
+if __name__ == "__main__":
+    # Check grammar
+    from decompyle3.parsers.dump import dump_and_check
+
+    p = Python38LambdaParser()
+    modified_tokens = set(
+        """JUMP_LOOP CONTINUE RETURN_END_IF COME_FROM
+           LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+           LAMBDA_MARKER RETURN_LAST
+        """.split()
+    )
+
+    dump_and_check(p, (3, 8), modified_tokens)

python/py/Lib/site-packages/decompyle3/parsers/p38pypy/__init__.py

@@ -0,0 +1,4 @@
+"""
+Here we have Python 3.8 PyPy grammars and associated customization
+for the both full language and the subset used in lambda expressions.
+"""

python/py/Lib/site-packages/decompyle3/parsers/p38pypy/base.py

@@ -0,0 +1,50 @@
+#  Copyright (c) 2020-2022, 2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from decompyle3.parsers.parse_heads import PythonBaseParser
+from decompyle3.parsers.reduce_check import (
+    break_invalid,
+    for38_invalid,
+    forelse38_invalid,
+    pop_return_check,
+    whilestmt38_check,
+    whileTruestmt38_check,
+)
+
+
+class Python38PyPyBaseParser(PythonBaseParser):
+    def __init__(self, start_symbol, debug_parser: dict = PARSER_DEFAULT_DEBUG):
+        super(Python38PyPyBaseParser, self).__init__(
+            start_symbol=start_symbol, debug_parser=debug_parser
+        )
+
+    def customize_grammar_rules38(self, tokens, customize):
+        self.customize_grammar_rules37(tokens, customize)
+        self.check_reduce["break"] = "tokens"
+        self.check_reduce["for38"] = "tokens"
+        self.check_reduce["forelsestmt38"] = "AST"
+        self.check_reduce["pop_return"] = "tokens"
+        self.check_reduce["whileTruestmt38"] = "AST"
+        self.check_reduce["whilestmt38"] = "tokens"
+        self.check_reduce["try_elsestmtl38"] = "AST"
+
+        self.reduce_check_table["break"] = break_invalid
+        self.reduce_check_table["for38"] = for38_invalid
+        self.reduce_check_table["forelsestmt38"] = forelse38_invalid
+        self.reduce_check_table["pop_return"] = pop_return_check
+        self.reduce_check_table["whilestmt38"] = whilestmt38_check
+        self.reduce_check_table["whileTruestmt38"] = whileTruestmt38_check

python/py/Lib/site-packages/decompyle3/parsers/p38pypy/full.py

@@ -0,0 +1,682 @@
+#  Copyright (c) 2017-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+spark grammar for Python 3.8 PyPy
+"""
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from spark_parser.spark import rule2str
+
+from decompyle3.parsers.p37.full import Python37Parser
+from decompyle3.parsers.p38pypy.full_custom import Python38PyPyFullCustom
+from decompyle3.parsers.p38pypy.lambda_expr import Python38PyPyLambdaParser
+from decompyle3.parsers.parse_heads import ParserError
+from decompyle3.scanners.tok import Token
+
+
+class Python38PyPyParser(
+    Python38PyPyLambdaParser, Python38PyPyFullCustom, Python37Parser
+):
+    def __init__(self, start_symbol: str = "stmts", debug_parser=PARSER_DEFAULT_DEBUG):
+        Python38PyPyLambdaParser.__init__(self, start_symbol, debug_parser)
+        self.customized = {}
+
+    def customize_grammar_rules(self, tokens, customize):
+        self.customize_grammar_rules_full38(tokens, customize)
+
+    ###############################################
+    #  Python 3.8 grammar rules with statements
+    ###############################################
+
+    def p_38_full_if_ifelse(self, args):
+        """
+        # cf_pt introduced to keep indices the same in ifelsestmtc
+        cf_pt              ::= COME_FROM POP_TOP
+        ifelsestmtc        ::= testexpr c_stmts cf_pt else_suite
+
+        # 3.8 can push a looping JUMP_LOOP into a JUMP_ from a statement that jumps to
+        # it
+        lastc_stmt         ::= ifpoplaststmtc
+        ifpoplaststmtc     ::= testexpr POP_TOP c_stmts_opt
+        ifelsestmtc        ::= testexpr c_stmts_opt jb_cfs else_suitec JUMP_LOOP
+                               come_froms
+
+        testtrue   ::= or_in_ifexp POP_JUMP_IF_TRUE
+
+
+        # The below ifelsetmtc is a really weird one for the inner if/else in:
+        #  if a:
+        #      while i:
+        #       if c:
+        #         j = j + 1
+        #                 # A JUMP_LOOP is here...
+        #       else:
+        #          break
+        #                 # but also a JUMP_LOOP is inserted here!
+        #  else:
+        #    j = 10
+
+        ifelsestmtc        ::= testexpr c_stmts_opt JUMP_LOOP else_suitec JUMP_LOOP
+        """
+
+    def p_38_full_stmt(self, args):
+        """
+        stmt               ::= async_with_stmt38
+        stmt               ::= for38
+        stmt               ::= forelselaststmt38
+        stmt               ::= forelselaststmtc38
+        stmt               ::= forelsestmt38
+        stmt               ::= try_elsestmtl38
+        stmt               ::= try_except38
+        stmt               ::= try_except38r
+        stmt               ::= try_except38r2
+        stmt               ::= try_except38r3
+        stmt               ::= try_except38r4
+        stmt               ::= try_except38r5
+        stmt               ::= try_except38r6
+        stmt               ::= try_except38r7
+        stmt               ::= try_except_as
+        stmt               ::= try_except_ret38
+        stmt               ::= try_except_ret38a
+        stmt               ::= tryfinallystmt_break
+        stmt               ::= tryfinally38astmt
+        stmt               ::= tryfinally38rstmt
+        stmt               ::= tryfinally38rstmt2
+        stmt               ::= tryfinally38rstmt3
+        stmt               ::= tryfinally38rstmt4
+        stmt               ::= tryfinally38rstmt5
+        stmt               ::= tryfinally38stmt
+        stmt               ::= tryfinally38_return
+        stmt               ::= tryfinally38a_return
+        stmt               ::= tryfinally38rstmt2
+        stmt               ::= whileTruestmt38
+        stmt               ::= whilestmt38
+
+        # FIXME: "break"" should be isolated to loops
+        stmt  ::= break
+
+        break ::= POP_BLOCK BREAK_LOOP
+        break ::= POP_BLOCK POP_TOP BREAK_LOOP
+        break ::= POP_TOP BREAK_LOOP
+        break ::= POP_EXCEPT BREAK_LOOP
+        break ::= POP_TOP CONTINUE JUMP_LOOP
+
+        # An except with nothing other than a single break
+        break_except ::= POP_EXCEPT POP_TOP BREAK_LOOP
+
+        # FIXME: this should be restricted to being inside a try block
+        stmt               ::= except_ret38
+        stmt               ::= except_ret38a
+
+        async_with_stmt38    ::= expr
+                                 BEFORE_ASYNC_WITH
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 SETUP_ASYNC_WITH
+                                 POP_TOP
+                                 c_stmts_opt
+                                 POP_BLOCK
+                                 BEGIN_FINALLY
+                                 COME_FROM_ASYNC_WITH
+                                 WITH_CLEANUP_START
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 WITH_CLEANUP_FINISH
+                                 END_FINALLY
+
+        async_with_stmt38    ::= expr
+                                 BEFORE_ASYNC_WITH
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 SETUP_ASYNC_WITH
+                                 POP_TOP
+                                 c_stmts_opt
+                                 POP_BLOCK
+                                 BEGIN_FINALLY
+                                 WITH_CLEANUP_START
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 WITH_CLEANUP_FINISH
+                                 POP_FINALLY
+
+        async_with_stmt38    ::= expr
+                                 BEFORE_ASYNC_WITH
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 SETUP_ASYNC_WITH
+                                 POP_TOP
+                                 c_stmts_opt
+                                 POP_BLOCK
+                                 BEGIN_FINALLY
+                                 WITH_CLEANUP_START
+                                 GET_AWAITABLE
+                                 LOAD_CONST
+                                 YIELD_FROM
+                                 WITH_CLEANUP_FINISH
+                                 POP_FINALLY
+                                 JUMP_LOOP
+
+        # Seems to be used to discard values before a return in a "for" loop
+        discard_top        ::= ROT_TWO POP_TOP
+        discard_tops       ::= discard_top+
+        pop_tops           ::= POP_TOP+
+
+        return             ::= return_expr
+                               discard_tops RETURN_VALUE
+
+        return             ::= pop_return
+        return             ::= popb_return
+        return             ::= pop_ex_return
+        except_stmt        ::= except_with_break
+        except_stmt        ::= except_with_break2
+        except_stmt        ::= pop_ex_return
+        except_stmt        ::= pop3_except_return38
+        except_stmt        ::= pop3_rot4_except_return38
+        except_stmt        ::= except_cond_pop3_rot4_except_return38
+
+        except_stmts       ::= except_stmt+
+        except_stmts_opt   ::= except_stmt*
+
+        pop_return         ::= POP_TOP return_expr RETURN_VALUE
+        popb_return        ::= return_expr POP_BLOCK RETURN_VALUE
+
+        # Return from exception where value is on stack
+        pop_ex_return      ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
+
+        # Return from exception where value is no on stack but is computed
+        pop_ex_return2      ::= POP_EXCEPT expr RETURN_VALUE
+
+
+        # The below are 3.8 "except:" (no except condition)
+
+        pop3_except_return38       ::= POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                                       CALL_FINALLY return
+
+        except_return38            ::= POP_BLOCK
+                                       CALL_FINALLY POP_TOP return
+
+        pop3_rot4_except_return38  ::= POP_TOP POP_TOP POP_TOP
+                                       except_stmts_opt return_expr ROT_FOUR
+                                       POP_EXCEPT POP_BLOCK CALL_FINALLY RETURN_VALUE
+
+
+        pop3_rot4_except_return38  ::= POP_TOP POP_TOP POP_TOP
+                                       except_stmts_opt return_expr ROT_FOUR
+                                       POP_EXCEPT POP_BLOCK ROT_TWO POP_TOP
+                                       CALL_FINALLY RETURN_VALUE
+                                       END_FINALLY COME_FROM POP_BLOCK
+                                       BEGIN_FINALLY COME_FROM
+
+        # The above but with an except condition name e.g. "except Exception:"
+        except_cond_pop3_rot4_except_return38 ::= except_cond1
+                                       except_stmts_opt return_expr ROT_FOUR
+                                       POP_EXCEPT POP_BLOCK CALL_FINALLY RETURN_VALUE
+                                       COME_FROM
+
+        except_stmt        ::= except_cond1 except_suite come_from_opt
+        except_stmt        ::= except_cond2 except_ret38b
+
+        get_iter           ::= expr GET_ITER
+        for38              ::= expr get_iter store for_block JUMP_LOOP _come_froms
+        for38              ::= expr get_for_iter store for_block JUMP_LOOP _come_froms
+        for38              ::= expr get_for_iter store for_block JUMP_LOOP _come_froms
+                               POP_BLOCK
+        for38              ::= expr get_for_iter store for_block _come_froms
+
+        forelsestmt38      ::= expr get_for_iter store for_block POP_BLOCK else_suite
+                               _come_froms
+        forelsestmt38      ::= expr get_for_iter store for_block JUMP_LOOP _come_froms
+                               else_suite _come_froms
+
+        c_stmt             ::= c_forelsestmt38
+        c_stmt             ::= pop_tops return
+        c_forelsestmt38    ::= expr get_for_iter store for_block POP_BLOCK else_suitec
+        c_forelsestmt38    ::= expr get_for_iter store for_block JUMP_LOOP _come_froms
+                               else_suitec
+
+        # continue is a weird one. In 3.8, CONTINUE_LOOP was removed.
+        # Inside an loop we can have this, which can only appear in side a try/except
+        # And it can also appear at the end of the try except.
+        continue           ::= POP_EXCEPT JUMP_LOOP
+
+        forelselaststmt38    ::= expr get_for_iter store for_block else_suitec
+                                 _come_froms
+        forelselaststmtc38   ::= expr get_for_iter store for_block else_suitec
+                                 _come_froms
+        # forelselaststmt38  ::= expr get_for_iter store for_block POP_BLOCK else_suitec
+        # forelselaststmtc38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec
+
+        returns_in_except   ::= _stmts except_return_value
+        returns_in_except2   ::= _stmts except_return_value2
+
+        except_return_value ::= POP_BLOCK return
+        except_return_value ::= expr POP_BLOCK RETURN_VALUE
+        except_return_value2 ::= POP_BLOCK return
+
+        whilestmt38        ::= _come_froms testexpr  c_stmts_opt COME_FROM JUMP_LOOP
+                                POP_BLOCK
+        whilestmt38        ::= _come_froms testexpr  c_stmts_opt JUMP_LOOP POP_BLOCK
+        whilestmt38        ::= _come_froms testexpr  c_stmts_opt JUMP_LOOP come_froms
+        whilestmt38        ::= _come_froms testexprc c_stmts_opt come_froms JUMP_LOOP
+                               _come_froms
+        whilestmt38        ::= _come_froms testexpr  returns               POP_BLOCK
+        whilestmt38        ::= _come_froms testexpr  c_stmts     JUMP_LOOP _come_froms
+        whilestmt38        ::= _come_froms testexpr  c_stmts     come_froms
+        whilestmt38        ::= _come_froms bool_op   c_stmts     JUMP_LOOP _come_froms
+
+        # while1elsestmt   ::=  c_stmts JUMP_LOOP
+        whileTruestmt      ::= _come_froms c_stmts              JUMP_LOOP _come_froms
+                               POP_BLOCK
+        while1stmt         ::= _come_froms c_stmts COME_FROM_LOOP
+        while1stmt         ::= _come_froms c_stmts COME_FROM JUMP_LOOP COME_FROM_LOOP
+        whileTruestmt38    ::= _come_froms c_stmts JUMP_LOOP _come_froms
+        whileTruestmt38    ::= _come_froms c_stmts JUMP_LOOP COME_FROM_EXCEPT_CLAUSE
+        whileTruestmt38    ::= _come_froms pass JUMP_LOOP
+
+        for_block          ::= _come_froms c_stmts_opt come_from_loops JUMP_LOOP
+
+        # Note there is a 3.7 except_cond1 that doesn't have the final POP_EXCEPT
+        except_cond1       ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP
+                               POP_EXCEPT
+
+        except_suite       ::= c_stmts_opt
+                               POP_EXCEPT POP_TOP JUMP_FORWARD POP_EXCEPT
+                               jump_except
+
+        try_elsestmtl38    ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               except_handler38 COME_FROM
+                               else_suitec opt_come_from_except
+        try_except         ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               except_handler38
+        try_except         ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               except_handler38
+                               jump_excepts
+                               come_from_except_clauses
+
+        c_try_except       ::= SETUP_FINALLY c_suite_stmts POP_BLOCK
+                               except_handler38
+
+        c_stmt             ::= c_tryfinallystmt38
+        c_stmt             ::= c_tryfinallybstmt38
+
+        c_tryfinallystmt38 ::= SETUP_FINALLY c_suite_stmts_opt
+                               POP_BLOCK
+                               CALL_FINALLY
+                               POP_BLOCK
+                               POP_EXCEPT
+                               CALL_FINALLY
+                               JUMP_FORWARD
+                               POP_BLOCK BEGIN_FINALLY
+                               COME_FROM
+                               COME_FROM_FINALLY
+                               c_suite_stmts_opt END_FINALLY
+
+        # try:
+        #    ..
+        #    break
+        # finally:
+        c_tryfinallybstmt38 ::= SETUP_FINALLY c_suite_stmts_opt
+                               POP_BLOCK
+                               CALL_FINALLY
+                               POP_BLOCK
+                               POP_EXCEPT
+                               CALL_FINALLY
+                               BREAK_LOOP
+                               POP_BLOCK BEGIN_FINALLY
+                               COME_FROM
+                               COME_FROM_FINALLY
+                               c_suite_stmts_opt END_FINALLY
+
+        c_tryfinallystmt38 ::= SETUP_FINALLY c_suite_stmts_opt
+                               POP_BLOCK BEGIN_FINALLY COME_FROM COME_FROM_FINALLY
+                               c_suite_stmts_opt END_FINALLY
+
+        try_except38       ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
+                               except_handler38a
+
+        # suite_stmts has a return
+        try_except38       ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                               except_handler38b
+        try_except38r      ::= SETUP_FINALLY return_except
+                               except_handler38b
+        return_except      ::= stmts POP_BLOCK return
+
+
+        # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there
+        # is a final return in the "except" block.
+        # So we treat the "return" separate from the other statements
+        cond_except_stmt      ::= except_cond1 except_stmts
+        cond_except_stmts_opt ::= cond_except_stmt*
+
+        try_except38r2     ::= SETUP_FINALLY
+                               suite_stmts_opt
+                               POP_BLOCK JUMP_FORWARD
+                               COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               cond_except_stmts_opt
+                               POP_EXCEPT return
+                               END_FINALLY
+                               COME_FROM
+
+        try_except38r3     ::= SETUP_FINALLY
+                               suite_stmts_opt
+                               POP_BLOCK JUMP_FORWARD
+                               COME_FROM_FINALLY
+                               cond_except_stmts_opt
+                               POP_EXCEPT return
+                               COME_FROM
+                               END_FINALLY
+                               COME_FROM
+
+
+         # I think this can be combined with the r5
+        try_except38r4     ::= SETUP_FINALLY
+                               returns_in_except
+                               COME_FROM_FINALLY
+                               except_cond1
+                               return
+                               COME_FROM
+                               END_FINALLY
+
+        try_except38r5     ::= SETUP_FINALLY
+                               returns_in_except
+                               COME_FROM_FINALLY
+                               except_cond1
+                               except_ret38d
+                               COME_FROM
+                               END_FINALLY
+
+        try_except38r5     ::= SETUP_FINALLY
+                               returns_in_except
+                               COME_FROM_FINALLY
+                               except_cond1
+                               except_suite
+                               COME_FROM
+                               END_FINALLY
+                               COME_FROM
+
+        try_except38r5     ::= SETUP_FINALLY
+                               returns_in_except
+                               COME_FROM_FINALLY
+                               except_cond2
+                               except_ret38b
+                               END_FINALLY COME_FROM
+
+        try_except38r6     ::= SETUP_FINALLY
+                               returns_in_except2
+                               COME_FROM_FINALLY
+                               POP_TOP POP_TOP POP_TOP
+                               except_ret38d
+                               END_FINALLY
+
+
+        try_except38r7     ::= SETUP_FINALLY
+                               suite_stmts_opt
+                               POP_BLOCK JUMP_FORWARD
+                               COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               return_expr
+                               ROT_FOUR POP_EXCEPT POP_BLOCK ROT_TWO POP_TOP
+                               CALL_FINALLY RETURN_VALUE
+                               END_FINALLY
+                               COME_FROM POP_BLOCK
+                               BEGIN_FINALLY
+                               COME_FROM
+                               COME_FROM_FINALLY
+
+
+        try_except_as      ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                               except_handler_as END_FINALLY COME_FROM
+        try_except_as      ::= SETUP_FINALLY suite_stmts
+                               except_handler_as END_FINALLY COME_FROM
+
+
+        try_except_ret38   ::= SETUP_FINALLY returns except_ret38a
+        try_except_ret38a  ::= SETUP_FINALLY returns except_handler38c
+                               END_FINALLY come_from_opt
+
+        # Note: there is a suite_stmts_opt which seems
+        # to be bookkeeping which is not expressed in source code
+        except_ret38       ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT
+                               CALL_FINALLY RETURN_VALUE COME_FROM
+                               COME_FROM_FINALLY
+                               suite_stmts_opt END_FINALLY
+        except_ret38a      ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               expr ROT_FOUR
+                               POP_EXCEPT RETURN_VALUE END_FINALLY
+
+        except_ret38b      ::= SETUP_FINALLY suite_stmts expr
+                               ROT_FOUR POP_BLOCK POP_EXCEPT
+                               CALL_FINALLY RETURN_VALUE COME_FROM
+                               COME_FROM_FINALLY
+                               suite_stmts_opt END_FINALLY
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+
+        except_ret38c      ::= SETUP_FINALLY suite_stmts expr
+                               ROT_FOUR POP_BLOCK POP_EXCEPT
+                               CALL_FINALLY POP_BLOCK CALL_FINALLY RETURN_VALUE
+                               COME_FROM
+                               COME_FROM_FINALLY
+                               expr STORE_FAST DELETE_FAST END_FINALLY
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+                               END_FINALLY come_any_froms
+
+        except_ret38d      ::= suite_stmts_opt
+                               expr ROT_FOUR
+                               POP_EXCEPT RETURN_VALUE
+
+        except_handler38   ::= jump COME_FROM_FINALLY
+                               except_stmts
+                               END_FINALLY
+                               opt_come_from_except
+
+        except_handler38a  ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               POP_EXCEPT POP_TOP stmts END_FINALLY
+        except_handler38b  ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                               POP_EXCEPT returns END_FINALLY
+        except_handler38c  ::= COME_FROM_FINALLY except_cond1 except_stmts
+                               COME_FROM
+        except_handler38c  ::= COME_FROM_FINALLY except_cond1 except_stmts
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+
+        except_handler_as  ::= COME_FROM_FINALLY except_cond2 tryfinallystmt
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+
+        except             ::= POP_TOP POP_TOP POP_TOP c_stmts_opt break
+                               POP_EXCEPT
+
+        # Except of a try inside a loop
+        except             ::= POP_TOP POP_TOP POP_TOP c_stmts_opt break
+                               POP_EXCEPT JUMP_LOOP
+
+        except             ::= POP_TOP POP_TOP POP_TOP c_stmts_opt
+                               POP_EXCEPT JUMP_LOOP
+
+        except_with_break  ::= POP_TOP POP_TOP POP_TOP c_stmts break_except
+                               POP_EXCEPT JUMP_LOOP
+
+        # Just except: break, no statements
+        except_with_break2 ::= POP_TOP POP_TOP POP_TOP break_except
+                               POP_EXCEPT JUMP_LOOP
+
+        except_with_return38 ::= POP_TOP POP_TOP POP_TOP stmts pop_ex_return2
+        except_with_return38 ::= POP_TOP POP_TOP POP_TOP pop_ex_return2
+
+        except_stmt         ::= except_with_return38
+
+
+        # In 3.8 any POP_EXCEPT comes before the "break" loop.
+        # We should add a rule to check that JUMP_FORWARD is indeed a "break".
+        break              ::=  POP_EXCEPT JUMP_FORWARD
+        break              ::=  POP_BLOCK POP_TOP JUMP_FORWARD
+
+        tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
+                               END_FINALLY
+
+        tryfinallystmt_break ::=
+                               SETUP_FINALLY suite_stmts_opt POP_BLOCK POP_EXCEPT
+                               CALL_FINALLY
+                               JUMP_FORWARD POP_BLOCK
+                               BEGIN_FINALLY COME_FROM COME_FROM_FINALLY suite_stmts_opt
+                               END_FINALLY
+
+
+        lc_setup_finally   ::= LOAD_CONST SETUP_FINALLY
+        call_finally_pt    ::= CALL_FINALLY POP_TOP
+        cf_cf_finally      ::= come_from_opt COME_FROM_FINALLY
+        pop_finally_pt     ::= POP_FINALLY POP_TOP
+        ss_end_finally     ::= suite_stmts END_FINALLY
+        sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns
+        sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK POP_EXCEPT CALL_FINALLY returns
+
+        suite_stmts_return ::= suite_stmts expr
+        suite_stmts_return ::= expr
+
+
+        # FIXME: DRY rules below
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally
+                               ss_end_finally
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally END_FINALLY
+                               suite_stmts
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               cf_cf_finally POP_FINALLY
+                               ss_end_finally
+        tryfinally38rstmt  ::= sf_pb_call_returns
+                               COME_FROM_FINALLY POP_FINALLY
+                               ss_end_finally
+
+        tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt
+                               returns
+                               cf_cf_finally pop_finally_pt
+                               ss_end_finally POP_TOP
+
+        tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE
+                               COME_FROM COME_FROM_FINALLY
+                               ss_end_finally
+
+        tryfinally38rstmt4 ::= lc_setup_finally suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM_FINALLY
+                               suite_stmts_return
+                               POP_FINALLY ROT_TWO POP_TOP
+                               RETURN_VALUE
+                               END_FINALLY POP_TOP
+
+
+        tryfinally38rstmt5 ::= lc_setup_finally try_except38r7 expr
+                               POP_FINALLY ROT_TWO POP_TOP
+                               RETURN_VALUE
+                               END_FINALLY POP_TOP
+
+        tryfinally38stmt   ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM_FINALLY
+                               POP_FINALLY suite_stmts_opt END_FINALLY
+
+        tryfinally38stmt   ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM
+                               COME_FROM_FINALLY
+                               suite_stmts_opt END_FINALLY
+
+        # try: .. finally: ending with return ...
+        tryfinally38_return ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               JUMP_FORWARD
+                               COME_FROM_FINALLY except_cond2 except_ret38c
+
+
+        tryfinally38a_return ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt except_return38
+                                 COME_FROM COME_FROM_FINALLY
+                                 suite_stmts_opt pop_finally_pt return
+                                 END_FINALLY POP_TOP
+
+
+        tryfinally38astmt  ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               BEGIN_FINALLY COME_FROM_FINALLY
+                               POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP
+        """
+
+    def p_38_full_walrus(self, args):
+        """
+        # named_expr is also known as the "walrus op" :=
+        expr              ::= named_expr
+        named_expr        ::= expr DUP_TOP store
+        """
+
+    # FIXME: try this
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        lhs = rule[0]
+        if lhs == "call_kw":
+            # Make sure we don't derive call_kw
+            nt = ast[0]
+            while not isinstance(nt, Token):
+                if nt[0] == "call_kw":
+                    return True
+                nt = nt[0]
+                pass
+            pass
+        n = len(tokens)
+        last = min(last, n - 1)
+        fn = self.reduce_check_table.get(lhs, None)
+        try:
+            if fn:
+                return fn(self, lhs, n, rule, ast, tokens, first, last)
+        except Exception:
+            import sys
+            import traceback
+
+            print(
+                f"Exception in {fn.__name__} {sys.exc_info()[1]}\n"
+                + f"rule: {rule2str(rule)}\n"
+                + f"offsets {tokens[first].offset} .. {tokens[last].offset}"
+            )
+            print(traceback.print_tb(sys.exc_info()[2], -1))
+            raise ParserError(tokens[last], tokens[last].off2int(), self.debug["rules"])
+
+        if lhs in ("aug_assign1", "aug_assign2") and ast[0][0] == "and":
+            return True
+        elif lhs == "annotate_tuple":
+            return not isinstance(tokens[first].attr, tuple)
+        elif lhs == "import_from37":
+            importlist37 = ast[3]
+            alias37 = importlist37[0]
+            if importlist37 == "importlist37" and alias37 == "alias37":
+                store = alias37[1]
+                assert store == "store"
+                return alias37[0].attr != store[0].attr
+            return False
+
+        return False
+
+        return False
+
+
+if __name__ == "__main__":
+    # Check grammar
+    from decompyle3.parsers.dump import dump_and_check
+
+    p = Python38PyPyParser()
+    modified_tokens = set(
+        """JUMP_LOOP CONTINUE RETURN_END_IF COME_FROM
+           LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+           LAMBDA_MARKER RETURN_LAST
+        """.split()
+    )
+
+    dump_and_check(p, (3, 8), modified_tokens)

python/py/Lib/site-packages/decompyle3/parsers/p38pypy/full_custom.py

@@ -0,0 +1,1309 @@
+#  Copyright (c) 2021-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# from decompyle3.parsers.reduce_check.import_from37 import import_from37_ok
+from decompyle3.parsers.p37.base import Python37BaseParser
+from decompyle3.parsers.p38.lambda_custom import Python38LambdaCustom
+from decompyle3.parsers.parse_heads import PythonBaseParser, nop_func
+from decompyle3.parsers.reduce_check import (  # joined_str_check,
+    break_invalid,
+    for38_invalid,
+    forelse38_invalid,
+    if_not_stmtc_invalid,
+    pop_return_check,
+    whilestmt38_check,
+    whileTruestmt38_check,
+)
+
+# from decompyle3.parsers.reduce_check.ifelsestmt_check import ifelsestmt_ok
+from decompyle3.parsers.reduce_check.ifstmt import ifstmt
+from decompyle3.parsers.reduce_check.or_cond_check import or_cond_check_invalid
+
+
+class Python38PyPyFullCustom(Python38LambdaCustom, PythonBaseParser):
+    def add_make_function_rule(self, rule, opname, attr, customize):
+        """Python 3.3 added an additional LOAD_STR before MAKE_FUNCTION and
+        this has an effect on many rules.
+        """
+        new_rule = rule % "LOAD_STR "
+        self.add_unique_rule(new_rule, opname, attr, customize)
+
+    @staticmethod
+    def call_fn_name(token):
+        """Customize CALL_FUNCTION to add the number of positional arguments"""
+        if token.attr is not None:
+            return f"{token.kind}_{token.attr}"
+        else:
+            return f"{token.kind}_0"
+
+    def remove_rules_38(self):
+        self.remove_rules(
+            """
+           stmt               ::= async_for_stmt37
+           stmt               ::= for
+           stmt               ::= forelsestmt
+           stmt               ::= try_except36
+           stmt               ::= async_forelse_stmt
+
+           # There is no SETUP_LOOP
+           setup_loop         ::= SETUP_LOOP _come_froms
+           forelselaststmt    ::= SETUP_LOOP expr get_for_iter store
+                                  for_block POP_BLOCK else_suitec _come_froms
+
+           forelsestmt        ::= SETUP_LOOP expr get_for_iter store
+           whileTruestmt      ::= SETUP_LOOP c_stmts_opt JUMP_LOOP COME_FROM_LOOP
+                                  for_block POP_BLOCK else_suite _come_froms
+
+           # async_for_stmt     ::= setup_loop expr
+           #                        GET_AITER
+           #                        SETUP_EXCEPT GET_ANEXT LOAD_CONST
+           #                        YIELD_FROM
+           #                        store
+           #                        POP_BLOCK JUMP_FORWARD bb_end_start DUP_TOP
+           #                        LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+           #                        END_FINALLY bb_end_start
+           #                        for_block
+           #                        COME_FROM
+           #                        POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
+           #                        COME_FROM_LOOP
+
+           # async_for_stmt37   ::= setup_loop expr
+           #                        GET_AITER
+           #                        SETUP_EXCEPT GET_ANEXT
+           #                        LOAD_CONST YIELD_FROM
+           #                        store
+           #                        POP_BLOCK JUMP_LOOP COME_FROM_EXCEPT DUP_TOP
+           #                        LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+           #                        END_FINALLY for_block COME_FROM
+           #                        POP_TOP POP_TOP POP_TOP POP_EXCEPT
+           #                        POP_TOP POP_BLOCK
+           #                        COME_FROM_LOOP
+
+           # async_forelse_stmt ::= setup_loop expr
+           #                        GET_AITER
+           #                        SETUP_EXCEPT GET_ANEXT LOAD_CONST
+           #                        YIELD_FROM
+           #                        store
+           #                        POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+           #                        LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
+           #                        END_FINALLY COME_FROM
+           #                        for_block
+           #                        COME_FROM
+           #                        POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
+           #                        else_suite COME_FROM_LOOP
+
+           for                ::= setup_loop expr get_for_iter store for_block POP_BLOCK
+           for                ::= setup_loop expr get_for_iter store for_block POP_BLOCK NOP
+
+           for_block          ::= c_stmts_opt COME_FROM_LOOP JUMP_LOOP
+           forelsestmt        ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suite
+           forelselaststmt    ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec
+           forelselaststmtc   ::= setup_loop expr get_for_iter store for_block POP_BLOCK else_suitec
+
+           try_except         ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                  except_handler opt_come_from_except
+
+           tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                  LOAD_CONST COME_FROM_FINALLY suite_stmts_opt
+                                  END_FINALLY
+           tryfinally36       ::= SETUP_FINALLY returns
+                                  COME_FROM_FINALLY suite_stmts_opt END_FINALLY
+           tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                      LOAD_CONST COME_FROM_FINALLY
+        """
+        )
+
+    # def custom_classfunc_rule(self, opname, token, customize, next_token):
+    #     """
+    #     call ::= expr {expr}^n CALL_FUNCTION_n
+    #     call ::= expr {expr}^n CALL_FUNCTION_VAR_n
+    #     call ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n
+    #     call ::= expr {expr}^n CALL_FUNCTION_KW_n
+
+    #     classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc {expr}^n-1 CALL_FUNCTION_n
+    #     """
+    #     args_pos, args_kw = self.get_pos_kw(token)
+
+    #     # Additional exprs for * and ** args:
+    #     #  0 if neither
+    #     #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
+    #     #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
+    #     # Yes, this computation based on instruction name is a little bit hoaky.
+    #     nak = (len(opname) - len("CALL_FUNCTION")) // 3
+    #     uniq_param = args_kw + args_pos
+    #     if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
+    #         rule = (
+    #             "async_call ::= expr "
+    #             + ("expr " * args_pos)
+    #             + ("kwarg " * args_kw)
+    #             + "expr " * nak
+    #             + token.kind
+    #             + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
+    #         )
+    #         self.add_unique_rule(rule, token.kind, uniq_param, customize)
+    #         self.add_unique_rule(
+    #             "expr ::= async_call", token.kind, uniq_param, customize
+    #         )
+
+    #     if opname.startswith("CALL_FUNCTION_VAR"):
+    #         token.kind = self.call_fn_name(token)
+    #         if opname.endswith("KW"):
+    #             kw = "expr "
+    #         else:
+    #             kw = ""
+    #         rule = (
+    #             "call ::= expr expr "
+    #             + ("expr " * args_pos)
+    #             + ("kwarg " * args_kw)
+    #             + kw
+    #             + token.kind
+    #         )
+
+    #         # Note: semantic actions make use of the fact of whether "args_pos"
+    #         # zero or not in creating a template rule.
+    #         self.add_unique_rule(rule, token.kind, args_pos, customize)
+    #     else:
+    #         token.kind = self.call_fn_name(token)
+    #         uniq_param = args_kw + args_pos
+
+    #         # Note: 3.5+ have subclassed this method; so we don't handle
+    #         # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here.
+    #         rule = (
+    #             "call ::= expr "
+    #             + ("expr " * args_pos)
+    #             + ("kwarg " * args_kw)
+    #             + "expr " * nak
+    #             + token.kind
+    #         )
+
+    #         self.add_unique_rule(rule, token.kind, uniq_param, customize)
+
+    #         if "LOAD_BUILD_CLASS" in self.seen_ops:
+    #             if (
+    #                 next_token == "CALL_FUNCTION"
+    #                 and next_token.attr == 1
+    #                 and args_pos > 1
+    #             ):
+    #                 rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % (
+    #                     ("expr " * (args_pos - 1)),
+    #                     opname,
+    #                     args_pos,
+    #                 )
+    #                 self.add_unique_rule(rule, token.kind, uniq_param, customize)
+
+    def customize_grammar_rules_full38(self, tokens, customize):
+
+        self.customize_grammar_rules_lambda38(tokens, customize)
+        self.customize_reduce_checks_full38(tokens, customize)
+        self.remove_rules_38()
+
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "CONTINUE",
+                "DELETE",
+                "FORMAT",
+                "GET",
+                "JUMP",
+                "LOAD",
+                "LOOKUP",
+                "MAKE",
+                "RETURN",
+                "RAISE",
+                "SETUP",
+                "UNPACK",
+                "WITH",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get added
+        # unconditionally and the rules are constant. So they need to be done
+        # only once and if we see the opcode a second we don't have to consider
+        # adding more rules.
+        #
+        custom_ops_processed = set()
+
+        # A set of instruction operation names that exist in the token stream.
+        # We use this to customize the grammar that we create.
+        # 2.6-compatible set comprehensions
+
+        # The initial initialization is done in lambea_expr.py
+        self.seen_ops = frozenset([t.kind for t in tokens])
+        self.seen_op_basenames = frozenset(
+            [opname[: opname.rfind("_")] for opname in self.seen_ops]
+        )
+
+        # Loop over instructions adding custom grammar rules based on
+        # a specific instruction seen.
+
+        if "PyPy" in customize:
+            self.addRule(
+                """
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy       ::= expr expr expr store store store
+              assign2_pypy       ::= expr expr store store
+              """,
+                nop_func,
+            )
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # The order of opname listed is roughly sorted below
+
+            if opname == "LOAD_ASSERT" and "PyPy" in customize:
+                rules_str = """
+                stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "BEFORE_ASYNC_WITH":
+                rules_str = """
+                   stmt            ::= async_with_stmt
+                   stmt            ::= async_with_as_stmt
+                   c_stmt          ::= c_async_with_stmt
+                """
+
+                if self.version < (3, 8):
+                    rules_str += """
+                      stmt                 ::= async_with_stmt SETUP_ASYNC_WITH
+                      c_stmt               ::= c_async_with_stmt SETUP_ASYNC_WITH
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              c_suite_stmts_opt
+                                              POP_BLOCK LOAD_CONST
+                                              async_with_post
+                      async_with_as_stmt   ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                    """
+                else:
+                    rules_str += """
+                      async_with_pre       ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
+                      async_with_post      ::= BEGIN_FINALLY COME_FROM_ASYNC_WITH
+                                               WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               WITH_CLEANUP_FINISH END_FINALLY
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt   ::= async_with_stmt
+                      async_with_stmt     ::= expr
+                                              async_with_pre
+                                              POP_TOP
+                                              c_suite_stmts
+                                              POP_BLOCK
+                                              BEGIN_FINALLY
+                                              WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                              WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
+                                              POP_BLOCK
+                                              BEGIN_FINALLY
+                                              COME_FROM_ASYNC_WITH
+                                              WITH_AWAITABLE
+                                              LOAD_CONST
+                                              YEILD_FROM
+                                              WITH_CLEANUP_FINISH
+                                              END_FINALLY
+
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                    """
+                self.addRule(rules_str, nop_func)
+
+            elif opname == "BUILD_STRING_2":
+                self.addRule(
+                    """
+                      expr                  ::= formatted_value_debug
+                      formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
+                      formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "BUILD_STRING_3":
+                self.addRule(
+                    """
+                      expr                  ::= formatted_value_debug
+                      formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3
+                      formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname in frozenset(
+                (
+                    "CALL_FUNCTION",
+                    "CALL_FUNCTION_EX_KW",
+                    "CALL_FUNCTION_VAR_KW",
+                    "CALL_FUNCTION_VAR",
+                    "CALL_FUNCTION_VAR_KW",
+                )
+            ) or opname.startswith("CALL_FUNCTION_KW"):
+
+                if opname == "CALL_FUNCTION" and token.attr == 1:
+                    rule = """
+                    classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
+                    classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
+                    """
+                    self.addRule(rule, nop_func)
+
+                # self.custom_classfunc_rule(opname, token, customize, tokens[i + 1])
+                # Note: don't add to custom_ops_processed.
+
+            elif opname_base == "CALL_METHOD":
+                # PyPy and Python 3.7+ only - DRY with parse2
+
+                if opname == "CALL_METHOD_KW":
+                    args_kw = token.attr
+                    rules_str = """
+                         expr ::= call_kw_pypy37
+                         pypy_kw_keys ::= LOAD_CONST
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+                    rule = (
+                        "call_kw_pypy37 ::= expr "
+                        + ("expr " * args_kw)
+                        + " pypy_kw_keys "
+                        + opname
+                    )
+                else:
+                    args_pos, args_kw = self.get_pos_kw(token)
+                    # number of apply equiv arguments:
+                    nak = (len(opname_base) - len("CALL_METHOD")) // 3
+                    rule = (
+                        "call ::= expr "
+                        + ("expr " * args_pos)
+                        + ("kwarg " * args_kw)
+                        + "expr " * nak
+                        + opname
+                    )
+
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "CONTINUE":
+                self.addRule("continue ::= CONTINUE", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "CONTINUE_LOOP":
+                self.addRule("continue ::= CONTINUE_LOOP", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_ATTR":
+                self.addRule("delete ::= expr DELETE_ATTR", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_DEREF":
+                self.addRule(
+                    """
+                   stmt           ::= del_deref_stmt
+                   del_deref_stmt ::= DELETE_DEREF
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_SUBSCR":
+                self.addRule(
+                    """
+                    delete ::= delete_subscript
+                    delete_subscript ::= expr expr DELETE_SUBSCR
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "FORMAT_VALUE_ATTR":
+                self.addRule(
+                    """
+                      expr                  ::= formatted_value_debug
+                      formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
+                                                expr FORMAT_VALUE_ATTR
+                      formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
+                      formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_AITER":
+                self.addRule(
+                    """
+                    async_for          ::= GET_AITER _come_froms
+                                           SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
+
+                    async_for_stmt38   ::= expr async_for
+                                           store for_block
+                                           COME_FROM_FINALLY
+                                           END_ASYNC_FOR
+
+                    # FIXME: COME_FROMs after the else_suite or
+                    # END_ASYNC_FOR distinguish which of for / forelse
+                    # is used. Add COME_FROMs and check of add up
+                    # control-flow detection phase.
+                    # async_forelse_stmt38 ::= expr async_for store
+                    # for_block COME_FROM_FINALLY END_ASYNC_FOR
+                    # else_suite
+
+                    async_forelse_stmt38 ::= expr async_for
+                                             store for_block
+                                             COME_FROM_FINALLY
+                                             END_ASYNC_FOR
+                                             else_suite
+                                             POP_TOP COME_FROM
+
+                    stmt                 ::= async_for_stmt38
+                    stmt                 ::= async_forelse_stmt38
+                    stmt                 ::= generator_exp_async
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "GET_ANEXT":
+                self.addRule(
+                    """
+                    stmt ::= genexpr_func_async
+                    stmt ::= BUILD_SET_0 genexpr_func_async
+                             RETURN_VALUE
+                             _come_froms
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "JUMP_IF_NOT_DEBUG":
+                self.addRule(
+                    """
+                    stmt        ::= assert_pypy
+                    stmt        ::= assert2_pypy", nop_func)
+                    assert_pypy ::=  JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM,
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLASSDEREF":
+                # Python 3.4+
+                self.addRule("expr ::= LOAD_CLASSDEREF", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLASSNAME":
+                self.addRule("expr ::= LOAD_CLASSNAME", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "RAISE_VARARGS_0":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt0
+                    last_stmt  ::= raise_stmt0
+                    raise_stmt0 ::= RAISE_VARARGS_0
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_1":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt1
+                    last_stmt  ::= raise_stmt1
+                    raise_stmt1 ::= expr RAISE_VARARGS_1
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_2":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt2
+                    last_stmt  ::= raise_stmt2
+                    raise_stmt2 ::= expr expr RAISE_VARARGS_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "RETURN_VALUE_LAMBDA":
+                self.addRule(
+                    """
+                    return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "SETUP_EXCEPT":
+                self.addRule(
+                    """
+                    try_except     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler opt_come_from_except
+                    c_try_except   ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler opt_come_from_except
+                    stmt           ::= tryelsestmt3
+                    tryelsestmt3   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler COME_FROM else_suite
+                                       opt_come_from_except
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_from_except_clauses
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_froms
+
+                    c_stmt         ::= c_tryelsestmt
+                    c_tryelsestmt  ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler
+                                       come_any_froms else_suitec
+                                       come_from_except_clauses
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "WITH_CLEANUP_START":
+                rules_str = """
+                  stmt        ::= with_null
+                  with_null   ::= with_suffix
+                  with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                self.addRule(rules_str, nop_func)
+
+            # FIXME: reconcile with same code in lambda_custom.py
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                  stmt        ::= with
+                  stmt        ::= with_as
+                  c_stmt      ::= c_with
+
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr SETUP_WITH POP_TOP
+                                  suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+
+                  with_as  ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                  with_as  ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                  with_as  ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                    with      ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   with_suffix
+                    """
+                else:
+                    rules_str += """
+                     # A return at the end of a withas stmt can be this.
+                     # FIXME: should this be a different kind of return?
+                     return      ::= return_expr POP_BLOCK
+                                     ROT_TWO
+                                     BEGIN_FINALLY
+                                     WITH_CLEANUP_START
+                                     WITH_CLEANUP_FINISH
+                                     POP_FINALLY
+                                     RETURN_VALUE
+
+                      with       ::= expr
+                                     SETUP_WITH POP_TOP suite_stmts_opt
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                     with_suffix
+
+
+                      with_as    ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+
+                      with_as    ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK BEGIN_FINALLY COME_FROM_WITH
+                                     with_suffix
+
+                      # with_as ::= expr SETUP_WITH store suite_stmts
+                      #                COME_FROM expr COME_FROM POP_BLOCK ROT_TWO
+                      #                BEGIN_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                      #                POP_FINALLY RETURN_VALUE COME_FROM_WITH
+                      #                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+                      with         ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                       BEGIN_FINALLY COME_FROM_WITH
+                                       with_suffix
+                    """
+                self.addRule(rules_str, nop_func)
+            pass
+
+        return
+
+    def customize_reduce_checks_full38(self, tokens, customize):
+        """
+        Extra tests when a reduction is made in the full grammar.
+
+        Reductions here are extended from those used in the lambda grammar
+        """
+        self.remove_rules_38()
+
+        self.check_reduce["and"] = "AST"
+        self.check_reduce["and_cond"] = "AST"
+        self.check_reduce["and_not"] = "AST"
+        self.check_reduce["annotate_tuple"] = "tokens"
+        self.check_reduce["aug_assign1"] = "AST"
+        self.check_reduce["aug_assign2"] = "AST"
+        self.check_reduce["c_forelsestmt38"] = "AST"
+        self.check_reduce["c_try_except"] = "AST"
+        self.check_reduce["c_tryelsestmt"] = "AST"
+        self.check_reduce["if_and_stmt"] = "AST"
+        self.check_reduce["if_and_elsestmtc"] = "AST"
+        self.check_reduce["if_not_stmtc"] = "AST"
+        self.check_reduce["ifelsestmt"] = "AST"
+        self.check_reduce["ifelsestmtc"] = "AST"
+        self.check_reduce["iflaststmt"] = "AST"
+        self.check_reduce["iflaststmtc"] = "AST"
+        self.check_reduce["ifstmt"] = "AST"
+        self.check_reduce["ifstmtc"] = "AST"
+        self.check_reduce["ifstmts_jump"] = "AST"
+        self.check_reduce["ifstmts_jumpc"] = "AST"
+        self.check_reduce["import_as37"] = "tokens"
+        self.check_reduce["import_from37"] = "AST"
+        self.check_reduce["import_from_as37"] = "tokens"
+        self.check_reduce["lastc_stmt"] = "tokens"
+        self.check_reduce["list_if_not"] = "AST"
+        self.check_reduce["while1elsestmt"] = "tokens"
+        self.check_reduce["while1stmt"] = "tokens"
+        self.check_reduce["whilestmt"] = "tokens"
+        self.check_reduce["not_or"] = "AST"
+        self.check_reduce["or"] = "AST"
+        self.check_reduce["or_cond"] = "tokens"
+        self.check_reduce["testtrue"] = "tokens"
+        self.check_reduce["testfalsec"] = "tokens"
+
+        self.check_reduce["break"] = "tokens"
+        self.check_reduce["forelselaststmt38"] = "AST"
+        self.check_reduce["forelselaststmtc38"] = "AST"
+        self.check_reduce["for38"] = "tokens"
+        self.check_reduce["ifstmt"] = "AST"
+        self.check_reduce["joined_str"] = "AST"
+        self.check_reduce["pop_return"] = "tokens"
+        self.check_reduce["whileTruestmt38"] = "AST"
+        self.check_reduce["whilestmt38"] = "tokens"
+        self.check_reduce["try_elsestmtl38"] = "AST"
+
+        self.reduce_check_table["break"] = break_invalid
+        self.reduce_check_table["if_not_stmtc"] = if_not_stmtc_invalid
+        self.reduce_check_table["for38"] = for38_invalid
+        self.reduce_check_table["c_forelsestmt38"] = forelse38_invalid
+        self.reduce_check_table["forelselaststmt38"] = forelse38_invalid
+        self.reduce_check_table["forelselaststmtc38"] = forelse38_invalid
+        # self.reduce_check_table["joined_str"] = joined_str_check.joined_str_invalid
+        self.reduce_check_table["or"] = or_cond_check_invalid
+        self.reduce_check_table["pop_return"] = pop_return_check
+        self.reduce_check_table["whilestmt38"] = whilestmt38_check
+        self.reduce_check_table["whileTruestmt38"] = whileTruestmt38_check
+
+        # Use update we don't destroy entries from lambda.
+        self.reduce_check_table.update(
+            {
+                # "ifelsestmt": ifelsestmt_ok,
+                "ifstmt": ifstmt,
+                # "import_from37": import_from37_ok,
+            }
+        )
+
+        self.check_reduce["ifelsestmt"] = "AST"
+        self.check_reduce["ifelsestmtc"] = "AST"
+        self.check_reduce["ifstmt"] = "AST"
+        # self.check_reduce["import_from37"] = "AST"
+
+    def customize_grammar_rules38(self, tokens, customize):
+        Python37BaseParser.customize_grammar_rules37(self, tokens, customize)
+        self.customize_reduce_checks_lambda38()
+        self.customize_reduce_checks_full38(tokens, customize)
+
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "CONTINUE",
+                "DELETE",
+                "FORMAT",
+                "GET",
+                "JUMP",
+                "LOAD",
+                "LOOKUP",
+                "MAKE",
+                "RETURN",
+                "RAISE",
+                "SETUP",
+                "UNPACK",
+                "WITH",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get added
+        # unconditionally and the rules are constant. So they need to be done
+        # only once and if we see the opcode a second we don't have to consider
+        # adding more rules.
+        #
+        custom_ops_processed = set()
+
+        # A set of instruction operation names that exist in the token stream.
+        # We use this to customize the grammar that we create.
+        # 2.6-compatible set comprehensions
+
+        # The initial initialization is done in lambda_expr.py
+        self.seen_ops = frozenset([t.kind for t in tokens])
+        self.seen_op_basenames = frozenset(
+            [opname[: opname.rfind("_")] for opname in self.seen_ops]
+        )
+
+        # Loop over instructions adding custom grammar rules based on
+        # a specific instruction seen.
+
+        if "PyPy" in customize:
+            self.addRule(
+                """
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy       ::= expr expr expr store store store
+              assign2_pypy       ::= expr expr store store
+              """,
+                nop_func,
+            )
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # The order of opname listed is roughly sorted below
+
+            if opname == "LOAD_ASSERT" and "PyPy" in customize:
+                rules_str = """
+                stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "BEFORE_ASYNC_WITH":
+                rules_str = """
+                   stmt            ::= async_with_stmt
+                   stmt            ::= async_with_as_stmt
+                   c_stmt          ::= c_async_with_stmt
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                      async_with_pre       ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               SETUP_ASYNC_WITH
+                      stmt                 ::= async_with_stmt SETUP_ASYNC_WITH
+                      c_stmt               ::= c_async_with_stmt SETUP_ASYNC_WITH
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts_opt
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts_opt
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store
+                                               suite_stmts_opt
+                                               POP_BLOCK LOAD_CONST
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              c_suite_stmts_opt
+                                              POP_BLOCK LOAD_CONST
+                                              async_with_post
+                      async_with_as_stmt   ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                      c_async_with_as_stmt ::= expr
+                                              async_with_pre
+                                              store
+                                              suite_stmts_opt
+                                              async_with_post
+                    """
+                else:
+                    rules_str += """
+                      async_with_pre       ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
+                      async_with_post      ::= BEGIN_FINALLY COME_FROM_ASYNC_WITH
+                                               WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                               WITH_CLEANUP_FINISH END_FINALLY
+                      async_with_stmt      ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt    ::= expr
+                                               async_with_pre
+                                               POP_TOP
+                                               c_suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_stmt   ::= async_with_stmt
+                      async_with_stmt     ::= expr
+                                              async_with_pre
+                                              POP_TOP
+                                              c_suite_stmts
+                                              POP_BLOCK
+                                              BEGIN_FINALLY
+                                              WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                              WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
+                                              POP_BLOCK
+                                              BEGIN_FINALLY
+                                              COME_FROM_ASYNC_WITH
+                                              WITH_AWAITABLE
+                                              LOAD_CONST
+                                              YEILD_FROM
+                                              WITH_CLEANUP_FINISH
+                                              END_FINALLY
+
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_TOP POP_BLOCK
+                                               async_with_post
+                      async_with_as_stmt   ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                      c_async_with_as_stmt ::= expr
+                                               async_with_pre
+                                               store suite_stmts
+                                               POP_BLOCK async_with_post
+                    """
+                self.addRule(rules_str, nop_func)
+
+            elif opname in frozenset(
+                (
+                    "CALL_FUNCTION",
+                    "CALL_FUNCTION_EX_KW",
+                    "CALL_FUNCTION_VAR_KW",
+                    "CALL_FUNCTION_VAR",
+                    "CALL_FUNCTION_VAR_KW",
+                )
+            ) or opname.startswith("CALL_FUNCTION_KW"):
+
+                if opname == "CALL_FUNCTION" and token.attr == 1:
+                    rule = """
+                    classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
+                    classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
+                    """
+                    self.addRule(rule, nop_func)
+
+                # self.custom_classfunc_rule(opname, token, customize, tokens[i + 1])
+                # Note: don't add to custom_ops_processed.
+
+            elif opname_base == "CALL_METHOD":
+                # PyPy and Python 3.7+ only - DRY with parse2
+
+                if opname == "CALL_METHOD_KW":
+                    args_kw = token.attr
+                    rules_str = """
+                         expr ::= call_kw_pypy37
+                         pypy_kw_keys ::= LOAD_CONST
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+                    rule = (
+                        "call_kw_pypy37 ::= expr "
+                        + ("expr " * args_kw)
+                        + " pypy_kw_keys "
+                        + opname
+                    )
+                else:
+                    args_pos, args_kw = self.get_pos_kw(token)
+                    # number of apply equiv arguments:
+                    nak = (len(opname_base) - len("CALL_METHOD")) // 3
+                    rule = (
+                        "call ::= expr "
+                        + ("expr " * args_pos)
+                        + ("kwarg " * args_kw)
+                        + "expr " * nak
+                        + opname
+                    )
+
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "CONTINUE":
+                self.addRule("continue ::= CONTINUE", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "CONTINUE_LOOP":
+                self.addRule("continue ::= CONTINUE_LOOP", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_ATTR":
+                self.addRule("delete ::= expr DELETE_ATTR", nop_func)
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_DEREF":
+                self.addRule(
+                    """
+                   stmt           ::= del_deref_stmt
+                   del_deref_stmt ::= DELETE_DEREF
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "DELETE_SUBSCR":
+                self.addRule(
+                    """
+                    delete ::= delete_subscript
+                    delete_subscript ::= expr expr DELETE_SUBSCR
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_AITER":
+                self.addRule(
+                    """
+                    stmt ::= generator_exp_async
+                    stmt ::= genexpr_func_async
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "GET_ANEXT":
+                self.addRule(
+                    """
+                    stmt ::= BUILD_SET_0 genexpr_func_async
+                             RETURN_VALUE
+                             bb_doms_end_opt
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "JUMP_IF_NOT_DEBUG":
+                self.addRule(
+                    """
+                    stmt        ::= assert_pypy
+                    stmt        ::= assert2_pypy", nop_func)
+                    assert_pypy ::=  JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM
+                    assert2_pypy ::= JUMP_IF_NOT_DEBUG expr POP_JUMP_IF_TRUE
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM,
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLASSDEREF":
+                # Python 3.4+
+                self.addRule("expr ::= LOAD_CLASSDEREF", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLASSNAME":
+                self.addRule("expr ::= LOAD_CLASSNAME", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "RAISE_VARARGS_0":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt0
+                    last_stmt  ::= raise_stmt0
+                    raise_stmt0 ::= RAISE_VARARGS_0
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_1":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt1
+                    last_stmt  ::= raise_stmt1
+                    raise_stmt1 ::= expr RAISE_VARARGS_1
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "RAISE_VARARGS_2":
+                self.addRule(
+                    """
+                    stmt        ::= raise_stmt2
+                    last_stmt  ::= raise_stmt2
+                    raise_stmt2 ::= expr expr RAISE_VARARGS_2
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "RETURN_VALUE_LAMBDA":
+                self.addRule(
+                    """
+                    return_expr_lambda ::= return_expr RETURN_VALUE_LAMBDA
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+            elif opname == "SETUP_EXCEPT":
+                self.addRule(
+                    """
+                    try_except     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler opt_come_from_except
+                    c_try_except   ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler opt_come_from_except
+                    stmt           ::= tryelsestmt3
+                    tryelsestmt3   ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler COME_FROM else_suite
+                                       opt_come_from_except
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_from_except_clauses
+
+                    tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                       except_handler else_suite come_froms
+
+                    c_stmt         ::= c_tryelsestmt
+                    c_tryelsestmt  ::= SETUP_EXCEPT c_suite_stmts POP_BLOCK
+                                       c_except_handler
+                                       come_any_froms else_suitec
+                                       come_from_except_clauses
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "WITH_CLEANUP_START":
+                rules_str = """
+                  stmt        ::= with_null
+                  with_null   ::= with_suffix
+                  with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                self.addRule(rules_str, nop_func)
+
+            # FIXME: reconcile with same code in lambda_custom.py
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                  stmt        ::= with
+                  stmt        ::= with_as
+                  c_stmt      ::= c_with
+
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+                  c_with      ::= expr SETUP_WITH POP_TOP
+                                  c_suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr SETUP_WITH POP_TOP
+                                  suite_stmts_opt
+                                  COME_FROM_WITH
+                                  with_suffix
+
+                  with_as  ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                  with_as     ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+
+                  with        ::= expr
+                                  SETUP_WITH POP_TOP suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                  with_as  ::= expr
+                                  SETUP_WITH store suite_stmts_opt
+                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                  with_suffix
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                    with      ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   with_suffix
+                    """
+                else:
+                    rules_str += """
+                     # A return at the end of a withas stmt can be this.
+                     # FIXME: should this be a different kind of return?
+                     return      ::= return_expr POP_BLOCK
+                                     ROT_TWO
+                                     BEGIN_FINALLY
+                                     WITH_CLEANUP_START
+                                     WITH_CLEANUP_FINISH
+                                     POP_FINALLY
+                                     RETURN_VALUE
+
+                      with       ::= expr
+                                     SETUP_WITH POP_TOP suite_stmts_opt
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                     with_suffix
+
+
+                      with_as    ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+
+                      with_as    ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK BEGIN_FINALLY COME_FROM_WITH
+                                     with_suffix
+
+                      # with_as   ::= expr SETUP_WITH store suite_stmts
+                      #               COME_FROM expr COME_FROM POP_BLOCK ROT_TWO
+                      #               BEGIN_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                      #               POP_FINALLY RETURN_VALUE COME_FROM_WITH
+                      #               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+                      with         ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                       BEGIN_FINALLY COME_FROM_WITH
+                                       with_suffix
+                    """
+                self.addRule(rules_str, nop_func)
+            pass
+
+        return

python/py/Lib/site-packages/decompyle3/parsers/p38pypy/heads.py

@@ -0,0 +1,53 @@
+"""
+All of the specific kinds of canned parsers for Python 3.8
+
+These are derived from "compile-modes" but we have others that
+can be used to parse common part of a larger grammar.
+
+For example:
+* a basic-block expression (no branching)
+* an unadorned expression (no POP_TOP needed afterwards)
+* A non-compound statement
+"""
+from decompyle3.parsers.p38pypy.full import Python38PyPyParser
+from decompyle3.parsers.p38pypy.lambda_expr import Python38PyPyLambdaParser
+from decompyle3.parsers.parse_heads import (  # FIXME: add; PythonParserSimpleStmt; PythonParserStmt
+    PythonParserEval,
+    PythonParserExec,
+    PythonParserExpr,
+    PythonParserLambda,
+    PythonParserSingle,
+)
+
+# Make sure to list Python38... classes first so we prefer to inherit methods from that first.
+# In particular methods like reduce_is_invalid() need to come from there rather than
+# a more generic place.
+
+
+class Python38PyPyParserEval(Python38PyPyLambdaParser, PythonParserEval):
+    def __init__(self, debug_parser):
+        PythonParserEval.__init__(self, debug_parser)
+
+
+class Python38PyPyParserExec(Python38PyPyParser, PythonParserExec):
+    def __init__(self, debug_parser):
+        PythonParserExec.__init__(self, debug_parser)
+
+
+class Python38PyPyParserExpr(Python38PyPyParser, PythonParserExpr):
+    def __init__(self, debug_parser):
+        PythonParserExpr.__init__(self, debug_parser)
+
+
+# Understand: Python38LambdaParser has to come before PythonParserLambda or we get a
+# MRO failure
+class Python38PyPyParserLambda(Python38PyPyLambdaParser, PythonParserLambda):
+    def __init__(self, debug_parser):
+        PythonParserLambda.__init__(self, debug_parser)
+
+
+# These classes are here just to get parser doc-strings for the
+# various classes inherited properly and start_symbols set properly.
+class Python38PyPyParserSingle(Python38PyPyParser, PythonParserSingle):
+    def __init__(self, debug_parser):
+        PythonParserSingle.__init__(self, debug_parser)

python/py/Lib/site-packages/decompyle3/parsers/p38pypy/lambda_custom.py

@@ -0,0 +1,774 @@
+#  Copyright (c) 2020-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Grammar Customization rules for Python 3.8's Lambda expression grammar.
+"""
+
+from decompyle3.parsers.p37.base import Python37BaseParser
+from decompyle3.parsers.p38pypy.base import Python38PyPyBaseParser
+from decompyle3.parsers.parse_heads import nop_func
+
+
+class Python38PyPyLambdaCustom(Python38PyPyBaseParser):
+    def __init__(self):
+        self.new_rules = set()
+        self.customized = {}
+
+    def remove_rules_pypy38(self):
+        self.remove_rules(
+            """
+            """
+        )
+
+    def customize_grammar_rules_lambda38(self, tokens, customize):
+        Python38PyPyBaseParser.customize_grammar_rules38(self, tokens, customize)
+
+        # self.remove_rules_pypy38()
+        self.check_reduce["call_kw"] = "AST"
+
+        # For a rough break out on the first word. This may
+        # include instructions that don't need customization,
+        # but we'll do a finer check after the rough breakout.
+        customize_instruction_basenames = frozenset(
+            (
+                "BEFORE",
+                "BUILD",
+                "CALL",
+                "DICT",
+                "GET",
+                "FORMAT",
+                "LIST",
+                "LOAD",
+                "MAKE",
+                "SETUP",
+                "UNPACK",
+            )
+        )
+
+        # Opcode names in the custom_ops_processed set have rules that get added
+        # unconditionally and the rules are constant. So they need to be done
+        # only once and if we see the opcode a second we don't have to consider
+        # adding more rules.
+        #
+        custom_ops_processed = frozenset()
+
+        # A set of instruction operation names that exist in the token stream.
+        # We use this customize the grammar that we create.
+        # 2.6-compatible set comprehensions
+        self.seen_ops = frozenset([t.kind for t in tokens])
+        self.seen_op_basenames = frozenset(
+            [opname[: opname.rfind("_")] for opname in self.seen_ops]
+        )
+
+        custom_ops_processed = {"DICT_MERGE"}
+
+        # Loop over instructions adding custom grammar rules based on
+        # a specific instruction seen.
+
+        if "PyPy" in customize:
+            self.addRule(
+                """
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy       ::= expr expr expr store store store
+              assign2_pypy       ::= expr expr store store
+              """,
+                nop_func,
+            )
+
+        n = len(tokens)
+
+        # Determine if we have an iteration CALL_FUNCTION_1.
+        has_get_iter_call_function1 = False
+        for i, token in enumerate(tokens):
+            if (
+                token == "GET_ITER"
+                and i < n - 2
+                and self.call_fn_name(tokens[i + 1]) == "CALL_FUNCTION_1"
+            ):
+                has_get_iter_call_function1 = True
+
+        for i, token in enumerate(tokens):
+            opname = token.kind
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            opname_base = opname[: opname.rfind("_")]
+
+            # Do a quick breakout before testing potentially
+            # each of the dozen or so instruction in if elif.
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_ops_processed
+            ):
+                continue
+
+            if opname == "BEFORE_ASYNC_WITH":
+                rules_str = """
+                  async_with_post    ::= COME_FROM_ASYNC_WITH
+                                         WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                         WITH_CLEANUP_FINISH END_FINALLY
+
+                  stmt               ::= async_with_as_stmt
+                  async_with_as_stmt ::= expr
+                                         async_with_pre
+                                         store
+                                         suite_stmts_opt
+                                         POP_BLOCK LOAD_CONST
+                                         async_with_post
+
+                  async_with_stmt     ::= expr
+                                          async_with_pre
+                                          POP_TOP
+                                          c_suite_stmts
+                                          POP_BLOCK
+                                          BEGIN_FINALLY
+                                          WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM
+                                          WITH_CLEANUP_FINISH POP_FINALLY POP_TOP JUMP_FORWARD
+                                          POP_BLOCK
+                                          BEGIN_FINALLY
+                                          COME_FROM_ASYNC_WITH
+                                          WITH_CLEANUP_START
+                                          GET_AWAITABLE
+                                          LOAD_CONST
+                                          YIELD_FROM
+                                          WITH_CLEANUP_FINISH
+                                          END_FINALLY
+
+
+                 async_with_stmt     ::= expr
+                                         async_with_pre
+                                         POP_TOP
+                                         suite_stmts_opt
+                                         POP_BLOCK LOAD_CONST
+                                         async_with_post
+                 async_with_stmt     ::= expr
+                                         async_with_pre
+                                         POP_TOP
+                                         suite_stmts_opt
+                                         async_with_post
+                """
+                self.addRule(rules_str, nop_func)
+
+            elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
+                if opname == "BUILD_CONST_DICT":
+                    rule = f"""
+                            add_consts          ::= ADD_VALUE*
+                            const_list          ::= COLLECTION_START add_consts {opname}
+                            dict                ::= const_list
+                            expr                ::= dict
+                        """
+                else:
+                    rule = f"""
+                            add_consts          ::= ADD_VALUE*
+                            const_list          ::= COLLECTION_START add_consts {opname}
+                            expr                ::= const_list
+                        """
+                self.addRule(rule, nop_func)
+            elif opname_base in (
+                "BUILD_LIST",
+                "BUILD_SET",
+                "BUILD_SET_UNPACK",
+                "BUILD_TUPLE",
+                "BUILD_TUPLE_UNPACK",
+            ):
+                v = token.attr
+
+                is_LOAD_CLOSURE = False
+                if opname_base == "BUILD_TUPLE":
+                    # If is part of a "load_closure", then it is not part of a
+                    # "list".
+                    is_LOAD_CLOSURE = True
+                    for j in range(v):
+                        if tokens[i - j - 1].kind != "LOAD_CLOSURE":
+                            is_LOAD_CLOSURE = False
+                            break
+                    if is_LOAD_CLOSURE:
+                        rule = "load_closure ::= %s%s" % (("LOAD_CLOSURE " * v), opname)
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+
+                elif opname_base == "BUILD_LIST":
+                    v = token.attr
+                    if v == 0:
+                        rule_str = """
+                           list        ::= BUILD_LIST_0
+                           list_unpack ::= BUILD_LIST_0 expr LIST_EXTEND
+                           list        ::= list_unpack
+                        """
+                        self.add_unique_doc_rules(rule_str, customize)
+
+                elif opname == "BUILD_TUPLE_UNPACK_WITH_CALL":
+                    # FIXME: should this be parameterized by EX value?
+                    self.addRule(
+                        """expr        ::= call_ex_kw3
+                           call_ex_kw3 ::= expr
+                                           build_tuple_unpack_with_call
+                                           expr
+                                           CALL_FUNCTION_EX_KW
+                        """,
+                        nop_func,
+                    )
+
+                if not is_LOAD_CLOSURE or v == 0:
+                    # We do this complicated test to speed up parsing of
+                    # pathelogically long literals, especially those over 1024.
+                    build_count = token.attr
+                    thousands = build_count // 1024
+                    thirty32s = (build_count // 32) % 32
+                    if thirty32s > 0:
+                        rule = "expr32 ::=%s" % (" expr" * 32)
+                        self.add_unique_rule(rule, opname_base, build_count, customize)
+                        pass
+                    if thousands > 0:
+                        self.add_unique_rule(
+                            "expr1024 ::=%s" % (" expr32" * 32),
+                            opname_base,
+                            build_count,
+                            customize,
+                        )
+                        pass
+                    collection = opname_base[opname_base.find("_") + 1 :].lower()
+                    rule = (
+                        ("%s ::= " % collection)
+                        + "expr1024 " * thousands
+                        + "expr32 " * thirty32s
+                        + "expr " * (build_count % 32)
+                        + opname
+                    )
+                    self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
+                    continue
+                continue
+
+            elif opname.startswith("BUILD_STRING"):
+                v = token.attr
+                rules_str = """
+                    expr                 ::= joined_str
+                    joined_str           ::= %sBUILD_STRING_%d
+                """ % (
+                    "expr " * v,
+                    v,
+                )
+                self.add_unique_doc_rules(rules_str, customize)
+                if "FORMAT_VALUE_ATTR" in self.seen_ops:
+                    rules_str = """
+                      formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
+                      expr                 ::= formatted_value_attr
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+            elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
+                v = token.attr
+                rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+            elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
+                v = token.attr
+                rule = (
+                    "build_tuple_unpack_with_call ::= "
+                    + "expr1024 " * int(v // 1024)
+                    + "expr32 " * int((v // 32) % 32)
+                    + "expr " * (v % 32)
+                    + opname
+                )
+                self.addRule(rule, nop_func)
+                rule = "starred ::= %s %s" % ("expr " * v, opname)
+                self.addRule(rule, nop_func)
+
+            elif opname == "FORMAT_VALUE":
+                rules_str = """
+                    expr              ::= formatted_value1
+                    formatted_value1  ::= expr FORMAT_VALUE
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+            elif opname == "FORMAT_VALUE_ATTR":
+                rules_str = """
+                expr              ::= formatted_value2
+                formatted_value2  ::= expr expr FORMAT_VALUE_ATTR
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "GET_AITER":
+                self.add_unique_doc_rules("get_aiter ::= expr GET_AITER", customize)
+
+                if not {"MAKE_FUNCTION_0", "MAKE_FUNCTION_CLOSURE"} in self.seen_ops:
+                    self.addRule(
+                        """
+                        expr                ::= dict_comp_async
+                        expr                ::= generator_exp_async
+                        expr                ::= list_comp_async
+
+                        dict_comp_async     ::= LOAD_DICTCOMP
+                                                LOAD_STR
+                                                MAKE_FUNCTION_0
+                                                get_aiter
+                                                CALL_FUNCTION_1
+
+                        dict_comp_async     ::= BUILD_MAP_0 LOAD_ARG
+                                                dict_comp_async
+
+                        generator_exp_async ::= load_genexpr LOAD_STR MAKE_FUNCTION_0
+                                                get_aiter CALL_FUNCTION_1
+
+                        list_comp_async     ::= LOAD_LISTCOMP LOAD_STR MAKE_FUNCTION_0
+                                                get_aiter CALL_FUNCTION_1
+                                                await
+
+                        list_comp_async     ::= LOAD_CLOSURE
+                                                BUILD_TUPLE_1
+                                                LOAD_LISTCOMP
+                                                LOAD_STR MAKE_FUNCTION_CLOSURE
+                                                get_aiter CALL_FUNCTION_1
+                                                await
+
+                        set_comp_async       ::= LOAD_SETCOMP
+                                                 LOAD_STR
+                                                 MAKE_FUNCTION_0
+                                                 get_aiter
+                                                 CALL_FUNCTION_1
+
+                        set_comp_async       ::= LOAD_CLOSURE
+                                                 BUILD_TUPLE_1
+                                                 LOAD_SETCOMP
+                                                 LOAD_STR MAKE_FUNCTION_CLOSURE
+                                                 get_aiter CALL_FUNCTION_1
+                                                 await
+                       """,
+                        nop_func,
+                    )
+                    custom_ops_processed.add(opname)
+
+                self.addRule(
+                    """
+                    dict_comp_async      ::= BUILD_MAP_0 LOAD_ARG
+                                             dict_comp_async
+
+                    expr                 ::= dict_comp_async
+                    expr                 ::= generator_exp_async
+                    expr                 ::= list_comp_async
+                    expr                 ::= set_comp_async
+
+                    func_async_middle   ::= POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT
+                                            DUP_TOP LOAD_GLOBAL COMPARE_OP
+                                            POP_JUMP_IF_TRUE
+                                            END_FINALLY _come_froms
+
+                    # async_iter         ::= block_break SETUP_EXCEPT GET_ANEXT
+                                             LOAD_CONST YIELD_FROM
+
+                    get_aiter            ::= expr GET_AITER
+
+                    list_afor            ::= get_aiter list_afor2
+
+                    return_expr_lambda   ::= list_comp_async
+
+
+                    list_afor2           ::= async_iter store list_iter
+                                             JUMP_LOOP COME_FROM_EXCEPT
+                                             END_ASYNC_FOR
+
+                    list_iter            ::= list_afor
+
+
+                    set_afor             ::= get_aiter set_afor2
+                    set_iter             ::= set_afor
+
+                    set_comp_async       ::= BUILD_SET_0 LOAD_ARG
+                                             set_comp_async
+
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_ANEXT":
+                self.addRule(
+                    """
+                    expr                 ::= genexpr_func_async
+                    expr                 ::= BUILD_MAP_0 genexpr_func_async
+                    expr                 ::= list_comp_async
+
+                    dict_comp_async      ::= BUILD_MAP_0 genexpr_func_async
+
+                    async_iter           ::= _come_froms
+                                              SETUP_EXCEPT
+                                              GET_ANEXT
+                                              LOAD_CONST
+                                              YIELD_FROM
+                                              POP_BLOCK
+
+                    func_async_prefix    ::= _come_froms SETUP_EXCEPT GET_ANEXT
+                                              LOAD_CONST YIELD_FROM
+
+                    genexpr_func_async   ::= LOAD_ARG async_iter
+                                             store
+                                             comp_iter
+                                             JUMP_LOOP
+                                             COME_FROM_EXCEPT
+                                             END_ASYNC_FOR
+
+                    genexpr_func_async   ::= LOAD_ARG func_async_prefix
+                                             store func_async_middle comp_iter
+                                             JUMP_LOOP COME_FROM
+                                             POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
+
+                    list_comp_async      ::= LOAD_ARG BUILD_LIST_FROM_ARG list_afor2
+                    list_comp_async      ::= BUILD_LIST_0 LOAD_ARG list_afor2
+
+                    set_afor2            ::= async_iter
+                                             store
+                                             set_iter
+                                             JUMP_LOOP
+                                             COME_FROM_FINALLY
+                                             END_ASYNC_FOR
+
+                    set_afor2            ::= expr_or_arg
+                                             set_iter_async
+
+                    set_comp_async       ::= BUILD_SET_0 set_afor2
+
+                    set_iter_async       ::= async_iter
+                                             store
+                                             set_iter
+                                             JUMP_LOOP
+                                             _come_froms
+                                             END_ASYNC_FOR
+
+                    return_expr_lambda   ::= genexpr_func_async
+                                             LOAD_CONST RETURN_VALUE
+                                             RETURN_VALUE_LAMBDA
+
+                    return_expr_lambda   ::= BUILD_SET_0 genexpr_func_async
+                                             RETURN_VALUE_LAMBDA
+                   """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "GET_AWAITABLE":
+                rule_str = """
+                    await      ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
+                    await_expr ::= expr await
+                    expr       ::= await_expr
+                """
+                self.add_unique_doc_rules(rule_str, customize)
+
+            elif opname == "GET_ITER":
+                self.addRule(
+                    """
+                    expr      ::= get_iter
+                    get_iter  ::= expr GET_ITER
+                    """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_ASSERT":
+                if "PyPy" in customize:
+                    rules_str = """
+                    stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+
+            elif opname == "LOAD_ATTR":
+                self.addRule(
+                    """
+                  expr      ::= attribute
+                  attribute ::= expr LOAD_ATTR
+                  """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_CLOSURE":
+                self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
+
+            elif opname == "LOAD_DICTCOMP":
+                if has_get_iter_call_function1:
+                    rule_pat = "dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1"
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    pass
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_GENEXPR":
+                self.addRule("load_genexpr ::= LOAD_GENEXPR", nop_func)
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_LISTCOMP":
+                self.add_unique_rule(
+                    "expr ::= list_comp", opname, token.attr, customize
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "LOAD_NAME":
+                if (
+                    token.attr == "__annotations__"
+                    and "SETUP_ANNOTATIONS" in self.seen_ops
+                ):
+                    token.kind = "LOAD_ANNOTATION"
+                    self.addRule(
+                        """
+                        stmt       ::= SETUP_ANNOTATIONS
+                        stmt       ::= ann_assign
+                        ann_assign ::= expr LOAD_ANNOTATION LOAD_STR STORE_SUBSCR
+                        """,
+                        nop_func,
+                    )
+                    pass
+            elif opname == "LOAD_SETCOMP":
+                # Should this be generalized and put under MAKE_FUNCTION?
+                if has_get_iter_call_function1:
+                    self.addRule("expr ::= set_comp", nop_func)
+                    rule_pat = "set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 get_iter CALL_FUNCTION_1"
+                    self.add_make_function_rule(rule_pat, opname, token.attr, customize)
+                    pass
+                custom_ops_processed.add(opname)
+            elif opname == "LOOKUP_METHOD":
+                # A PyPy speciality - DRY with parse3
+                self.addRule(
+                    """
+                             expr      ::= attribute
+                             attribute ::= expr LOOKUP_METHOD
+                             """,
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
+
+            elif opname == "MAKE_FUNCTION_CLOSURE":
+                if "LOAD_DICTCOMP" in self.seen_ops:
+                    # Is there something general going on here?
+                    rule = """
+                       dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
+                                     MAKE_FUNCTION_CLOSURE expr
+                                     GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+                elif "LOAD_SETCOMP" in self.seen_ops:
+                    rule = """
+                       set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
+                                    MAKE_FUNCTION_CLOSURE expr
+                                    GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+
+            elif opname == "MAKE_FUNCTION_CLOSURE_POS":
+
+                args_pos, args_kw, annotate_args, closure = token.attr
+                stack_count = args_pos + args_kw + annotate_args
+
+                if closure:
+
+                    if args_pos:
+                        # This was seen ion line 447 of Python 3.8
+                        # This is needed for Python 3.8 line 447 of site-packages/nltk/tgrep.py
+                        # line 447:
+                        #    lambda i: lambda n, m=None, l=None: ...
+                        # which has
+                        #  L. 447         0  LOAD_CONST               (None, None)
+                        #                 2  LOAD_CLOSURE             'i'
+                        #                 4  LOAD_CLOSURE             'predicate'
+                        #                 6  BUILD_TUPLE_2         2
+                        #                 8  LOAD_LAMBDA              '<code_object <lambda>>'
+                        #                10  LOAD_STR                 '_tgrep_relation_action.<locals>.<lambda>.<locals>.<lambda>'
+                        #                12  MAKE_FUNCTION_CLOSURE_POS   'default, closure'
+                        # FIXME: Possibly we need to generalize for more nested lambda's of lambda's?
+                        rule = """
+                             expr        ::= lambda_body
+                             lambda_body ::= %s%s%s%s
+                             """ % (
+                            "expr " * stack_count,
+                            "load_closure " * closure,
+                            "BUILD_TUPLE_2 LOAD_LAMBDA LOAD_STR ",
+                            opname,
+                        )
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = """
+                             expr        ::= lambda_body
+                             lambda_body ::= %s%s%s%s
+                             """ % (
+                            "expr " * stack_count,
+                            "load_closure " * closure,
+                            "LOAD_LAMBDA LOAD_STR ",
+                            opname,
+                        )
+
+                    else:
+                        rule = """
+                             expr        ::= lambda_body
+                             lambda_body ::= %s%s%s""" % (
+                            "load_closure " * closure,
+                            "LOAD_LAMBDA LOAD_STR ",
+                            opname,
+                        )
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
+                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+
+                # Removes POP_BLOCK LOAD_CONST from 3.6-
+                with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                if self.version < (3, 8):
+                    rules_str += """
+                    with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                    """
+                else:
+                    rules_str += """
+                    with        ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   BEGIN_FINALLY COME_FROM_WITH
+                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                                   END_FINALLY
+                    """
+                self.addRule(rules_str, nop_func)
+                pass
+            pass
+
+    def custom_classfunc_rule(self, opname, token, customize, next_token):
+
+        args_pos, args_kw = self.get_pos_kw(token)
+
+        # Additional exprs for * and ** args:
+        #  0 if neither
+        #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
+        #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
+        # Yes, this computation based on instruction name is a little bit hoaky.
+        nak = (len(opname) - len("CALL_FUNCTION")) // 3
+        uniq_param = args_kw + args_pos
+
+        if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
+            rule_str = """
+                await      ::= GET_AWAITABLE LOAD_CONST YIELD_FROM
+                await_expr ::= expr await
+                expr       ::= await_expr
+            """
+            self.add_unique_doc_rules(rule_str, customize)
+            rule = (
+                "async_call ::= expr "
+                + ("expr " * args_pos)
+                + ("kwarg " * args_kw)
+                + "expr " * nak
+                + token.kind
+                + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
+            )
+            self.add_unique_rule(rule, token.kind, uniq_param, customize)
+            self.add_unique_rule(
+                "expr ::= async_call", token.kind, uniq_param, customize
+            )
+
+        if opname.startswith("CALL_FUNCTION_KW"):
+            self.addRule("expr ::= call_kw36", nop_func)
+            values = "expr " * token.attr
+            rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
+            self.add_unique_rule(rule, token.kind, token.attr, customize)
+        elif opname == "CALL_FUNCTION_EX_KW":
+            # Note that we don't add to customize token.kind here. Instead, the non-terminal
+            # names call_ex_kw... are is in semantic actions.
+            self.addRule(
+                """expr        ::= call_ex_kw4
+                                   call_ex_kw4 ::= expr
+                                   expr
+                                   expr
+                                   CALL_FUNCTION_EX_KW
+                """,
+                nop_func,
+            )
+            if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                self.addRule(
+                    """expr        ::= call_ex_kw
+                       call_ex_kw  ::= expr expr build_map_unpack_with_call
+                                       CALL_FUNCTION_EX_KW
+                    """,
+                    nop_func,
+                )
+            if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                # FIXME: should this be parameterized by EX value?
+                self.addRule(
+                    """expr        ::= call_ex_kw3
+                                       call_ex_kw3 ::= expr
+                                       build_tuple_unpack_with_call
+                                       expr
+                                       CALL_FUNCTION_EX_KW
+                    """,
+                    nop_func,
+                )
+                if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                    # FIXME: should this be parameterized by EX value?
+                    self.addRule(
+                        """expr        ::= call_ex_kw2
+                                           call_ex_kw2 ::= expr
+                                           build_tuple_unpack_with_call
+                                           build_map_unpack_with_call
+                                           CALL_FUNCTION_EX_KW
+                        """,
+                        nop_func,
+                    )
+
+        elif opname == "CALL_FUNCTION_EX":
+            self.addRule(
+                """
+                expr        ::= call_ex
+                starred     ::= expr
+                call_ex     ::= expr starred CALL_FUNCTION_EX
+                """,
+                nop_func,
+            )
+            if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
+                self.addRule(
+                    """
+                        expr        ::= call_ex_kw
+                        call_ex_kw  ::= expr expr
+                                        build_map_unpack_with_call CALL_FUNCTION_EX
+                        """,
+                    nop_func,
+                )
+            if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
+                self.addRule(
+                    """
+                        expr        ::= call_ex_kw3
+                        call_ex_kw3 ::= expr
+                                        build_tuple_unpack_with_call
+                                        %s
+                                        CALL_FUNCTION_EX
+                        """
+                    % "expr "
+                    * token.attr,
+                    nop_func,
+                )
+                pass
+
+            # FIXME: Is this right?
+            self.addRule(
+                """
+                        expr        ::= call_ex_kw4
+                        call_ex_kw4 ::= expr
+                                        expr
+                                        expr
+                                        CALL_FUNCTION_EX
+                        """,
+                nop_func,
+            )
+            pass
+        else:
+            Python37BaseParser.custom_classfunc_rule(
+                self, opname, token, customize, next_token
+            )

python/py/Lib/site-packages/decompyle3/parsers/p38pypy/lambda_expr.py

@@ -0,0 +1,100 @@
+#  Copyright (c) 2020-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Differences over Python 3.7 for Python 3.8 in the Earley-algorithm lambda grammar
+"""
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from decompyle3.parsers.p37.lambda_expr import Python37LambdaParser
+from decompyle3.parsers.p38pypy.lambda_custom import Python38PyPyLambdaCustom
+from decompyle3.parsers.parse_heads import PythonBaseParser, PythonParserLambda
+
+
+class Python38PyPyLambdaParser(
+    Python38PyPyLambdaCustom, Python37LambdaParser, PythonParserLambda
+):
+    def p_38walrus(self, args):
+        """
+        # named_expr is also known as the "walrus op" :=
+        expr              ::= named_expr
+        named_expr        ::= expr DUP_TOP store
+        """
+
+    def p_lambda_start(self, args):
+        """
+        return_expr_lambda ::= genexpr_func LOAD_CONST RETURN_VALUE_LAMBDA
+        """
+
+    def p_pypy38_comprehension(self, args):
+        """
+        list_comp  ::= LOAD_ARG
+                       BUILD_LIST_FROM_ARG
+                       COME_FROM FOR_ITER
+                       store lc_body
+                       JUMP_LOOP _come_froms
+
+        list_afor2 ::= async_iter store list_iter
+                       JUMP_LOOP COME_FROM_EXCEPT
+                       END_ASYNC_FOR
+
+
+        lc_body ::= expr LIST_APPEND
+        """
+
+    def p_expr38(self, args):
+        """
+        expr ::= if_exp_compare38
+
+        if_exp_compare38 ::= or_in_ifexp jump_if_false_cf expr jf_cfs expr come_froms
+
+        list_iter        ::= list_if_not38
+        list_if_not38    ::= expr pjump_ift expr pjump_ift _come_froms list_iter
+                             come_from_opt
+
+        or_in_ifexp      ::= expr_pjit expr
+        or_in_ifexp      ::= or_in_ifexp POP_JUMP_IF_TRUE expr
+        """
+
+    def __init__(
+        self,
+        start_symbol: str = "lambda_start",
+        debug_parser: dict = PARSER_DEFAULT_DEBUG,
+    ):
+        PythonParserLambda.__init__(
+            self, debug_parser=debug_parser, start_symbol=start_symbol
+        )
+        PythonBaseParser.__init__(
+            self, start_symbol=start_symbol, debug_parser=debug_parser
+        )
+        Python38PyPyLambdaCustom.__init__(self)
+
+    def customize_grammar_rules(self, tokens, customize):
+        self.customize_grammar_rules_lambda38(tokens, customize)
+
+
+if __name__ == "__main__":
+    # Check grammar
+    from decompyle3.parsers.dump import dump_and_check
+
+    p = Python38PyPyLambdaParser()
+    modified_tokens = set(
+        """JUMP_LOOP CONTINUE RETURN_END_IF COME_FROM
+           LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+           LAMBDA_MARKER RETURN_LAST
+        """.split()
+    )
+
+    dump_and_check(p, (3, 8), modified_tokens)

python/py/Lib/site-packages/decompyle3/parsers/parse_heads.py

@@ -0,0 +1,427 @@
+#  Copyright (c) 2022-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Here we have the top-level parse grammar types with their rules and the start symbols
+for them.
+
+Specific Python versions such as for Python 3.10 subclass these and
+add in grammar rules that are custom to them.
+
+However at the top-level they are all the same and share the same start symbol
+and start-symbol grammar rule.
+
+"""
+# The below adds a special "start" rule for the kind of thing that we want to
+# decompile
+
+from typing import Union
+
+from spark_parser import GenericASTBuilder
+
+from decompyle3.parsers.treenode import SyntaxTree
+
+
+def nop_func(self, args):
+    return None
+
+
+class ParserError(Exception):
+    def __init__(self, token, offset: int, debug: bool):
+        self.token = token
+        self.offset = offset
+        self.debug = debug
+
+    def __str__(self) -> str:
+        return "Parse error at or near `%r' instruction at offset %s\n" % (
+            self.token,
+            self.offset,
+        )
+
+
+class PythonBaseParser(GenericASTBuilder):
+    def __init__(self, debug_parser, start_symbol, is_lambda=False):
+
+        # Note: order of debug_parser, and start_symbol is reverse from above.
+        # This is because (at least at one time), start_symbol can be defaulted
+        # in the setup, while debug_parser could have been but wasn't.
+        GenericASTBuilder.__init__(self, SyntaxTree, start_symbol, debug_parser)
+
+        # FIXME: customize per python parser version
+
+        # These are the non-terminals we should collect into a list.
+        # For example instead of:
+        #   stmts -> stmts stmt -> stmts stmt stmt ...
+        # collect as stmts -> stmt stmt ...
+        nt_list = [
+            "and_parts",
+            "attributes",
+            "add_consts",
+            "dicts_unmap",
+            "doms_end",
+            "exprs",
+            "kvlist",
+            "kwargs",
+            "lists",
+            "or_parts",
+            "stmts",
+        ]
+        self.collect = frozenset(nt_list)
+
+        # For these items we need to keep the 1st epslion reduction since
+        # the nonterminal name is used in a semantic action.
+        self.keep_epsilon = frozenset(("kvlist_n", "kvlist"))
+
+        # ??? Do we need a debug option to skip eliding singleton reductions?
+        # Time will tell if it if useful in debugging
+
+        # FIXME: optional_nt is a misnomer. It's really about there being a
+        # singleton reduction that we can simplify. It also happens to be optional
+        # in its other derivation
+        self.optional_nt |= frozenset(("suite_stmts", "c_stmts_opt", "stmt", "sstmt"))
+
+        # Reduce singleton reductions in these nonterminals:
+        # FIXME: would love to do sstmts, stmts and
+        # so on but that would require major changes to the
+        # semantic actions
+        self.singleton = frozenset(("str", "store", "inplace_op"))
+        # Instructions filled in from scanner
+        self.insts = []
+
+        # True if we are parsing inside a lambda expression.
+        # because a lambda expression are written on a single line, certain line-oriented
+        # statements behave differently
+        self.is_lambda = is_lambda
+
+        self.start_symbol = start_symbol
+        self.new_rules = set()
+
+        # Placeholder for Python version tuple
+        self.version = (None, None)
+
+    def ast_first_offset(self, ast) -> Union[int, str]:
+        return ast.offset if hasattr(ast, "offset") else self.ast_first_offset(ast[0])
+
+    def add_unique_rule(
+        self, rule, opname: str, arg_count: int, customize: dict
+    ) -> None:
+        """Add rule to grammar, but only if it hasn't been added previously
+        opname and stack_count are used in the customize() semantic
+        the actions to add the semantic action rule. Stack_count is
+        used in custom opcodes like MAKE_FUNCTION to indicate how
+        many arguments it has. Often it is not used.
+        """
+        if rule not in self.new_rules:
+            # print("XXX ", rule) # debug
+            self.new_rules.add(rule)
+            self.addRule(rule, nop_func)
+            customize[opname] = arg_count
+            pass
+        return
+
+    def add_unique_rules(self, rules: list, customize: dict) -> None:
+        """Add rules (a list of string) to grammar. Note that
+        the rules must not be those that set arg_count in the
+        custom dictionary.
+        """
+        for rule in rules:
+            if len(rule) == 0:
+                continue
+            opname = rule.split("::=")[0].strip()
+            self.add_unique_rule(rule, opname, 0, customize)
+        return
+
+    def add_unique_doc_rules(self, rules_str: str, customize: dict) -> None:
+        """Add rules (a docstring-like list of rules) to grammar.
+        Note that the rules must not be those that set arg_count in the
+        custom dictionary.
+        """
+        # print(rules_str)
+        rules = [r.strip() for r in rules_str.split("\n")]
+        self.add_unique_rules(rules, customize)
+        return
+
+    def cleanup(self):
+        """
+        Remove recursive references to allow garbage
+        collector to collect this object.
+        """
+        for dict in (self.rule2func, self.rules, self.rule2name):
+            for i in list(dict.keys()):
+                dict[i] = None
+        for i in dir(self):
+            setattr(self, i, None)
+
+    def debug_reduce(self, rule, tokens, parent, last_token_pos):
+        """Customized format and print for our kind of tokens
+        which gets called in debugging grammar reduce rules
+        """
+
+        def fix(c):
+            s = str(c)
+            last_token_pos = s.find("_")
+            if last_token_pos == -1:
+                return s
+            else:
+                return s[:last_token_pos]
+
+        prefix = ""
+        if parent and tokens:
+            p_token = tokens[parent]
+            if hasattr(p_token, "linestart") and p_token.linestart:
+                prefix = "L.%3d: " % p_token.linestart
+            else:
+                prefix = "       "
+            if hasattr(p_token, "offset"):
+                prefix += "%3s" % fix(p_token.offset)
+                if len(rule[1]) > 1:
+                    prefix += "-%-3s " % fix(tokens[last_token_pos - 1].offset)
+                else:
+                    prefix += "     "
+        else:
+            prefix = "               "
+
+        print("%s%s ::= %s (%d)" % (prefix, rule[0], " ".join(rule[1]), last_token_pos))
+
+    def error(self, instructions, index):
+        # Find the last line boundary
+        start, finish = -1, -1
+        for start in range(index, -1, -1):
+            if instructions[start].linestart:
+                break
+            pass
+        for finish in range(index + 1, len(instructions)):
+            if instructions[finish].linestart:
+                break
+            pass
+        if start >= 0:
+            err_token = instructions[index]
+            print("Instruction context:")
+            for i in range(start, finish):
+                if i != index:
+                    indent = "   "
+                else:
+                    indent = "-> "
+                print("%s%s" % (indent, instructions[i]))
+            raise ParserError(err_token, err_token.offset, self.debug["reduce"])
+        else:
+            raise ParserError(None, -1, self.debug["reduce"])
+
+    def get_pos_kw(self, token):
+        """Return then the number of positional parameters and
+        represented by the attr field of token"""
+        # Low byte indicates number of positional parameters,
+        # high byte number of keyword parameters
+        args_pos = token.attr & 0xFF
+        args_kw = (token.attr >> 8) & 0xFF
+        return args_pos, args_kw
+
+    def nonterminal(self, nt, args):
+        n = len(args)
+
+        # # Use this to find lots of singleton rule
+        # if n == 1 and nt not in self.singleton:
+        #     print("XXX", nt)
+
+        if nt in self.collect and n > 1:
+            #
+            #  Collect iterated thingies together. That is rather than
+            #  stmts -> stmts stmt -> stmts stmt -> ...
+            #  stmms -> stmt stmt ...
+            #
+            if not hasattr(args[0], "append"):
+                # Was in self.optional_nt as a single item, but we find we have
+                # more than one now...
+                rv = GenericASTBuilder.nonterminal(self, nt, [args[0]])
+            else:
+                rv = args[0]
+                pass
+            # In a  list-like entity where the first item goes to epsilon,
+            # drop that and save the 2nd item as the first one
+            if len(rv) == 0 and nt not in self.keep_epsilon:
+                rv = args[1]
+            else:
+                rv.append(args[1])
+        elif n == 1 and args[0] in self.singleton:
+            rv = GenericASTBuilder.nonterminal(self, nt, args[0])
+            del args[0]  # save memory
+        elif n == 1 and nt in self.optional_nt:
+            rv = args[0]
+        else:
+            rv = GenericASTBuilder.nonterminal(self, nt, args)
+        return rv
+
+    def off2inst(self, token):
+        """
+        Return the corresponding instruction for this token
+        """
+        offset = token.off2int(prefer_last=False)
+        return self.insts[self.offset2inst_index[offset]]
+
+    def __ambiguity(self, children):
+        # only for debugging! to be removed hG/2000-10-15
+        print(children)
+        return GenericASTBuilder.ambiguity(self, children)
+
+    def resolve(self, list):
+        if len(list) == 2 and "function_def" in list and "assign" in list:
+            return "function_def"
+        if "grammar" in list and "expr" in list:
+            return "expr"
+        return GenericASTBuilder.resolve(self, list)
+
+
+class PythonParserExpr(PythonBaseParser):
+    """This corresponds to a single grammar expression: "expr". It matches smaller
+    units, so it is something to parse for that might be used when larger
+    pieces of code can't decompile.
+
+    """
+
+    def p_start_rule_expr(self, args):
+        """
+        expr_start       ::= expr return_value_opt
+        return_value_opt ::= RETURN_VALUE?
+        """
+
+    def __init__(self, debug_parser, start_symbol="expr_start"):
+        super(PythonParserExpr, self).__init__(
+            debug_parser=debug_parser, start_symbol=start_symbol
+        )
+
+
+PythonParserEval = PythonParserExpr
+
+
+class PythonParserExec(PythonBaseParser):
+    """
+    This corresponds to the compile-mode == "exec" of the `compile()` builtin
+    or exec() builtin function
+    """
+
+    # def p_exec(self, args):
+    #     """
+    #     stmts ::= stmt+
+    #     """
+
+    def __init__(self, debug_parser, start_symbol="stmts"):
+        super(PythonParserExec, self).__init__(
+            debug_parser=debug_parser, start_symbol=start_symbol
+        )
+
+
+class PythonParserLambda(PythonBaseParser):
+    """
+    This corresponds to the Python lambda definitions
+    """
+
+    def p_start_rule_lambda(self, args):
+        """
+        lambda_start ::= return_expr_lambda
+        """
+
+    # lambda_start is the highest level nonterminal. However
+    # we can pass in other nonterminals like "expr" for a different
+    # parse.
+    def __init__(self, debug_parser, start_symbol="lambda_start"):
+        super(PythonParserLambda, self).__init__(
+            start_symbol=start_symbol, debug_parser=debug_parser
+        )
+
+
+class PythonParserSingle(PythonBaseParser):
+    def p_start_rule_single(self, args):
+        """
+        # Single-mode interactive compilation
+        single_start ::= expr PRINT_EXPR
+        single_start ::= stmt
+        """
+
+    def __init__(self, debug_parser, start_symbol="single_start"):
+        super(PythonParserSingle, self).__init__(
+            start_symbol=start_symbol, debug_parser=debug_parser
+        )
+
+
+class PythonParser(PythonBaseParser):
+    def __init__(self, compile_mode, debug_parser):
+        # FIXME: go over.
+        if compile_mode == "single":
+            PythonParserSingle.__init__(self, debug_parser=debug_parser)
+        elif compile_mode == "lambda":
+            PythonParserLambda.__init__(self, debug_parser=debug_parser)
+        elif compile_mode == "eval":
+            PythonParserEval.__init__(self, debug_parser=debug_parser)
+        elif compile_mode == "exec":
+            PythonParserExec.__init__(self, debug_parser=debug_parser)
+        elif compile_mode == "eval_expr":
+            PythonParserEval.__init__(self, debug_parser=debug_parser)
+
+        else:
+            raise BaseException(
+                f'compile_mode should be either "exec", "single", "lambda", or "eval_expr"; got {compile_mode}'
+            )
+
+        # FIXME: customize per python parser version
+
+        # These are the non-terminals we should collect into a list.
+        # For example instead of:
+        #   stmts -> stmts stmt -> stmts stmt stmt ...
+        # collect as stmts -> stmt stmt ...
+        nt_list = [
+            "_stmts",
+            "and_parts",
+            "attributes",
+            "except_stmts",
+            "exprlist",
+            "importlist",
+            "kvlist",
+            "kwargs",
+            "or_parts",
+            # FIXME:
+            # If we add c_stmts, we can miss adding a c_stmt,
+            # test_float.py test_set_format() is an example.
+            # Investigate
+            # "c_stmts",
+            "stmts",
+            # Python 3.7+
+            "importlist37",
+        ]
+        self.collect = frozenset(nt_list)
+
+        # For these items we need to keep the 1st epslion reduction since
+        # the nonterminal name is used in a semantic action.
+        self.keep_epsilon = frozenset(("kvlist_n", "kvlist"))
+
+        # ??? Do we need a debug option to skip eliding singleton reductions?
+        # Time will tell if it if useful in debugging
+
+        # FIXME: optional_nt is a misnomer. It's really about there being a
+        # singleton reduction that we can simplify. It also happens to be optional
+        # in its other derivation
+        self.optional_nt |= frozenset(("suite_stmts", "c_stmts_opt", "stmt", "sstmt"))
+
+        # Reduce singleton reductions in these nonterminals:
+        # FIXME: would love to do expr, sstmts, stmts and
+        # so on but that would require major changes to the
+        # semantic actions
+        self.singleton = frozenset(
+            ("str", "store", "_stmts", "suite_stmts_opt", "inplace_op")
+        )
+        # Instructions filled in from scanner
+        self.insts = []
+
+        # true if we are parsing inside a lambda expression.
+        # because a lambda expression are written on a single line, certain line-oriented
+        # statements behave differently
+        self.is_lambda = False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/__init__.py

@@ -0,0 +1,52 @@
+"""
+Here we have checks done before a grammar rule reduction for that nonterminal takes place.
+
+These check the validity of rule reduction based on properties that aren't in
+the tokens. These checks basically have full access to everything.
+Optionally, it can have access to the tree built for the reduction nonterminal
+it checks.
+"""
+
+
+from decompyle3.parsers.reduce_check.and_check import and_invalid
+from decompyle3.parsers.reduce_check.and_cond_check import and_cond_check
+from decompyle3.parsers.reduce_check.and_not_check import and_not_check
+from decompyle3.parsers.reduce_check.break38_check import break_invalid
+from decompyle3.parsers.reduce_check.c_tryelsestmt import *  # noqa
+from decompyle3.parsers.reduce_check.for38_check import for38_invalid
+from decompyle3.parsers.reduce_check.forelse38_check import forelse38_invalid
+from decompyle3.parsers.reduce_check.if_and_elsestmt import *  # noqa
+from decompyle3.parsers.reduce_check.if_and_stmt import if_and_stmt
+from decompyle3.parsers.reduce_check.if_not_stmtc import if_not_stmtc_invalid
+from decompyle3.parsers.reduce_check.ifelsestmt import ifelsestmt
+from decompyle3.parsers.reduce_check.iflaststmt import *  # noqa
+from decompyle3.parsers.reduce_check.ifstmt import *  # noqa
+from decompyle3.parsers.reduce_check.ifstmts_jump import ifstmts_jump_invalid
+from decompyle3.parsers.reduce_check.lastc_stmt import *  # noqa
+from decompyle3.parsers.reduce_check.list_if_not import *  # noqa
+from decompyle3.parsers.reduce_check.not_or_check import *  # noqa
+from decompyle3.parsers.reduce_check.or_check import *  # noqa
+from decompyle3.parsers.reduce_check.or_cond_check import *  # noqa
+from decompyle3.parsers.reduce_check.pop_return import pop_return_check
+from decompyle3.parsers.reduce_check.testtrue import *  # noqa
+from decompyle3.parsers.reduce_check.tryexcept import *  # noqa
+from decompyle3.parsers.reduce_check.while1elsestmt import *  # noqa
+from decompyle3.parsers.reduce_check.while1stmt import *  # noqa
+from decompyle3.parsers.reduce_check.whilestmt import *  # noqa
+from decompyle3.parsers.reduce_check.whilestmt38 import whilestmt38_check
+from decompyle3.parsers.reduce_check.whileTruestmt38 import *  # noqa
+
+__all__ = [
+    "and_invalid",
+    "and_cond_check",
+    "and_not_check",
+    "break_invalid",
+    "for38_invalid",
+    "forelse38_invalid",
+    "if_and_stmt",
+    "if_not_stmtc_invalid",
+    "ifstmts_jump_invalid",
+    "ifelsestmt",
+    "pop_return_check",
+    "whilestmt38_check",
+]

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/and_check.py

@@ -0,0 +1,105 @@
+#  Copyright (c) 2020, 2022, 2024 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+NOT_POP_FOLLOW_OPS = frozenset(
+    """
+LOAD_ASSERT RAISE_VARARGS_1 STORE_FAST STORE_DEREF STORE_GLOBAL STORE_ATTR STORE_NAME
+""".split()
+)
+
+
+def and_invalid(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    # a LOAD_ASSERT is not an expression and not part of an "and"
+    # FIXME: the below really should have been done in the ingest
+    # phase.
+    ltm1 = tokens[last - 1]
+    rhs = rule[1]
+    if ltm1 == "LOAD_ASSERT" or (
+        ltm1 == "LOAD_GLOBAL" and ltm1.attr == "AssertionError"
+    ):
+        return True
+
+    expr_pjif = tree[0]
+    if expr_pjif == "expr_pjif":
+        jump = expr_pjif[1]
+    elif expr_pjif == "expr_jifop_cfs":
+        expr_jifop_cfs = expr_pjif
+        jump = expr_jifop_cfs[1]
+        if expr_jifop_cfs[0][0] == "or":
+            # FIXME check if the "or" jumps to the same place as jump.attr
+            return True
+        if first > 0:
+            ftm1 = tokens[first - 1]
+            if ftm1 == "JUMP_IF_TRUE_OR_POP" and ftm1.attr == jump.attr:
+                return True
+        else:
+            jump = tree[1]
+
+    elif rhs == ("and_parts", "expr") and expr_pjif[0] == "expr_pjif":
+        expr_pjif = expr_pjif[0]
+        jump = expr_pjif[1]
+    else:
+        # Probably not needed: was expr POP_JUMP_IF_FALSE
+        jump = tree[1]
+
+    if jump.kind.startswith("POP_JUMP_IF_"):
+        if last == n:
+            return True
+        jump_target = jump.attr
+        jump_offset = jump.offset
+
+        if tokens[first].off2int() <= jump_target < tokens[last].off2int():
+            return True
+
+        if rule == ("and", ("expr_pjif", "expr_pjif")):
+            jump2_target = tree[1][1].attr
+            return jump_target != jump2_target
+        elif rule == ("and", ("expr_pjif", "expr", "POP_JUMP_IF_TRUE")):
+            jump2_target = tree[2].attr
+            return jump_target == jump2_target
+        elif rule == ("and", ("expr_pjif", "expr")):
+            if tokens[last] == "POP_JUMP_IF_FALSE":
+                # Ok if jump_target doesn't jump to last instruction
+                return jump_target != tokens[last].attr
+            elif tokens[last] in ("POP_JUMP_IF_TRUE", "JUMP_IF_TRUE_OR_POP"):
+                # Ok if jump_target jumps to a COME_FROM after
+                # the last instruction or jumps right after the last instruction
+                if last + 1 < n and tokens[last + 1] == "COME_FROM":
+                    return jump_target != tokens[last + 1].off2int()
+                return jump_target + 2 != tokens[last].attr
+        elif rule == ("and", ("expr_pjif", "expr", "COME_FROM")):
+            return tree[-1].attr != jump_offset
+        elif (
+            rule == ("and", ("and_parts", "expr"))
+            and jump_target > tokens[last].off2int()
+            and tokens[last].kind.startswith("JUMP_IF_")
+            and jump_target < tokens[last].attr
+        ):
+            # This could be an "(i and j) or k"
+            # or:
+            #    - and: expr, POP_JUMP_IF_FALSE jump_target, expr
+            #    - JUMP_IF_TRUE_OR_POP end_or
+            #    - jump_target: expr
+            # end_or:
+            return False
+
+        return jump_target != tokens[last].off2int()
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/and_cond_check.py

@@ -0,0 +1,32 @@
+#  Copyright (c) 2020 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def and_cond_check(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    rhs = rule[1]
+    if rhs[0] in ("and_parts", "testfalse") and rhs[1] == "expr_pjif":
+        and_parts = ast[0]
+        last_expr_pjif = ast[1]
+        test_jump_target = last_expr_pjif[-1].attr
+        expr_pjif = and_parts[0]
+        while expr_pjif == "and_parts":
+            expr_pjif = expr_pjif[0]
+            if expr_pjif == "expr_pjif" and test_jump_target != expr_pjif[-1].attr:
+                return True
+            pass
+        if expr_pjif == "expr_pjif":
+            return test_jump_target != expr_pjif[-1].attr
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/and_not_check.py

@@ -0,0 +1,22 @@
+#  Copyright (c) 2020 Rocky Bernstein
+
+
+def and_not_check(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    if ast[0] == "expr_pjif":
+        jmp = ast[0][1]
+    else:
+        jmp = ast[1]
+    if jmp.kind.startswith("POP_JUMP_IF_"):
+        if last == n:
+            return True
+        jump_target = jmp.attr
+
+        if tokens[first].off2int() <= jump_target < tokens[last].off2int():
+            return True
+        if rule == ("and_not", ("expr_pjif", "expr_pjit")):
+            jmp2_target = ast[1][1].attr
+            return jump_target != jmp2_target
+        return jump_target != tokens[last].off2int()
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/break38_check.py

@@ -0,0 +1,34 @@
+#  Copyright (c) 2020, 2022 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def break_invalid(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+
+    if rule[1] != ("POP_EXCEPT", "JUMP_FORWARD"):
+        return False
+
+    # Look for a JUMP_LOOP instruction either after
+    # the end of this rule or before the place where
+    # we JUMP_FORWARD to
+    if last + 1 < n and tokens[last + 1] == "JUMP_LOOP":
+        return False
+
+    # FIXME: put jump_loop classification in a subroutine. Preferably in xdis.
+    jump_target_prev = self.insts[self.offset2inst_index[tokens[first + 1].attr] - 1]
+    is_jump_loop = (
+        jump_target_prev.is_jump() and jump_target_prev.arg < jump_target_prev.offset
+    )
+    return not is_jump_loop

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/c_tryelsestmt.py

@@ -0,0 +1,73 @@
+#  Copyright (c) 2020 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from decompyle3.parsers.treenode import SyntaxTree
+
+
+def c_tryelsestmt(self, lhs, n, rule, ast, tokens, first, last):
+    # Check the end of the except handler that there isn't a jump from
+    # inside the except handler to the end. If that happens
+    # then this is a "try" with no "else".
+    except_handler = ast[3]
+
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    if except_handler == "except_handler_else":
+        except_handler = except_handler[0]
+
+    come_from = except_handler[-1]
+    # We only care about the *first* come_from because that is the
+    # the innermost one. So if the "tryelse" is invalid (should be a "try")
+    # it will be invalid here.
+    if come_from == "COME_FROM":
+        first_come_from = except_handler[-1]
+    elif come_from == "END_FINALLY":
+        first_come_from = None
+    elif come_from == "except_return":
+        return False
+    else:
+        assert come_from in ("come_froms", "opt_come_from_except")
+        first_come_from = come_from[0]
+        if not hasattr(first_come_from, "attr"):
+            # optional come from
+            return False
+
+    leading_jump = except_handler[0]
+
+    # We really don't care that this is a jump per-se. But
+    # we could also check that this jumps to the end of the except if
+    # desired.
+    if isinstance(leading_jump, SyntaxTree):
+        leading_jump = leading_jump.first_child()
+
+    # If there is a jump in the except that goes to the same place as
+    # except_handler_first_offset, then this is a "try" without an else.
+    except_stmt = except_handler[2]
+    if except_stmt in ("c_except_stmts", "except_stmts"):
+        first_except = except_stmt[0]
+        first_except_offset = first_except.first_child().off2int(prefer_last=False)
+        i = self.offset2inst_index[first_except_offset]
+        else_offset = leading_jump.attr
+        inst = self.insts[i]
+        while inst.offset < else_offset:
+            if inst.is_jump() and inst.argval == else_offset:
+                return True
+            i += 1
+            inst = self.insts[i]
+            pass
+        pass
+
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/for38_check.py

@@ -0,0 +1,84 @@
+#  Copyright (c) 2020, 2022-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from decompyle3.scanners.tok import off2int
+
+
+def for38_invalid(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    """The only difference between a "for" and a "for else" is that
+    jumps within the "for" never go past the "FOR_ITER" offset.
+    """
+
+    first_offset = tokens[first].off2int(prefer_last=False)
+    last_offset = tokens[last].off2int(prefer_last=False)
+    if last_offset == -1:
+        last_offset = tokens[last - 1].off2int(prefer_last=False)
+
+    start = self.offset2inst_index[first_offset]
+    end = off2int(self.offset2inst_index[last_offset], prefer_last=True)
+
+    # In the loop below, we expect the first "FOR_ITER" to
+    # be before any jumps that go to the end of it (in the case of "for")
+    # or beyond it (in the case of "for else").
+
+    for_body_end_offset = None
+    for i in range(start, end):
+        inst = self.insts[i]
+
+        # Hack alert for magic number 2's below: in Python 3.8+ instructions are 2 bytes
+        # inst.argval - 2 is the offset of the instruction *before* inst.argval and
+        # +2 for the instruction that follows.
+
+        if not for_body_end_offset and inst.opname == "FOR_ITER":
+            # There can be some slop in "last" as to where the body ends. If the rule
+            # ends in "JUMP_LOOP", then "last" doesn't need adjusting.
+            for_body_end_offset = (
+                inst.argval if rule[1][-1] == "JUMP_LOOP" else inst.argval - 2
+            )
+            if self.insts[end].has_extended_arg:
+                last_offset += 2
+            if last_offset < for_body_end_offset:
+                # "for" body isn't big enough
+                return True
+            continue
+        if (
+            for_body_end_offset
+            and inst.is_jump()
+            and inst.argval > for_body_end_offset + 2
+        ):
+            # Another weird case.
+            # Guard against misclassified things like:
+            #   if a:
+            #     for n in l:
+            #       if b: break # jumps past "else" which is after the end of the "for"
+            #       eird case.
+            # Guard against misclassified things like:
+            #   if a:
+            #     for n in l:
+            #       if b: break # jumps past "else" which is after the end of the "for"
+            #       elif c:
+            #         r = 2
+            #   else:
+            #        r = 3
+            # The way we distinguish this is to check if the instruction after the body end
+            # starts with a jump, the start of the encompassing if/else.
+            # The "else" part of a "for/else" never starts with a jump.
+            body_end_next_inst = self.insts[
+                self.offset2inst_index[for_body_end_offset + 2]
+            ]
+            return not body_end_next_inst.is_jump()
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/forelse38_check.py

@@ -0,0 +1,60 @@
+#  Copyright (c) 2023 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def forelse38_invalid(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    """The only difference between a "forelse" and and a "for" is that
+    that the "come_from" location contains something other than
+    for_iter.
+    """
+
+    saw_break = False
+    saw_break_to_last = False
+    last_offset = tokens[last].off2int()
+
+    # for i in range(first, last):
+    #     print(tokens[i])
+
+    else_start = None
+    for node in tree:
+        if node.kind.startswith("else"):
+            else_start = node.first_child().off2int()
+    assert else_start is not None
+
+    for i in range(first, last):
+        t = tokens[i]
+        if t.off2int() >= else_start:
+            break
+        if t == "BREAK_LOOP":
+            if else_start <= t.attr < last_offset:
+                # We should not be jumping into the "else" part.
+                return True
+            saw_break = True
+            saw_break_to_last = t.attr == last_offset
+            # if saw_break_to_last:
+            #     from trepan.api import debug; debug()
+        if t.kind == "JUMP_FORWARD":
+            # We should be jumping to the "else" part.
+            if not t.attr == else_start:
+                return True
+
+    # If we haven't seen a BREAK_LOOP, then
+    # "for/else" and "for" are the same. But here
+    # we should prefer the simpler "for"
+    if saw_break:
+        return not saw_break_to_last
+    return True

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/if_and_elsestmt.py

@@ -0,0 +1,47 @@
+#  Copyright (c) 2020 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def if_and_elsestmt(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    # Make sure jumps don't extend beyond the end of the if statement.
+    last_offset = tokens[last].off2int()
+    for i in range(first, last):
+        t = tokens[i]
+        # instead of POP_JUMP_IF, should we use op attributes?
+        if t.kind.startswith("POP_JUMP_IF_"):
+            pjif_target = t.attr
+            if pjif_target > last_offset:
+                # In come cases, where we have long bytecode, a
+                # "POP_JUMP_IF_TRUE/FALSE" offset might be too
+                # large for the instruction; so instead it
+                # jumps to a JUMP_FORWARD. Allow that here.
+                if tokens[last] == "JUMP_FORWARD":
+                    return tokens[last].attr != pjif_target
+                return True
+            elif lhs == "ifstmtc" and tokens[first].off2int() > pjif_target:
+                # A conditional JUMP to the loop is expected for "ifstmtc"
+                return False
+            pass
+        pass
+    pass
+
+    if not ast:
+        return False
+
+    if rule[1][:2] == ("expr_pjif", "expr_pjif"):
+        # The two POP_JUMP_IF_FALSE should go to the same place for an "and"
+        return ast[0][1].attr != ast[1][1].attr
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/if_and_stmt.py

@@ -0,0 +1,52 @@
+#  Copyright (c) 2020 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def if_and_stmt(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    # Make sure jumps don't extend beyond the end of the if statement.
+    last_offset = tokens[last].off2int()
+    for i in range(first, last):
+        t = tokens[i]
+        # instead of POP_JUMP_IF, should we use op attributes?
+        if t.kind.startswith("POP_JUMP_IF_"):
+            pjif_target = t.attr
+            if pjif_target > last_offset:
+                # In come cases, where we have long bytecode, a
+                # "POP_JUMP_IF_TRUE/FALSE" offset might be too
+                # large for the instruction; so instead it
+                # jumps to a JUMP_FORWARD. Allow that here.
+                if tokens[last] == "JUMP_FORWARD":
+                    return tokens[last].attr != pjif_target
+                return True
+            elif lhs == "ifstmtc" and tokens[first].off2int() > pjif_target:
+                # A conditional JUMP to the loop is expected for "ifstmtc"
+                return False
+            pass
+        pass
+    pass
+
+    if not ast:
+        return False
+
+    if rule[1][:-1] == ("expr_pjif", "expr", "COME_FROM", "stmts"):
+        # POP_JUMP_IF_FALSE should go to the COME_FROM
+        return ast[2].attr != ast[0][1].off2int(prefer_last=False)
+    else:
+        end_if_jump = ast[1]
+        end_if_offset = end_if_jump.attr
+        # stmts = ast[-2]
+        # come_froms = ast[-1]
+        return end_if_offset < tokens[last].off2int(prefer_last=False)

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/if_not_stmtc.py

@@ -0,0 +1,27 @@
+#  Copyright (c) 2020, 2023 Rocky Bernstein
+
+
+def if_not_stmtc_invalid(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    assert len(tree) >= 2
+    testexprc, ifstmts_jumpc = tree[:2]
+
+    if testexprc != "testexprc" or ifstmts_jumpc != "ifstmts_jumpc":
+        return False
+
+    # print("XXX1", testexprc)
+    # print("XXX2", ifstmts_jumpc)
+
+    # Make sure the testexprc does not jump inside the "then"
+    last_offset = tokens[last].off2int()
+    then_jump = testexprc.last_child()
+    if not then_jump.kind.startswith("POP_JUMP_IF_"):
+        return False
+    then_jump_offset = then_jump.attr
+    # print("XXX", then_jump)
+    # print("XXX", then_jump_offset, "<=", last_offset)
+
+    if then_jump_offset <= last_offset:
+        return True
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/ifelsestmt.py

@@ -0,0 +1,281 @@
+#  Copyright (c) 2020, 2022-2023 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from decompyle3.scanners.tok import Token
+
+
+# When we use dominators, presumbaly this will be a lto cleaner
+def ifelsestmt(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+
+    if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP":
+        # ifelsestmt jumped outside of loop. No good.
+        return True
+
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    first_offset = tokens[first].off2int()
+    last_offset = tokens[last].off2int(prefer_last=False)
+
+    # FIXME: It is conceivable the below could be handled strictly in the grammar.
+    # If we have an optional else, then we *must* have a COME_FROM for it.
+    # Otherwise this is fine as an "if" without the "else"
+
+    if rule[1][2] == "jf_cfs":
+        jf_cfs = tree[2]
+        jump = jf_cfs[0]
+        # The jf_cfs should jump to the end of the ifelse, and not beyond it.
+        # Think about: should we also check that it isn't to the
+        # *interior* of the "else" part?
+        jump = jf_cfs[0]
+        if jump == "JUMP_FORWARD" and jump.attr > last_offset:
+            # There is one situation where jf_cfs does not jump to the end of ifelse,
+            # and that when this is inside *another* ifelse:
+            # if x:
+            #   if y:
+            #     ...
+            #   else:
+            #     jumps to the end of the *enclosing* ifelse
+            # else
+            # ...
+            #
+            # We can detect this because there is a JUMP_FORWARD a the end of the ifelese
+            # that also jumps to the same location
+            if not (tokens[last] == "JUMP_FORWARD" and tokens[last].attr == jump.attr):
+                return True
+
+    if rule[1][2:4] == ("jf_cfs", "\\e_else_suite_opt"):
+        come_froms = jf_cfs[1]
+        if isinstance(come_froms, Token):
+            come_from_target = come_froms.attr
+        else:
+            if len(come_froms) == 0:
+                # We are seeing in optional else's:
+                #   XX       JUMP_FORWARD         XX+2
+                #   XX+2_00  COME_FROM            XX
+                # and these aren't caught by our "if/then" rules
+                return tokens[last].off2int() != jf_cfs[0].attr
+            come_from_target = come_froms[-1].attr
+
+        if come_from_target < first_offset:
+            return True
+
+    # Make sure all the offsets from the "COME_FROMs" at the
+    # end of the "if" come from somewhere inside the "if".
+    # Since the come_froms are ordered so that lowest
+    # offset COME_FROM is last, it is sufficient to test
+    # just the last one.
+    # Note: We may find Example A defeats this rule.
+    if len(tree) == 5:
+        end_come_froms = tree[-1]
+        if end_come_froms == "opt_come_from_except" and len(end_come_froms) > 0:
+            end_come_froms = end_come_froms[0]
+            pass
+        while not isinstance(end_come_froms, Token) and len(end_come_froms):
+            end_come_froms = end_come_froms[-1]
+        if isinstance(end_come_froms, Token):
+            if first_offset > end_come_froms.attr:
+                return True
+            elif first_offset > end_come_froms.attr:
+                return True
+
+    testexpr = tree[0]
+
+    if_condition = testexpr[0]
+
+    # Check that the condition portion of the "if"
+    # jumps to the "else" part, and that the
+    # end of the "then" portion jumps to a reasonable
+    # place, e.g. not somewhere in the middle of the "else"
+    # portion.
+    if if_condition in ("testtrue", "testfalse", "and_cond"):
+
+        then_end = tree[2]
+        else_suite = tree[3]
+        if else_suite == "else_suite_opt" and len(else_suite):
+            else_suite = else_suite[0]
+
+        if else_suite not in ("else_suite", "else_suitec"):
+            # May need to handle later.
+            return False
+
+        # We may need this later:
+
+        # not_or = if_condition[0]
+        # if not_or == "not_or":
+        #     # "if not_or" needs special attention to distinguish it from "if and".
+        #     # If the jump is to the beginning of the "else" part, this is an "and".
+        #     not_or_jump_expr = not_or[-1]
+        #     if not_or_jump_expr == "_come_froms":
+        #         not_or_jump_expr = not_or[-2]
+        #     not_or_jump_offset = not_or_jump_expr.last_child().attr
+        #     if not_or_jump_offset == else_suite.first_child().offset:
+        #         return True
+
+        # If there is a COME_FROM at the end, it (the outermost COME_FROM) needs to come
+        # from within the "if-then" part
+        if isinstance(then_end, Token):
+            then_end_come_from = then_end
+        else:
+            then_end_come_from = then_end.last_child()
+
+        if then_end_come_from == "COME_FROM" and then_end_come_from.attr < first_offset:
+            return True
+
+        # If there any instructions in the "then" part that jump to the beginning of the
+        # "else" then this is not a proper if/else. Note that we might generalize this
+        # to jump *anywhere* in the else body instead of the first instruction.
+        else_start_offset = else_suite.first_child().off2int(prefer_last=False)
+
+        then_start = tree[1].first_child()
+        if then_start is None:
+            return False
+
+        then_start_offset = tree[1].first_child().off2int(prefer_last=False)
+
+        i = self.offset2inst_index[then_start_offset]
+        inst = self.insts[i]
+        while inst.offset < else_start_offset:
+            if inst.is_jump() and inst.argval == else_start_offset:
+                return True
+            i += 1
+            inst = self.insts[i]
+
+        if last_offset == -1:
+            last_offset = tokens[last - 1].off2int(prefer_last=False)
+
+        if else_suite == "else_suitec" and then_end in (
+            "jb_elsec",
+            "jb_cfs",
+            "jump_forward_else",
+        ):
+            stmts = tree[1]
+            jb_else = then_end
+            come_from = jb_else[-1]
+            if come_from in ("come_froms", "_come_froms") and len(come_from):
+                come_from = come_from[-1]
+            if come_from == "COME_FROM":
+                if come_from.attr > stmts.first_child().off2int():
+                    return True
+                pass
+            pass
+        elif else_suite == "else_suite" and then_end == "jf_cfs":
+            stmts = tree[1]
+            jf_cfs = then_end
+            if jf_cfs[0].attr < last_offset:
+                return True
+
+        if if_condition == "and_cond" and if_condition[1] == "expr_pjif":
+            if_condition = if_condition[1]
+
+        if_condition_last = if_condition.last_child()
+        if if_condition_last.kind.startswith("POP_JUMP_IF_"):
+            if last == n:
+                last -= 1
+
+            jmp = if_condition_last
+            jump_target = jmp.attr
+
+            # Below we check that jump_target is jumping to a feasible
+            # location. It should be to the transition after the "then"
+            # block and to the beginning of the "else" block.
+            # However the "if/else" is inside a loop the false test can be
+            # back to the loop.
+
+            # FIXME: the below logic for jf_cfs could probably be
+            # simplified.
+            jump_else_end = tree[2]
+            if jump_else_end == "jf_cf_pop":
+                jump_else_end = jump_else_end[0]
+
+            # jump_to_jump = False
+            if jump_else_end == "JUMP_FORWARD":
+                # jump_to_jump = True
+                endif_target = int(jump_else_end.attr)
+                if endif_target != last_offset:
+                    return True
+
+            if jump_target == last_offset:
+                # jump_target should be jumping to the end of the if/then/else
+                # but is it jumping to the beginning of the "else"
+                return True
+            if (
+                jump_else_end in ("jf_cfs", "jump_forward_else")
+                and jump_else_end[0] == "JUMP_FORWARD"
+            ):
+                # If the "else" jump jumps before the end of the the "if .. else end",
+                # then this is not this kind of "ifelsestmt".
+                jump_else_forward = jump_else_end[0]
+                jump_else_forward_target = jump_else_forward.attr
+                if jump_else_forward_target < last_offset:
+                    return True
+                pass
+            if (
+                jump_else_end in ("jf_cfs", "come_froms")
+                and jump_else_end[-1] == "COME_FROM"
+            ):
+                if jump_else_end[-1].off2int() != jump_target:
+                    return True
+
+                # If the end of the "then" jumps to back to a loop,
+                # then the end of the "else" must jump somewhere too
+                # and not fall through.
+                if jump_else_end == "jb_cfs":
+                    i = self.offset2inst_index[last_offset]
+                    inst = self.insts[i]
+                    if not inst.is_jump():
+                        return True
+                    pass
+                pass
+
+            # If we have a jump_back, i.e we are in a loop, then a "end_then" of
+            # the "else" can't be a fallthrough kind of instruction. In other
+            # words, tokens[last] should have be a COME_FROM. Otherwise the
+            # "else" suite should be extended to cover the next instruction at
+            # tokens[last].
+            if jump_else_end in ("jb_elsec", "jb_cfs") and tokens[last] not in (
+                "COME_FROM",
+                "JUMP_LOOP",
+                "COME_FROM_LOOP",
+            ):
+                return True
+
+            # If the part before the "else" statement doesn't have a JUMP in it,
+            # i.e. is a "COME_FROM", then the statement before he COME_FROM should
+            # not fallthrough. Otherwise we have an "if" statement, not "if/else".
+
+            # if lhs == "ifelsestmtc":
+            #     print("XXX", first, last, tokens[first], tokens[last])
+            #     from trepan.api import debug; debug()
+
+            if jump_else_end == "come_froms":
+                jump_else_end = jump_else_end.last_child()
+            if jump_else_end == "COME_FROM":
+                come_from_offset = jump_else_end.off2int(prefer_last=False)
+                before_come_from = self.insts[
+                    self.offset2inst_index[come_from_offset] - 1
+                ]
+                # FIXME: When xdis next changes, this will be a field in the instruction
+                no_follow = before_come_from.opcode in self.opc.nofollow
+                return not (before_come_from.is_jump() or no_follow)
+
+            if first_offset > jump_target:
+                return True
+
+            return (jump_target > last_offset) and tokens[last] != "JUMP_FORWARD"
+
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/iflaststmt.py

@@ -0,0 +1,200 @@
+#  Copyright (c) 2020, 2022 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from decompyle3.scanners.tok import Token
+
+
+def iflaststmt(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    testexpr = tree[0]
+    rhs = rule[1]
+
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    # FIXME: should this be done in the caller?
+    if tokens[last] == "RETURN_LAST":
+        last -= 1
+
+    # If there is a fall-through it shouldn't be somewhere
+    # inside iflaststmt, since the point of this is to handle
+    # if statements that *don't* fall though.
+    if tokens[last] == "COME_FROM":
+        come_from_offset = tokens[last].attr
+        if tokens[first].off2int() <= come_from_offset <= tokens[last].off2int():
+            return True
+
+    if rhs[0:2] in (
+        ("testexpr", "stmts"),
+        ("testexpr", "c_stmts"),
+        ("testexprc", "c_stmts"),
+    ):
+
+        # "stmts" (end of then) should not end in a fallthrough instruction
+        # other wise this is just a plain ol' stmt.
+        ltm1 = tokens[last - 1]
+        if ltm1 == "COME_FROM":
+            return True
+        then_end = self.off2inst(ltm1)
+
+        # FIXME: fallthrough should be an xdis thing. Until then...
+        if then_end.opcode not in self.opc.nofollow and tokens[last] != "JUMP_LOOP":
+            return True
+
+        # If there is a trailing if-jump (forward) at the end of "testexp", it should
+        # to the end of "stmts".
+
+        # If there was backward jump, the LHS would be "iflaststmtc".
+        # Note that there might not be a COME_FROM before "stmts" because there can be a fall
+        # through to it.
+        stmt_offset = tree[1].first_child().off2int(prefer_last=False)
+        inst_offset = self.offset2inst_index[stmt_offset]
+
+        test_expr_offset = tree[0].first_child().off2int(prefer_last=False)
+        test_inst_offset = self.offset2inst_index[test_expr_offset]
+
+        last_offset = tokens[last].off2int(prefer_last=False)
+
+        # Make sure there are *forward* jumps outside offset range of this construct.
+        # This helps distinguish:
+        #     while True:
+        #        if testexpr
+        # from:
+        #     while testexpr
+        for i in range(test_inst_offset, inst_offset):
+            inst = self.insts[i]
+            if inst.is_jump() and inst.argval > last_offset:
+                return True
+
+        testexpr_last_inst = self.insts[inst_offset - 1]
+        if testexpr_last_inst.is_jump():
+            target_offset = testexpr_last_inst.argval
+            if target_offset != last_offset:
+                if target_offset < last_offset:
+                    # Look for this kind of situation from: iflaststmtc ::= testexprc c_stmts
+                    #
+                    # L.  13        78  LOAD_NAME                ncols
+                    #               80  POP_JUMP_IF_FALSE_LOOP    40  'to 40'
+                    #
+                    # L.  14        82  POP_TOP
+                    #               84  CONTINUE             20  'to 20'
+                    #                   ^^^^^^^^^
+                    #
+                    #  L.  15        86  JUMP_LOOP            40  'to 40'
+                    #                    COME_FROM ...
+                    #                    ^^^ last
+                    return tokens[last - 2] != "CONTINUE"
+
+                # There is still this weird case:
+                # if a:
+                #   if b:
+                #     x += 3
+                #     # jumps to same place as "if a then.." end jump.
+                # else:
+                #    ...
+                # we are going to hack this by looking for another jump to the same target. Sigh.
+                i = inst_offset
+                inst = self.insts[i]
+                while inst.offset < target_offset:
+                    if inst.is_jump() and inst.argval == target_offset:
+                        return False
+                    i += 1
+                    inst = self.insts[i]
+                    pass
+                last_index = self.offset2inst_index[last_offset]
+                last_inst = self.insts[last_index]
+                # Jumping beyond last_offset is okay since this may be the
+                # inner "if" jumping around the "else" situation above.
+                if last_inst.is_jump():
+                    return target_offset == last_offset
+                else:
+                    # A fallthrough can't jump *beyond* the end in the nested
+                    # "if" around and outer "else"
+                    return True
+            pass
+        pass
+
+    if testexpr[0] == "testexpr":
+        testexpr = testexpr[0]
+    if testexpr[0] in ("testtrue", "testtruec", "testfalse", "testfalsec"):
+
+        if_condition = testexpr[0]
+        if_condition_len = len(if_condition)
+        if_bool = if_condition[0]
+        if (
+            if_condition_len == 1
+            and if_bool in ("nand", "and")
+            and rhs == ("testexpr", "stmts")
+        ):
+            # (n)and rules have precedence
+            return True
+        elif if_bool == "not_or":
+            then_end = if_bool[-1]
+            if isinstance(then_end, Token):
+                then_end_come_from = then_end
+            else:
+                then_end_come_from = if_bool[-2].last_child()
+
+            # If there jump location is right after then end of this rule, then we have
+            # an "and", not a "not_or"
+
+            if (
+                then_end_come_from == "POP_JUMP_IF_FALSE"
+                and then_end_come_from.attr == tokens[last].off2int()
+            ):
+                return True
+            pass
+
+        if if_condition_len > 1 and if_condition[1].kind.startswith("POP_JUMP_IF_"):
+            if last == n:
+                last -= 1
+            jump_target = if_condition[1].attr
+            first_offset = tokens[first].off2int()
+            if first_offset <= jump_target < tokens[last].off2int():
+                return True
+            # jump_target less than tokens[first] is okay - is to a loop
+            # jump_target equal tokens[last] is also okay: normal non-optimized non-loop jump
+
+            if (last + 1) < n:
+                if tokens[last - 1] == "JUMP_LOOP":
+                    if jump_target > first_offset:
+                        # The end of the iflaststmt if test jumps backward to a loop
+                        # but the false branch of the "if" doesn't also jump back.
+                        # No good. This is probably an if/else instead.
+                        return True
+                    pass
+                elif (
+                    tokens[last + 1] == "COME_FROM_LOOP"
+                    and tokens[last] != "BREAK_LOOP"
+                ):
+                    # iflastsmtc is not at the end of a loop, but jumped outside of loop. No good.
+                    # FIXME: check that tokens[last] == "POP_BLOCK"? Or allow for it not to appear?
+                    return True
+
+            # If the instruction before "first" is a "POP_JUMP_IF_FALSE" which goes
+            # to the same target as jump_target, then this not nested "if .. if .."
+            # but rather "if ... and ..."
+            if first > 0 and tokens[first - 1] == "POP_JUMP_IF_FALSE":
+                return tokens[first - 1].attr == jump_target
+
+            if jump_target > tokens[last].off2int():
+                if jump_target == tokens[last - 1].attr:
+                    # if c1 [jump] jumps exactly the end of the iflaststmt...
+                    return False
+                pass
+            pass
+        pass
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/ifstmt.py

@@ -0,0 +1,230 @@
+#  Copyright (c) 2020, 2022-2024 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#  Example A: an example where we have weird COME_FROMs
+#
+#   if a:
+#     if b:   # false jumps around outer else
+#       raise
+#   elif c:
+#      a = 2
+#   #end is jump to by "if not b" above
+
+
+def ifstmt(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+
+    # print("XXX", tokens[first].offset , tokens[last].offset, rule)
+    # for t in range(first, last):
+    #     print(tokens[t])
+    # print("=" * 40)
+
+    if rule == ("ifstmt", ("bool_op", "stmts", "\\e__come_froms")):
+        return False
+
+    ltm1_index = last - 1
+    while tokens[ltm1_index] == "COME_FROM":
+        ltm1_index -= 1
+    ltm1 = tokens[ltm1_index]
+
+    first_offset = tokens[first].off2int(prefer_last=False)
+
+    # The below doesn't work for Example A above
+    # # Test that the outermost COME_FROM, if it exists, must be *somewhere*
+    # # in the range of the if stmt.
+    # if ltm1 == "COME_FROM" and ltm1.attr < first_offset:
+    #     return True
+
+    if not tree:
+        return False
+
+    ifstmts_jump = tree[1]
+    if ifstmts_jump.kind.startswith("ifstmts_jump"):
+        come_from = ifstmts_jump[0]
+        if come_from == "COME_FROM" and come_from.attr < first_offset:
+            return True
+
+    testexpr = tree[0]
+
+    test = testexpr[0]
+
+    # We have two grammar rules: ifstmtc and if_not_stmtc
+    # which are the same:
+    #    xxx  ::= testexprc ifstmts_jumpc _come_froms
+    # and these need to be disambiguated
+    # When  ifstmts_jumpc goes back to to a loop
+    # and testexprc is testtruec, then we have if_not_stmtc.
+
+    if lhs == "ifstmtc" and test == "testtruec" and ifstmts_jump == "ifstmts_jumpc":
+        if len(test) > 1:
+            return test[1] != "POP_JUMP_IF_FALSE_LOOP"
+
+    if lhs == "if_not_stmtc" and ifstmts_jump == "ifstmts_jumpc":
+        if test == "testexpr":
+            test = test[0]
+        if test in ("testfalsec", "testfalse"):
+            return True
+
+        if test in ("testtruec", "testtrue") and ifstmts_jump == "ifstmts_jumpc":
+            if test[0] == "expr_pjit":
+                test = test[0]
+            if len(test) > 1:
+                return test[1] == "POP_JUMP_IF_FALSE_LOOP"
+
+    pop_jump_if = None
+
+    if test in ("testexpr", "testexprc"):
+        test = test[0]
+
+    pop_jump_if = None
+    if test in ("testtrue", "testtruec", "testfalse"):
+
+        if len(test) == 1 and test[0].kind.startswith("expr_pji"):
+            pop_jump_if = test[0][1]
+        elif len(test) > 1 and test[1].kind.startswith("POP_JUMP_IF_"):
+            pop_jump_if = test[1]
+
+        if pop_jump_if:
+            jump_target = pop_jump_if.attr
+            if last == n:
+                last -= 1
+
+            # Get reasonable offset "end if" offset
+            endif_offset = ltm1.off2int(prefer_last=True)
+            if endif_offset == -1:
+                endif_offset = tokens[last - 2].off2int(prefer_last=True)
+
+            if first_offset <= jump_target < endif_offset:
+                if rule[1] == ("testexpr", "stmts", "come_froms"):
+                    come_froms = tree[2]
+
+                    if hasattr(come_froms, "first_child"):
+                        come_from_offset = come_froms.first_child()
+                    else:
+                        assert come_froms.kind.startswith("COME_FROM")
+                        come_from_offset = come_froms.off2int()
+                    return jump_target != come_from_offset
+                # FIXME: investigate why this happens for "if"s with EXTENDED_ARG POP_JUMP_IF_FALSE.
+                # An example is decompyle3/semantics/transform.py n_ifelsestmt.py
+                elif rule[1][-1] == "\\e__come_froms":
+                    return True
+                pass
+
+            endif_inst_index = self.offset2inst_index[ltm1.off2int(prefer_last=False)]
+
+            # FIXME: RAISE_VARARGS is an instance of a no-follow instruction.
+            # Should this be generalized? For example for RETURN ?
+            if ltm1.kind.startswith("RAISE_VARARGS"):
+                endif_inst_index += 2
+
+            if endif_inst_index + 1 == len(self.insts):
+                return False
+            endif_next_inst = self.insts[endif_inst_index + 1]
+
+            # jump_target equal tokens[last] is also okay: normal non-optimized non-loop jump
+            if jump_target > endif_next_inst.offset:
+                # test for Example A where "if b" jumps around the outer "else"
+                if jump_target == tokens[last - 1].attr:
+                    return False
+                if last < n and tokens[last].kind.startswith("JUMP"):
+                    # Distinguish code like:
+                    #
+                    #   if a and not b:  # there are two jumps to "else" here
+                    #     real = 2       # there is a jump around the else here
+                    #  else:
+                    #     real = 3
+                    #
+                    # and don't confuse with:
+                    #
+                    #   if a:
+                    #     if not b:      # the test below excludes this inner "if"
+                    #        real = 2
+                    #   real = 3
+                    # which is wrong
+                    if (
+                        first > 0
+                        and tokens[first - 1].kind.startswith("POP_JUMP_IF_")
+                        and tokens[first - 1].attr == jump_target
+                    ):
+                        return True
+                    return False
+                return True
+            elif jump_target < first_offset:
+                # jump_target less than tokens[first] is okay - is to a loop
+                assert test == "testtruec"  # and lhs == "ifsmtc"
+                # Since the "if" test is backwards, there shouldn't
+                # be a "COME_FROM", but should be some sort of
+                # instruction that does "not' fall through, like a jump
+                # return, or raise.
+                if ltm1 == "COME_FROM":
+                    before_come_from = self.insts[
+                        self.offset2inst_index[endif_offset] - 1
+                    ]
+                    # FIXME: When xdis next changes, this will be a field in the instruction
+                    no_follow = before_come_from.opcode in self.opc.nofollow
+                    return not (before_come_from.is_jump() or no_follow)
+            elif pop_jump_if == "POP_JUMP_IF_TRUE":
+                # Make sure pop_jump_if doesn't jump inside the "then" part of the "if"
+                # print("WOOT", pop_jump_if.attr - endif_offset)
+                # We leave some slop for endif_offset being one instruction behind.
+
+                return not ((pop_jump_if.attr - endif_offset) in (0, 2))
+        pass
+
+    # If there is a final COME_FROM and that test jumps to that, this is a strong
+    # indication that this is ok, so we'll skip jumps jumping too far test.
+    if (
+        pop_jump_if is not None
+        and ltm1 == "COME_FROM"
+        and ltm1.attr == pop_jump_if.off2int()
+    ):
+        return False
+
+    # Make sure jumps don't extend beyond the end of the if statement.
+    # This is done after the weird stuff above. There is a problem with the
+    # below is that it suffers from example A the "if b" jumping around
+    # the outer else. So we do this after all of the above and
+    # rely on the above COME_FROM test.
+
+    last_offset = tokens[last].off2int()
+    for i in range(first, last):
+        t = tokens[i]
+        # instead of POP_JUMP_IF, should we use op attributes?
+        if t.kind.startswith("POP_JUMP_IF_"):
+            pjif_target = t.attr
+            if pjif_target > last_offset:
+                # In some cases, where we have long bytecode, a
+                # "POP_JUMP_IF_TRUE/FALSE" offset might be too
+                # large for the instruction; so instead it
+                # jumps to a JUMP_FORWARD. Allow that here.
+                if tokens[last] == "JUMP_FORWARD":
+                    return tokens[last].attr != pjif_target
+                return True
+            # elif lhs == "ifstmtc" and tokens[first].off2int() > pjif_target:
+            #     # A conditional JUMP to the loop is expected for "ifstmtc"
+            #     return True
+            pass
+        pass
+
+    # If the "if_stmt" includes a COME_FROM from before the beginning of the "if", then
+    # no good. If the "if stmt" covers the non-COME_FROM instructions, there will have
+    # been a prior reduction that doesn't include the last COME_FROM.
+    if ltm1 == "COME_FROM":
+        return ltm1.attr < first_offset
+    elif tokens[last] == "COME_FROM":
+        return tokens[last].attr < first_offset
+
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/ifstmts_jump.py

@@ -0,0 +1,51 @@
+#  Copyright (c) 2020, 2023 Rocky Bernstein
+
+from decompyle3.scanners.tok import Token
+
+
+def ifstmts_jump_invalid(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+
+    come_froms = tree[-1]
+    # This is complicated, but note that the JUMP_IF instruction comes immediately
+    # *before* _ifstmts_jump so that's what we have to test
+    # the COME_FROM against. This can be complicated by intervening
+    # POP_TOP, and pseudo COME_FROM, ELSE instructions
+    #
+    pop_jump_index = first - 1
+    while pop_jump_index > 0 and tokens[pop_jump_index] in (
+        "POP_TOP",
+        "JUMP_FORWARD",
+        "COME_FROM",
+    ):
+        pop_jump_index -= 1
+
+    if pop_jump_index == 0:
+        return True
+
+    jump_token = tokens[pop_jump_index]
+    if jump_token.op not in self.opc.JUMP_OPS:
+        return False
+
+    # FIXME: something is fishy when and EXTENDED ARG is needed before the
+    # pop_jump_index instruction to get the argument. In this case, the
+    # _ifsmtst_jump can jump to a spot beyond the come_froms.
+    # That is going on in the non-EXTENDED_ARG case is that the POP_JUMP_IF
+    # jumps to a JUMP_(FORWARD) which is changed into an EXTENDED_ARG POP_JUMP_IF
+    # to the jumped forwarded address
+    if jump_token.attr > 256:
+        return False
+
+    pop_jump_offset = jump_token.off2int(prefer_last=False)
+    if isinstance(come_froms, Token):
+        if jump_token.attr < pop_jump_offset and tree[0] != "pass":
+            # This is a jump backwards to a loop. All bets are off here when there the
+            # unless statement is "pass" which has no instructions associated with it.
+            return False
+        return come_froms.attr is not None and pop_jump_offset > come_froms.attr
+
+    elif len(come_froms) == 0:
+        return False
+    else:
+        return pop_jump_offset > come_froms[-1].attr

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/joined_str_check.py

@@ -0,0 +1,47 @@
+#  Copyright (c) 2022 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def joined_str_invalid(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    # In Python 3.8, there is a new "=" specifier.
+    # See https://docs.python.org/3/whatsnew/3.8.html#f-strings-support-for-self-documenting-expressions-and-debugging
+    # We detect this here inside joined_str by looking for an
+    # expr->LOAD_STR which has an "=" added at the end
+    # and is equal without the "=" to expr->formated_value2->LOAD_CONST
+    # converted to a string.
+    expr1 = tree[0]
+    if expr1 != "expr":
+        return False
+    load_str = expr1[0]
+    if load_str != "LOAD_STR":
+        return False
+    format_value_equal = load_str.attr
+    if format_value_equal[-1] != "=":
+        return False
+    expr2 = tree[1]
+    if expr2 != "expr":
+        return False
+    formatted_value = expr2[0]
+    if not formatted_value.kind.startswith("formatted_value"):
+        return False
+    expr2a = formatted_value[0]
+    if expr2a != "expr":
+        return False
+    load_const = expr2a[0]
+    if load_const == "LOAD_CONST":
+        format_value2 = load_const.attr
+        return str(format_value2) == format_value_equal[:-1]
+    return True

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/lastc_stmt.py

@@ -0,0 +1,41 @@
+#  Copyright (c) 2020 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def lastc_stmt(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    # A lastc_stmt really has to be the last thing in a block,
+    # a statement that doesn't fall through to the next instruction, or
+    # in the case of "POP_BLOCK" is about to end.
+    # Otherwise this kind of stmt should flow through to the next.
+    # However that larger, set of stmts could be a lastc_stmt, but come back
+    # here with that larger set of stmts.
+
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    if tokens[last] == "COME_FROM":
+        last -= 1
+
+    # FIXME: use instruction properties.
+    return tokens[last] not in (
+        "BREAK_LOOP",
+        "COME_FROM_LOOP",
+        "CONTINUE",
+        "JUMP_LOOP",
+        "POP_BLOCK",
+        "RETURN_VALUE",
+    )

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/list_if_not.py

@@ -0,0 +1,33 @@
+#  Copyright (c) 2020-2022 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def list_if_not(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    assert rule[1][:-1] == ("expr", "list_if_not_end", "list_iter")
+    pop_jump_if = tree[1][0][0]
+    assert pop_jump_if.kind.startswith("POP_JUMP_IF_TRUE")
+    # The jump should not be somewhere inside the list_if_not,
+    # unless the list_iter is another "list_if"
+    if (
+        tokens[first].off2int(prefer_last=False)
+        < pop_jump_if.attr
+        < tokens[last].off2int(prefer_last=True)
+    ):
+        list_iter = tree[2]
+        assert list_iter == "list_iter"
+        return list_iter[0] != "list_if"
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/not_or_check.py

@@ -0,0 +1,54 @@
+#  Copyright (c) 2020, 2022 Rocky Bernstein
+
+# NOTE: this seems only used in 3.7. And I am not sure it is needed
+def not_or_check(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+
+    # Note (exp1 and exp2) and (not exp1 or exp2) are close, especially in
+    # an control structure like an "if".
+    # "exp1 and exp2":
+    #   exp1; POP_JUMP_IF_FALSE endif; exp2; POP_JUMP_IF_FALSE endif; then
+    #
+    # "not exp1 or exp2":
+    #   exp1; POP_JUMP_IF_FALSE then; exp2 POP_JUMP_IF_FALSE endif; then
+
+    # The difference is whether the POP_JUMPs go to the same place or not.
+
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    expr_pjif = tree[0]
+
+    end_token = tokens[last - 1]
+    jump_offset = end_token.attr
+    if end_token.kind.startswith("POP_JUMP_IF_FALSE"):
+
+        while expr_pjif == "and_parts":
+            expr_pjif = expr_pjif[0]
+            pass
+        assert expr_pjif == "expr_pjif"
+        if expr_pjif[-1].attr != jump_offset:
+            return True
+
+        # More "and" in a condition vs. "not or":
+        # Intuitively it has to do with where we go with the "and" or
+        # "not or". Right now if there are loop jumps involved
+        # we are saying this is "and", but this empirical and not on
+        # solid ground.
+
+        # If test jump is a backwards then, we have an "and", not a "not or".
+        first_offset = tokens[first].off2int()
+        if end_token.attr < first_offset:
+            return True
+        # Similarly if the test jump goes to another jump it is (probably?) an "and".
+        jump_target_inst_index = self.offset2inst_index[jump_offset]
+        inst = self.insts[jump_target_inst_index - 1]
+        if inst.is_jump:
+            return True
+
+        # Check that the jump is *around* the "then", not to the "then".
+        return jump_offset != tokens[last].offset
+        pass
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/or_check.py

@@ -0,0 +1,75 @@
+#  Copyright (c) 2020, 2022-2023 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# FIXME: we need to distinguish "or" as an expression which doesn't have
+# a "POP" instruction and "or" as a condition which does have the "POP"
+# instruction. Until then we use NOT_POP_FOLLOW_UPS as a hack to distinguish the
+# two
+NOT_POP_FOLLOW_OPS = frozenset(
+    """
+LOAD_ASSERT RAISE_VARARGS_1 STORE_FAST STORE_DEREF STORE_GLOBAL STORE_ATTR STORE_NAME
+""".split()
+)
+
+
+def or_check37_invalid(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+
+    expr_pjit = ast[0]
+    if expr_pjit in ("expr_pjit", "or_parts"):
+
+        if expr_pjit == "or_parts":
+            expr_pjit = expr_pjit[0]
+        if expr_pjit != "expr_pjit":
+            return False
+
+        # See FIXME: above
+        if tokens[last] in NOT_POP_FOLLOW_OPS or tokens[last - 1] in NOT_POP_FOLLOW_OPS:
+            return True
+
+        # The following test needs to prevent "or" from being
+        # mistaken for part of an "assert"t statement.
+
+        # The below then is useful until we get better control-flow analysis.
+        # Note it is too hard in the scanner right nowto turn the LOAD_GLOBAL into
+        # into LOAD_ASSERT. However in 3.9ish code generation does this by default.
+        load_global = tokens[last - 1]
+        if load_global == "LOAD_GLOBAL" and load_global.attr == "AssertionError":
+            return True
+
+        first_offset = tokens[first].off2int()
+        jump_if_true_target = expr_pjit[1].attr
+        if jump_if_true_target < first_offset:
+            return False
+
+        jump_if_false = tokens[last]
+        # If the jmp is backwards
+        if jump_if_false.kind.startswith("POP_JUMP_IF_FALSE"):
+            jump_if_false_offset = jump_if_false.off2int()
+            if jump_if_false == "POP_JUMP_IF_FALSE_LOOP":
+                # For a backwards loop, well compare to the instruction *after*
+                # then POP_JUMP...
+                jump_if_false = tokens[last + 1]
+            return not (
+                (
+                    jump_if_false_offset
+                    <= jump_if_true_target
+                    <= jump_if_false_offset + 2
+                )
+                or jump_if_true_target < tokens[first].off2int()
+            )
+
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/or_cond_check.py

@@ -0,0 +1,28 @@
+#  Copyright (c) 2020, 2023-2024 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def or_cond_check_invalid(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    if rule == ("or_cond", ("or_parts", "expr_pjif", "come_froms")):
+        if tokens[last - 1] == "COME_FROM":
+            return tokens[last - 1].attr < tokens[first].off2int()
+    last_offset = tokens[last].off2int()
+    for i in range(first, last):
+        t = tokens[i]
+        if t.kind.startswith("POP_JUMP_IF"):
+            if t.attr > last_offset:
+                return True
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/pop_return.py

@@ -0,0 +1,11 @@
+#  Copyright (c) 2020 Rocky Bernstein
+
+
+def pop_return_check(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+
+    # If the first instruction of return_expr (the instruction after POP_TOP) is
+    # has a linestart, then the POP_TOP was probably part of the previous
+    # statement, such as a call() where the return value is discarded.
+    return tokens[first + 1].linestart

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/testtrue.py

@@ -0,0 +1,22 @@
+#  Copyright (c) 2020 Rocky Bernstein
+
+
+def testtrue(self, lhs: str, n: int, rule, ast, tokens, first: int, last: int) -> bool:
+    # FIXME: make this work for all versions
+    if self.version != 3.7:
+        return False
+    if rule == ("testtrue", ("expr", "POP_JUMP_IF_TRUE")):
+        pjit = tokens[min(last - 1, n - 2)]
+        # If we have a backwards (looping) jump then this is
+        # really a testfalse. But "asserts" work funny
+        if pjit == "POP_JUMP_IF_TRUE_LOOP":
+            assert_next = tokens[min(last + 1, n - 1)]
+            return assert_next != "RAISE_VARARGS_1"
+    elif rule == ("testfalsec", ("expr", "POP_JUMP_IF_TRUE")):
+        pjit = tokens[min(last - 1, n - 2)]
+        # If we have a backwards (looping) jump then this is
+        # really a testtrue. But "asserts" work funny
+        if pjit.kind == "POP_JUMP_IF_TRUE_LOOP":
+            assert_next = tokens[min(last + 1, n - 1)]
+            return assert_next == "RAISE_VARARGS_1"
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/tryexcept.py

@@ -0,0 +1,76 @@
+#  Copyright (c) 2020 Rocky Bernstein
+
+
+def tryexcept(self, lhs, n, rule, ast, tokens, first, last):
+    come_from_except = ast[-1]
+    if rule in (
+        (
+            "try_except",
+            (
+                "SETUP_EXCEPT",
+                "suite_stmts_opt",
+                "POP_BLOCK",
+                "except_handler",
+                "opt_come_from_except",
+            ),
+        ),
+        (
+            "c_try_except",
+            (
+                "SETUP_EXCEPT",
+                "c_suite_stmts",
+                "POP_BLOCK",
+                "c_except_handler",
+                "opt_come_from_except",
+            ),
+        ),
+    ):
+        if come_from_except[0] == "COME_FROM":
+            # There should be at least two COME_FROMs, one from an
+            # exception handler and one from the try. Otherwise
+            # we have a try/else.
+            return True
+        pass
+
+    elif rule == (
+        "try_except",
+        (
+            "SETUP_EXCEPT",
+            "suite_stmts_opt",
+            "POP_BLOCK",
+            "except_handler",
+            "\\e_opt_come_from_except",
+        ),
+    ):
+        # Find END_FINALLY.
+        for i in range(last, first, -1):
+            if tokens[i] == "END_FINALLY":
+                jump_before_finally = tokens[i - 1]
+                if jump_before_finally.kind.startswith("JUMP"):
+                    if jump_before_finally == "JUMP_FORWARD":
+                        # If there is a JUMP_FORWARD before
+                        # the END_FINALLY to some jumps place
+                        # beyond tokens[last].off2int() then
+                        # this is a try/else rather than an
+                        # try (no else).
+                        return tokens[i - 1].attr > tokens[last].off2int(
+                            prefer_last=True
+                        )
+                    elif jump_before_finally == "JUMP_LOOP":
+                        # If there is a JUMP_LOOP before the
+                        # END_FINALLY then this is a looping
+                        # jump, but then jumps in the except
+                        # handlers have to also be a looping
+                        # jump or this is a try/else rather
+                        # than an try (no else).
+                        except_handler = ast[3]
+                        if (
+                            except_handler == "except_handler"
+                            and except_handler[0] == "JUMP_FORWARD"
+                        ):
+                            return True
+                        return False
+                    pass
+                pass
+            pass
+        return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/while1elsestmt.py

@@ -0,0 +1,25 @@
+#  Copyright (c) 2020-2021 Rocky Bernstein
+
+
+def while1elsestmt(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    if last == n:
+        # Adjust for fuzziness in parsing
+        last -= 1
+
+    if tokens[last] == "COME_FROM_LOOP":
+        last -= 1
+    elif tokens[last - 1] == "COME_FROM_LOOP":
+        last -= 2
+    if tokens[last] in ("JUMP_LOOP", "CONTINUE"):
+        # These indicate inside a loop, but token[last]
+        # should not be in a loop.
+        # FIXME: Not quite right: refine by using target
+        return True
+
+    # if SETUP_LOOP target spans the else part, then this is
+    # not while1else. Also do for whileTrue?
+    last += 1
+    # 3.8+ Doesn't have SETUP_LOOP
+    return self.version < (3, 8) and tokens[first].attr > tokens[last].off2int()

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/while1stmt.py

@@ -0,0 +1,52 @@
+#  Copyright (c) 2020, 2022 Rocky Bernstein
+
+
+def while1stmt(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+
+    # If there is a fall through to the COME_FROM_LOOP, then this is
+    # not a while 1. So the instruction before should either be a
+    # JUMP_LOOP or the instruction before should not be the target of a
+    # jump. (Well that last clause i not quite right; that target could be
+    # from dead code. Ugh. We need a more uniform control flow analysis.)
+    if last == n or tokens[last - 1] == "COME_FROM_LOOP":
+        cfl = last - 1
+    else:
+        cfl = last
+    assert tokens[cfl] == "COME_FROM_LOOP"
+
+    for loop_end in range(cfl - 1, first, -1):
+        if tokens[loop_end] != "POP_BLOCK":
+            break
+    if tokens[loop_end].kind not in ("JUMP_LOOP", "RETURN_VALUE", "RAISE_VARARGS_1"):
+        if not tokens[loop_end].kind.startswith("COME_FROM"):
+            return True
+    # Check that the SETUP_LOOP jumps to the offset after the
+    # COME_FROM_LOOP
+    if 0 <= last and tokens[last] in ("COME_FROM_LOOP", "JUMP_LOOP"):
+        # jump_back should be right before COME_FROM_LOOP?
+        last += 1
+    if last == n:
+        last -= 1
+    offset = tokens[last].off2int()
+    assert tokens[first] == "SETUP_LOOP"
+
+    # Scan for jumps out of the loop. Skip the initial "SETUP_LOOP" instruction.
+    # If there is a JUMP_LOOP at the end, jumping to that is not breaking out
+    # of the loop. However after that, any "POP_BLOCK"s or "COME_FROM_LOOP"s
+    # are considered to break out of the loop.
+    if tokens[loop_end] == "JUMP_LOOP":
+        loop_end += 1
+    loop_end_offset = tokens[loop_end].off2int(prefer_last=False)
+    for t in range(first + 1, loop_end):
+        token = tokens[t]
+        # token could be a pseudo-op like "LOAD_STR", which is not in
+        # token.opc.  We will replace that with LOAD_CONST as an
+        # example of an instruction that is not in token.opc.JUMP_OPS
+        if token.opc.opmap.get(token.kind, "LOAD_CONST") in token.opc.JUMP_OPS:
+            if token.attr >= loop_end_offset:
+                return True
+
+    # SETUP_LOOP location must jump either to the last token or the token after the last one
+    return tokens[first].attr not in (offset, offset + 2)

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/whileTruestmt38.py

@@ -0,0 +1,73 @@
+#  Copyright (c) 2022-2023 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def whileTruestmt38_check(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    # When we are missing a COME_FROM_LOOP, the
+    # "while" statement is nested inside an if/else
+    # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
+    # Check this.
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    if not tokens[last].kind.startswith("COME_FROM") and tokens[
+        last - 1
+    ].kind.startswith("COME_FROM"):
+        last -= 1
+    while tokens[last].kind.startswith("COME_FROM"):
+        last -= 1
+    if rule[-1][-1] == "\\e__come_froms":
+        jump_loop = tree[-2]
+    else:
+        # This might not be needed
+        jump_loop = tokens[last]
+    if jump_loop == "JUMP_LOOP":
+        jump_target = jump_loop.attr
+        if jump_target < tokens[first].off2int(prefer_last=False):
+            return True
+
+        c_stmts = tree[1]
+        if c_stmts == "c_stmts":
+            # Distinguish:
+            #   while True:
+            #      if expr:
+            # from:
+            #   while expr:
+            #
+            # We distinguish by checking to see if the "if expr" jumps *outside* of
+            # the loop bound.
+
+            # First, see if we have "ifstmt" as the first statement inside "while True"
+            c_stmts_offset = c_stmts.first_child().off2int()
+            first_stmt = c_stmts[0]
+            while first_stmt in ("_stmts", "stmts"):
+                first_stmt = first_stmt[0]
+            if first_stmt == "ifstmt":
+                # Next check for a testexpr and get the last instruction of that
+                testexpr = first_stmt[0]
+                if testexpr == "testexpr":
+                    pop_jump_if = testexpr.last_child()
+                    # Do we have POP_JUMP_IF with a jump outside of the loop?
+                    if (
+                        pop_jump_if.kind.startswith("POP_JUMP_IF")
+                        and pop_jump_if.attr > tokens[last].off2int()
+                    ):
+                        # Fail here, but we expect a "while expr" pattern to succeed elsewhere.
+                        return True
+            return c_stmts_offset != jump_target
+
+    return False

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/whilestmt.py

@@ -0,0 +1,31 @@
+#  Copyright (c) 2020 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def whilestmt(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    # When we are missing a COME_FROM_LOOP, the
+    # "while" statement is nested inside an if/else
+    # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
+    # Check this.
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    return tokens[last - 1] == "POP_BLOCK" and tokens[last] not in (
+        "JUMP_FORWARD",
+        "COME_FROM_LOOP",
+        "COME_FROM",
+    )

python/py/Lib/site-packages/decompyle3/parsers/reduce_check/whilestmt38.py

@@ -0,0 +1,41 @@
+#  Copyright (c) 2022 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def whilestmt38_check(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    # When we are missing a COME_FROM_LOOP, the
+    # "while" statement is nested inside an if/else
+    # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
+    # Check this.
+    # print("XXX", first, last, rule)
+    # for t in range(first, last):
+    #     print(tokens[t])
+    # print("=" * 40)
+
+    if tokens[last] != "COME_FROM" and tokens[last - 1] == "COME_FROM":
+        last -= 1
+    if tokens[last - 1].kind.startswith("RAISE_VARARGS"):
+        return True
+    while tokens[last] == "COME_FROM":
+        last -= 1
+    # In a "while" loop, (in contrast to "for" loop), the loop jump is
+    # always to the first offset
+    first_offset = tokens[first].off2int()
+    if tokens[last] == "JUMP_LOOP" and (
+        tokens[last].attr == first_offset or tokens[last - 1].attr == first_offset
+    ):
+        return False
+    return True

python/py/Lib/site-packages/decompyle3/parsers/treenode.py

@@ -0,0 +1,77 @@
+import sys
+from decompyle3.scanners.tok import NoneToken, Token
+from spark_parser.ast import AST as spark_AST
+
+intern = sys.intern
+
+
+class SyntaxTree(spark_AST):
+    def __init__(self, *args, transformed_by=None, **kwargs):
+        self.transformed_by = transformed_by
+        super(SyntaxTree, self).__init__(*args, **kwargs)
+
+    def isNone(self):
+        """An SyntaxTree None token. We can't use regular list comparisons
+        because SyntaxTree token offsets might be different"""
+        return len(self.data) == 1 and NoneToken == self.data[0]
+
+    def __repr__(self) -> str:
+        return self.__repr1__("", None)
+
+    def __repr1__(self, indent, sibNum=None) -> str:
+        rv = str(self.kind)
+        if sibNum is not None:
+            rv = "%2d. %s" % (sibNum, rv)
+        enumerate_children = False
+        if len(self) > 1:
+            rv += " (%d)" % (len(self))
+            enumerate_children = True
+        if self.transformed_by is not None:
+            rv += f" (transformed by {self.transformed_by})"
+            pass
+        rv = indent + rv
+        indent += "    "
+        i = 0
+        for node in self:
+            if hasattr(node, "__repr1__"):
+                if enumerate_children:
+                    child = node.__repr1__(indent, i)
+                else:
+                    child = node.__repr1__(indent, None)
+            else:
+                inst = node.format(line_prefix="")
+                if inst.startswith("\n"):
+                    # Nuke leading \n
+                    inst = inst[1:]
+                if enumerate_children:
+                    child = indent + "%2d. %s" % (i, inst)
+                else:
+                    child = indent + inst
+                pass
+            rv += "\n" + child
+            i += 1
+        return rv
+
+    def first_child(self):
+        for child in self:
+            if isinstance(child, Token):
+                return child
+            if child is not None:
+                child = child.first_child()
+                if isinstance(child, Token):
+                    return child
+                pass
+        return None
+
+    def last_child(self):
+        if len(self) > 0:
+            child_index = -1
+            child = self[-1]
+            while isinstance(child, SyntaxTree) and len(child) == 0:
+                # Skip over empty nonterminal reductions
+                child_index -= 1
+                child = self[child_index]
+            if not isinstance(child, SyntaxTree):
+                return child
+            return child.last_child()
+        return self

python/py/Lib/site-packages/decompyle3/scanner.py

@@ -0,0 +1,589 @@
+#  Copyright (c) 2016, 2018-2021, 2024-2025 by Rocky Bernstein
+#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 1999 John Aycock
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+"""
+scanner/ingestion module. From here we call various version-specific
+scanners, e.g. for Python 3.7 or 3.8.
+"""
+
+import importlib
+from abc import ABC
+from array import array
+from collections import namedtuple
+from types import ModuleType
+from typing import Optional, Union
+
+import xdis
+from xdis import (
+    Bytecode,
+    canonic_python_version,
+    code2num,
+    extended_arg_val,
+    instruction_size,
+    next_offset,
+)
+from xdis.version_info import IS_PYPY, version_tuple_to_str
+
+from decompyle3.scanners.tok import Token
+
+# The byte code versions we support.
+# Note: these all have to be tuples
+PYTHON_VERSIONS = frozenset(((3, 7), (3, 8)))
+
+CANONIC2VERSION = dict(
+    (canonic_python_version[version_tuple_to_str(python_version)], python_version)
+    for python_version in PYTHON_VERSIONS
+)
+
+L65536 = 65536
+
+
+def long(num):
+    return num
+
+
+class Code:
+    """
+    Class for representing code-objects.
+
+    This is similar to the original code object, but additionally
+    the diassembled code is stored in the attribute '_tokens'.
+    """
+
+    def __init__(self, co, scanner, classname=None, show_asm=None):
+        # Full initialization is given below, but for linters
+        # well set up some initial values.
+        self.co_code = None  # Really either bytes for >= 3.0 and string in < 3.0
+
+        for i in dir(co):
+            if i.startswith("co_"):
+                setattr(self, i, getattr(co, i))
+        self._tokens, self._customize = scanner.ingest(co, classname, show_asm=show_asm)
+
+
+class Scanner(ABC):
+    def __init__(self, version: tuple, show_asm=None, is_pypy=False):
+        self.version = version
+        self.show_asm = show_asm
+        self.is_pypy = is_pypy
+
+        # Temporary initialization.
+        self.opc = ModuleType("uninitialized")
+
+        if version[:2] in PYTHON_VERSIONS:
+            v_str = f"""opcode_{version_tuple_to_str(version, start=0, end=2, delimiter="")}"""
+            module_name = f"xdis.opcodes.{v_str}"
+            if is_pypy:
+                module_name += "pypy"
+            self.opc = importlib.import_module(module_name)
+        else:
+            raise TypeError(
+                f"{version_tuple_to_str(version)} is not a Python version I know about"
+            )
+
+        self.opname = self.opc.opname
+
+        # FIXME: This weird Python2 behavior is not Python3
+        self.resetTokenClass()
+
+    def build_instructions(self, co):
+        """
+        Create a list of instructions (a structured object rather than
+        an array of bytes) and store that in self.insts
+        """
+        # FIXME: remove this when all subsidiary functions have been removed.
+        # We should be able to get everything from the self.insts list.
+        self.code = array("B", co.co_code)
+
+        bytecode = Bytecode(co, self.opc)
+        self.build_prev_op()
+        self.insts = self.remove_extended_args(list(bytecode))
+        self.lines = self.build_lines_data(co)
+        self.offset2inst_index = {}
+        for i, inst in enumerate(self.insts):
+            self.offset2inst_index[inst.offset] = i
+            offset = inst.offset
+            inst_size = inst.inst_size
+            while inst_size > 0:
+                self.offset2inst_index[offset] = i
+                offset += 2
+                inst_size -= 2
+
+        return bytecode
+
+    def build_lines_data(self, code_obj):
+        """
+        Generate various line-related helper data.
+        """
+
+        # Offset: lineno pairs, only for offsets which start line.
+        # Locally we use list for more convenient iteration using indices
+        linestarts = list(self.opc.findlinestarts(code_obj))
+        self.linestarts = dict(linestarts)
+        if not self.linestarts:
+            return []
+
+        # 'List-map' which shows line number of current op and offset of
+        # first op on following line, given offset of op as index
+        lines = []
+        LineTuple = namedtuple("LineTuple", ["l_no", "next"])
+
+        # Iterate through available linestarts, and fill
+        # the data for all code offsets encountered until
+        # last linestart offset
+        _, prev_line_no = linestarts[0]
+        offset = 0
+        for start_offset, line_no in linestarts[1:]:
+            while offset < start_offset:
+                lines.append(LineTuple(prev_line_no, start_offset))
+                offset += 1
+            prev_line_no = line_no
+
+        # Fill remaining offsets with reference to last line number
+        # and code length as start offset of following non-existing line
+        codelen = len(self.code)
+        while offset < codelen:
+            lines.append(LineTuple(prev_line_no, codelen))
+            offset += 1
+        return lines
+
+    def build_prev_op(self):
+        """
+        Compose 'list-map' which allows to jump to previous
+        op, given offset of current op as index.
+        """
+        code = self.code
+        codelen = len(code)
+        # 2.x uses prev 3.x uses prev_op. Sigh
+        # Until we get this sorted out.
+        self.prev = self.prev_op = [0]
+        for offset in self.op_range(0, codelen):
+            op = code[offset]
+            for _ in range(instruction_size(op, self.opc)):
+                self.prev_op.append(offset)
+
+    def is_jump_forward(self, offset: int) -> bool:
+        """
+        Return True if the code at offset is some sort of jump forward.
+        That is, it is ether "JUMP_FORWARD" or an absolute jump that
+        goes forward.
+        """
+        opname = self.get_inst(offset).opname
+        if opname == "JUMP_FORWARD":
+            return True
+        if opname != "JUMP_ABSOLUTE":
+            return False
+        return offset < self.get_target(offset)
+
+    def ingest(self, co, classname=None, code_objects={}, show_asm=None):
+        """
+        Code to tokenize disassembly. Subclasses must implement this.
+        """
+        raise NotImplementedError("This method should have been implemented")
+
+    def prev_offset(self, offset: int) -> int:
+        return self.insts[self.offset2inst_index[offset] - 1].offset
+
+    def get_inst(self, offset: int):
+        """
+        Returns the instruction from ``self.insts`` that has at offset
+        ``offset``.
+
+        Instructions can get moved as a result of ``EXTENDED_ARGS`` removal.
+        So if ``offset`` is not in self.offset2inst_index, then
+        we assume that it was an instruction moved back.
+        We check that assumption though by looking at
+        self.code's opcode.
+        Sadly instructions can get moved forward too.
+        So we have to check which direction we are going.
+        """
+        offset_increment = instruction_size(self.opc.EXTENDED_ARG, self.opc)
+        if offset not in self.offset2inst_index:
+            if self.code[offset] != self.opc.EXTENDED_ARG:
+                target_name = self.opc.opname[self.code[offset]]
+                # JUMP_ABSOLUTE can be like this where
+                # the inst offset is at what used to be an EXTENDED_ARG
+                # so find the first extended arg.
+                next_offset = offset - offset_increment
+                while next_offset not in self.offset2inst_index:
+                    next_offset -= offset_increment
+                    assert self.code[next_offset] == self.opc.EXTENDED_ARG
+                inst = self.insts[self.offset2inst_index[next_offset]]
+                assert inst.opname == target_name, inst
+            else:
+                next_offset = offset + offset_increment
+                while next_offset not in self.offset2inst_index:
+                    next_offset += offset_increment
+
+                inst = self.insts[self.offset2inst_index[next_offset]]
+
+            assert inst.has_extended_arg is True
+            return inst
+
+        return self.insts[self.offset2inst_index[offset]]
+
+    def get_target(self, offset: int, extended_arg: int = 0) -> int:
+        """
+        Get next instruction offset for op located at given <offset>.
+        NOTE: extended_arg is no longer used
+        """
+        inst = self.get_inst(offset)
+        if inst.opcode in self.opc.JREL_OPS | self.opc.JABS_OPS:
+            target = inst.argval
+        else:
+            # No jump offset, so use fall-through offset
+            target = next_offset(inst.opcode, self.opc, inst.offset)
+        return target
+
+    def get_argument(self, pos: int):
+        arg = self.code[pos + 1] + self.code[pos + 2] * 256
+        return arg
+
+    def next_offset(self, op, offset: int) -> int:
+        return xdis.next_offset(op, self.opc, offset)
+
+    def first_instr(self, start: int, end: int, instr, target=None, exact=True):
+        """
+        Find the first <instr> in the block from start to end.
+        <instr> is any python bytecode instruction or a list of opcodes
+        If <instr> is an opcode with a target (like a jump), a target
+        destination can be specified which must match precisely if exact
+        is True, or if exact is False, the instruction which has a target
+        closest to <target> will be returned.
+
+        Return index to it or None if not found.
+        """
+        code = self.code
+        assert start >= 0 and end <= len(code)
+
+        if not isinstance(instr, list):
+            instr = [instr]
+
+        result_offset = None
+        current_distance = len(code)
+        for offset in self.op_range(start, end):
+            op = code[offset]
+            if op in instr:
+                if target is None:
+                    return offset
+                dest = self.get_target(offset)
+                if dest == target:
+                    return offset
+                elif not exact:
+                    new_distance = abs(target - dest)
+                    if new_distance < current_distance:
+                        current_distance = new_distance
+                        result_offset = offset
+        return result_offset
+
+    def last_instr(
+        self, start: int, end: int, instr, target=None, exact=True
+    ) -> Optional[int]:
+        """
+        Find the last <instr> in the block from start to end.
+        <instr> is any python bytecode instruction or a list of opcodes
+        If <instr> is an opcode with a target (like a jump), a target
+        destination can be specified which must match precisely if exact
+        is True, or if exact is False, the instruction which has a target
+        closest to <target> will be returned.
+
+        Return index to it or None if not found.
+        """
+
+        code = self.code
+        # Make sure requested positions do not go out of
+        # code bounds
+        if not (start >= 0 and end <= len(code)):
+            return None
+
+        if not isinstance(instr, list):
+            instr = [instr]
+
+        result_offset = None
+        current_distance = self.insts[-1].offset - self.insts[0].offset
+        extended_arg = 0
+        # FIXME: use self.insts rather than code[]
+        for offset in self.op_range(start, end):
+            op = code[offset]
+
+            if op == self.opc.EXTENDED_ARG:
+                arg = code2num(code, offset + 1) | extended_arg
+                extended_arg = extended_arg_val(self.opc, arg)
+                continue
+
+            if op in instr:
+                if target is None:
+                    result_offset = offset
+                else:
+                    dest = self.get_target(offset, extended_arg)
+                    if dest == target:
+                        current_distance = 0
+                        result_offset = offset
+                    elif not exact:
+                        new_distance = abs(target - dest)
+                        if new_distance <= current_distance:
+                            current_distance = new_distance
+                            result_offset = offset
+                            pass
+                        pass
+                    pass
+                pass
+            extended_arg = 0
+            pass
+        return result_offset
+
+    def inst_matches(self, start, end, instr, target=None, include_beyond_target=False):
+        """
+        Find all `instr` in the block from start to end.
+        `instr` is a Python opcode or a list of opcodes
+        If `instr` is an opcode with a target (like a jump), a target
+        destination can be specified which must match precisely.
+
+        Return a list with indexes to them or [] if none found.
+        """
+        try:
+            None in instr
+        except Exception:
+            instr = [instr]
+
+        first = self.offset2inst_index[start]
+        result = []
+        for inst in self.insts[first:]:
+            if inst.opcode in instr:
+                if target is None:
+                    result.append(inst.offset)
+                else:
+                    t = self.get_target(inst.offset)
+                    if include_beyond_target and t >= target:
+                        result.append(inst.offset)
+                    elif t == target:
+                        result.append(inst.offset)
+                        pass
+                    pass
+                pass
+            if isinstance(inst.offset, int) and inst.offset >= end:
+                break
+            pass
+
+        # FIXME: put in a test
+        # check = self.all_instr(start, end, instr, target, include_beyond_target)
+        # assert result == check
+
+        return result
+
+    # FIXME: this is broken on 3.6+. Replace remaining (2.x-based) calls
+    # with inst_matches
+
+    def all_instr(
+        self, start: int, end: int, instr, target=None, include_beyond_target=False
+    ):
+        """
+        Find all `instr` in the block from start to end.
+        `instr` is any Python opcode or a list of opcodes
+        If `instr` is an opcode with a target (like a jump), a target
+        destination can be specified which must match precisely.
+
+        Return a list with indexes to them or [] if none found.
+        """
+
+        code = self.code
+        assert start >= 0 and end <= len(code)
+
+        if not isinstance(instr, list):
+            instr = [instr]
+
+        result = []
+        extended_arg = 0
+        for offset in self.op_range(start, end):
+            op = code[offset]
+
+            if op == self.opc.EXTENDED_ARG:
+                arg = code2num(code, offset + 1) | extended_arg
+                extended_arg = extended_arg_val(self.opc, arg)
+                continue
+
+            if op in instr:
+                if target is None:
+                    result.append(offset)
+                else:
+                    t = self.get_target(offset, extended_arg)
+                    if include_beyond_target and t >= target:
+                        result.append(offset)
+                    elif t == target:
+                        result.append(offset)
+                        pass
+                    pass
+                pass
+            extended_arg = 0
+            pass
+
+        return result
+
+    def opname_for_offset(self, offset):
+        return self.opc.opname[self.code[offset]]
+
+    def op_name(self, op):
+        return self.opc.opname[op]
+
+    def op_range(self, start, end):
+        """
+        Iterate through positions of opcodes, skipping
+        arguments.
+        """
+        while start < end:
+            yield start
+            start += instruction_size(self.code[start], self.opc)
+
+    def remove_extended_args(self, instructions):
+        """Go through instructions removing extended ARG.
+        get_instruction_bytes previously adjusted the operand values
+        to account for these"""
+        new_instructions = []
+        last_was_extarg = False
+        n = len(instructions)
+        starts_line = False
+        for i, inst in enumerate(instructions):
+            if (
+                inst.opname == "EXTENDED_ARG"
+                and i + 1 < n
+                and instructions[i + 1].opname != "MAKE_FUNCTION"
+            ):
+                last_was_extarg = True
+                starts_line = inst.starts_line
+                is_jump_target = inst.is_jump_target
+                offset = inst.offset
+                continue
+            if last_was_extarg:
+                # j = self.stmts.index(inst.offset)
+                # self.lines[j] = offset
+
+                new_inst = inst._replace(
+                    starts_line=starts_line,
+                    is_jump_target=is_jump_target,
+                    offset=offset,
+                )
+                inst = new_inst
+                if i < n:
+                    new_prev = self.prev_op[instructions[i].offset]
+                    j = instructions[i + 1].offset
+                    old_prev = self.prev_op[j]
+                    while self.prev_op[j] == old_prev and j < n:
+                        self.prev_op[j] = new_prev
+                        j += 1
+
+            last_was_extarg = False
+            new_instructions.append(inst)
+        return new_instructions
+
+    def remove_mid_line_ifs(self, ifs):
+        """
+        Go through passed offsets, filtering ifs
+        located somewhere mid-line.
+        """
+
+        # FIXME: this doesn't work for Python 3.6+
+
+        filtered = []
+        for i in ifs:
+            # For each offset, if line number of current and next op
+            # is the same
+            if self.lines[i].l_no == self.lines[i + 3].l_no:
+                # Skip last op on line if it is some sort of POP_JUMP.
+                if self.code[self.prev[self.lines[i].next]] in (
+                    self.opc.PJIT,
+                    self.opc.PJIF,
+                ):
+                    continue
+            filtered.append(i)
+        return filtered
+
+    def resetTokenClass(self):
+        return self.setTokenClass(Token)
+
+    def restrict_to_parent(self, target: int, parent) -> int:
+        """Restrict target to parent structure boundaries."""
+        if not (parent["start"] < target < parent["end"]):
+            target = parent["end"]
+        return target
+
+    def setTokenClass(self, token_class: Token) -> Token:
+        self.Token = token_class
+        return self.Token
+
+
+def get_scanner(version: Union[str, tuple], is_pypy=False, show_asm=None) -> Scanner:
+    # If version is a string, turn that into the corresponding float.
+    if isinstance(version, str):
+        if version not in canonic_python_version:
+            raise RuntimeError(f"Unknown Python version in xdis {version}")
+        canonic_version = canonic_python_version[version]
+        if canonic_version not in CANONIC2VERSION:
+            raise RuntimeError(
+                f"Unsupported Python version {version} (canonic {canonic_version})"
+            )
+        version = CANONIC2VERSION[canonic_version]
+
+    # Pick up appropriate scanner
+    if version[:2] in PYTHON_VERSIONS:
+        v_str = version_tuple_to_str(version, start=0, end=2, delimiter="")
+        try:
+            import importlib
+
+            if is_pypy:
+                scan = importlib.import_module(f"decompyle3.scanners.pypy{v_str}")
+            else:
+                scan = importlib.import_module(f"decompyle3.scanners.scanner{v_str}")
+            if False:
+                print(scan)  # Avoid unused scan
+        except ImportError:
+            if is_pypy:
+                exec(
+                    f"import decompyle3.scanners.pypy{v_str} as scan",
+                    locals(),
+                    globals(),
+                )
+            else:
+                exec(
+                    f"import decompyle3.scanners.scanner{v_str} as scan",
+                    locals(),
+                    globals(),
+                )
+        if is_pypy:
+            scanner = eval(
+                f"scan.ScannerPyPy{v_str}(show_asm=show_asm)", locals(), globals()
+            )
+        else:
+            scanner = eval(
+                f"scan.Scanner{v_str}(show_asm=show_asm)", locals(), globals()
+            )
+    else:
+        raise RuntimeError(
+            "Unsupported Python version, "
+            f"{version_tuple_to_str(version)}, for decompilation"
+        )
+    return scanner
+
+
+if __name__ == "__main__":
+    import inspect
+
+    my_co = inspect.currentframe().f_code
+    from xdis.version_info import PYTHON_VERSION_TRIPLE
+
+    scanner = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY, True)
+    tokens, customize = scanner.ingest(my_co, {}, show_asm="after")

python/py/Lib/site-packages/decompyle3/scanners/__init__.py

@@ -0,0 +1,29 @@
+"""Here we have "scanners" for the different Python versions.
+"scanner" is a compiler-centric term, but it is really a bit different from
+a traditional  compiler scanner/lexer.
+
+Here we start out with text disasembly and change that to be more
+ameanable to parsing in which we look only at the opcode name, and not
+and instruction's operand.
+
+In some cases this is done by changing the opcode name. For example
+"LOAD_CONST" it customized based on the type of its operand into
+"LOAD_ASSERT", "LOAD_CODE", "LOAD_STR".
+
+instructions that take a variable number of arguments will have the argument count
+suffixed to the opcode name. "CALL", "MAKE_FUNCTION", "BUILD_TUPLE", "BUILD_LIST",
+work this way for example
+
+We also add pseudo instructions like "COME_FROM" which have an operand
+
+Instead of full grammars, we have full grammars for certain Python versions
+and the others indicate differences between a neighboring version.
+
+For example Python 2.6, 2.7, 3.2, and 3.7 are largely "base" versions
+which work off of scanner2.py, scanner3.py, and scanner37base.py.
+
+Some examples:
+Python 3.3 diffs off of 3.2; 3.1 and 3.0 diff off of 3.2; Python 1.0..Python 2.5 diff off of
+Python 2.6 and Python 3.8 diff off of 3.7
+
+"""

python/py/Lib/site-packages/decompyle3/scanners/pypy37.py

@@ -0,0 +1,26 @@
+#  Copyright (c) 2021 by Rocky Bernstein
+"""
+Python PyPy 3.7 decompiler scanner.
+
+Does some additional massaging of xdis-disassembled instructions to
+make things easier for decompilation.
+"""
+
+import decompyle3.scanners.scanner37 as scan
+
+# bytecode verification, verify(), uses JUMP_OPS from here
+from xdis.opcodes import opcode_37pypy as opc  # is this right?
+
+JUMP_OPs = opc.JUMP_OPS
+
+
+# We base this off of 3.7
+class ScannerPyPy37(scan.Scanner37):
+    def __init__(self, show_asm):
+        # There are no differences in initialization between
+        # pypy 3.7 and 3.7
+        scan.Scanner37.__init__(self, show_asm, is_pypy=True)
+        self.version = (3, 7)
+        self.opc = opc
+        self.is_pypy = True
+        return

python/py/Lib/site-packages/decompyle3/scanners/pypy38.py

@@ -0,0 +1,25 @@
+#  Copyright (c) 2021 by Rocky Bernstein
+"""
+Python PyPy 3.8 decompiler scanner.
+
+Does some additional massaging of xdis-disassembled instructions to
+make things easier for decompilation.
+"""
+
+import decompyle3.scanners.scanner38 as scan
+
+# bytecode verification, verify(), uses JUMP_OPS from here
+from xdis.opcodes import opcode_38pypy as opc
+
+JUMP_OPs = opc.JUMP_OPS
+
+
+# We base this off of 3.8
+class ScannerPyPy38(scan.Scanner38):
+    def __init__(self, show_asm):
+        # There are no differences in initialization between
+        # pypy 3.8 and 3.8
+        scan.Scanner38.__init__(self, show_asm, is_pypy=True)
+        self.version = (3, 8)
+        self.opc = opc
+        return