image-match/python/py/Lib/site-packages/wandb/jupyter.py

from __future__ import annotations

import json
import logging
import os
import re
import shutil
import sys
import traceback
from base64 import b64encode
from typing import Any

import IPython
import IPython.display
import requests
from IPython.core.magic import Magics, line_cell_magic, magics_class
from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring
from requests.compat import urljoin

import wandb
import wandb.util
from wandb.sdk import wandb_setup
from wandb.sdk.lib import filesystem

logger = logging.getLogger(__name__)


def display_if_magic_is_used(run: wandb.Run) -> bool:
    """Display a run's page if the cell has the %%wandb cell magic.

    Args:
        run: The run to display.

    Returns:
        Whether the %%wandb cell magic was present.
    """
    if not _current_cell_wandb_magic:
        return False

    _current_cell_wandb_magic.display_if_allowed(run)
    return True


class _WandbCellMagicState:
    """State for a cell with the %%wandb cell magic."""

    def __init__(self, *, height: int) -> None:
        """Initializes the %%wandb cell magic state.

        Args:
            height: The desired height for displayed iframes.
        """
        self._height = height
        self._already_displayed = False

    def display_if_allowed(self, run: wandb.Run) -> None:
        """Display a run's iframe if one is not already displayed.

        Args:
            run: The run to display.
        """
        if self._already_displayed:
            return
        self._already_displayed = True

        _display_wandb_run(run, height=self._height)


_current_cell_wandb_magic: _WandbCellMagicState | None = None


def _display_by_wandb_path(path: str, *, height: int) -> None:
    """Display a wandb object (usually in an iframe) given its URI.

    Args:
        path: A path to a run, sweep, project, report, etc.
        height: Height of the iframe in pixels.
    """
    api = wandb.Api()

    try:
        obj = api.from_path(path)

        IPython.display.display_html(
            obj.to_html(height=height),
            raw=True,
        )
    except wandb.Error:
        traceback.print_exc()
        IPython.display.display_html(
            f"Path {path!r} does not refer to a W&B object you can access.",
            raw=True,
        )


def _display_wandb_run(run: wandb.Run, *, height: int) -> None:
    """Display a run (usually in an iframe).

    Args:
        run: The run to display.
        height: Height of the iframe in pixels.
    """
    IPython.display.display_html(
        run.to_html(height=height),
        raw=True,
    )


@magics_class
class WandBMagics(Magics):
    def __init__(self, shell):
        super().__init__(shell)

    @magic_arguments()
    @argument(
        "path",
        default=None,
        nargs="?",
        help="The path to a resource you want to display.",
    )
    @argument(
        "-h",
        "--height",
        default=420,
        type=int,
        help="The height of the iframe in pixels.",
    )
    @line_cell_magic
    def wandb(self, line: str, cell: str | None = None) -> None:
        """Display wandb resources in Jupyter.

        This can be used as a line magic:

            %wandb USERNAME/PROJECT/runs/RUN_ID

        Or as a cell magic:

            %%wandb -h 1024
            with wandb.init() as run:
                run.log({"loss": 1})
        """
        global _current_cell_wandb_magic

        args = parse_argstring(self.wandb, line)
        path: str | None = args.path
        height: int = args.height

        if path:
            _display_by_wandb_path(path, height=height)
            displayed = True
        elif run := wandb_setup.singleton().most_recent_active_run:
            _display_wandb_run(run, height=height)
            displayed = True
        else:
            displayed = False

        # If this is being used as a line magic ("%wandb"), we are done.
        # When used as a cell magic ("%%wandb"), we must run the cell.
        if cell is None:
            return

        if not displayed:
            _current_cell_wandb_magic = _WandbCellMagicState(height=height)

        try:
            IPython.get_ipython().run_cell(cell)
        finally:
            _current_cell_wandb_magic = None


def notebook_metadata_from_jupyter_servers_and_kernel_id():
    # When running in VS Code's notebook extension,
    # the extension creates a temporary file to start the kernel.
    # This file is not actually the same as the notebook file.
    #
    # The real notebook path is stored in the user namespace
    # under the key "__vsc_ipynb_file__"
    try:
        from IPython import get_ipython

        ipython = get_ipython()
        if ipython is not None:
            notebook_path = ipython.kernel.shell.user_ns.get("__vsc_ipynb_file__")
            if notebook_path:
                return {
                    "root": os.path.dirname(notebook_path),
                    "path": notebook_path,
                    "name": os.path.basename(notebook_path),
                }
    except ModuleNotFoundError:
        return None

    servers, kernel_id = jupyter_servers_and_kernel_id()
    for s in servers:
        if s.get("password"):
            raise ValueError("Can't query password protected kernel")
        res = requests.get(
            urljoin(s["url"], "api/sessions"), params={"token": s.get("token", "")}
        ).json()
        for nn in res:
            if (
                isinstance(nn, dict)
                and nn.get("kernel")
                and "notebook" in nn
                and nn["kernel"]["id"] == kernel_id
            ):
                return {
                    "root": s.get("root_dir", s.get("notebook_dir", os.getcwd())),
                    "path": nn["notebook"]["path"],
                    "name": nn["notebook"]["name"],
                }

    if not kernel_id:
        return None


def notebook_metadata(silent: bool) -> dict[str, str]:
    """Attempt to query jupyter for the path and name of the notebook file.

    This can handle different jupyter environments, specifically:

    1. Colab
    2. Kaggle
    3. JupyterLab
    4. Notebooks
    5. Other?
    """
    error_message = (
        "Failed to detect the name of this notebook. You can set it manually"
        " with the WANDB_NOTEBOOK_NAME environment variable to enable code"
        " saving."
    )
    try:
        jupyter_metadata = notebook_metadata_from_jupyter_servers_and_kernel_id()

        # Colab:
        # request the most recent contents
        ipynb = attempt_colab_load_ipynb()
        if ipynb is not None and jupyter_metadata is not None:
            return {
                "root": "/content",
                "path": jupyter_metadata["path"],
                "name": jupyter_metadata["name"],
            }

        # Kaggle:
        if wandb.util._is_kaggle():
            # request the most recent contents
            ipynb = attempt_kaggle_load_ipynb()
            if ipynb:
                return {
                    "root": "/kaggle/working",
                    "path": ipynb["metadata"]["name"],
                    "name": ipynb["metadata"]["name"],
                }

        if jupyter_metadata:
            return jupyter_metadata
    except Exception:
        logger.exception(error_message)

    wandb.termerror(error_message)
    return {}


def jupyter_servers_and_kernel_id():
    """Return a list of servers and the current kernel_id.

    Used to query for the name of the notebook.
    """
    try:
        import ipykernel  # type: ignore

        kernel_id = re.search(
            "kernel-(.*).json", ipykernel.connect.get_connection_file()
        ).group(1)
        # We're either in jupyterlab or a notebook, lets prefer the newer jupyter_server package
        serverapp = wandb.util.get_module("jupyter_server.serverapp")
        notebookapp = wandb.util.get_module("notebook.notebookapp")
        servers = []
        if serverapp is not None:
            servers.extend(list(serverapp.list_running_servers()))
        if notebookapp is not None:
            servers.extend(list(notebookapp.list_running_servers()))
    except (AttributeError, ValueError, ImportError):
        return [], None

    return servers, kernel_id


def attempt_colab_load_ipynb():
    colab = wandb.util.get_module("google.colab")
    if colab:
        # This isn't thread safe, never call in a thread
        response = colab._message.blocking_request("get_ipynb", timeout_sec=5)
        if response:
            return response["ipynb"]


def attempt_kaggle_load_ipynb():
    kaggle = wandb.util.get_module("kaggle_session")
    if not kaggle:
        return None

    try:
        client = kaggle.UserSessionClient()
        parsed = json.loads(client.get_exportable_ipynb()["source"])
        # TODO: couldn't find a way to get the name of the notebook...
        parsed["metadata"]["name"] = "kaggle.ipynb"
    except Exception:
        wandb.termerror("Unable to load kaggle notebook.")
        logger.exception("Unable to load kaggle notebook.")
        return None

    return parsed


class Notebook:
    def __init__(self, settings: wandb.Settings) -> None:
        self.outputs: dict[int, Any] = {}
        self.settings = settings
        self.shell = IPython.get_ipython()

    def save_display(self, exc_count, data_with_metadata):
        self.outputs[exc_count] = self.outputs.get(exc_count, [])

        # byte values such as images need to be encoded in base64
        # otherwise nbformat.v4.new_output will throw a NotebookValidationError
        data = data_with_metadata["data"]
        b64_data = {}
        for key in data:
            val = data[key]
            if isinstance(val, bytes):
                b64_data[key] = b64encode(val).decode("utf-8")
            else:
                b64_data[key] = val

        self.outputs[exc_count].append(
            {"data": b64_data, "metadata": data_with_metadata["metadata"]}
        )

    def probe_ipynb(self):
        """Return notebook as dict or None."""
        relpath = self.settings.x_jupyter_path
        if relpath and os.path.exists(relpath):
            with open(relpath) as json_file:
                data = json.load(json_file)
                return data

        colab_ipynb = attempt_colab_load_ipynb()
        if colab_ipynb:
            return colab_ipynb

        kaggle_ipynb = attempt_kaggle_load_ipynb()
        if kaggle_ipynb and len(kaggle_ipynb["cells"]) > 0:
            return kaggle_ipynb

        return

    def save_ipynb(self) -> bool:
        if not self.settings.save_code:
            logger.info("not saving jupyter notebook")
            return False
        ret = False
        try:
            ret = self._save_ipynb()
        except Exception:
            wandb.termerror("Failed to save notebook.")
            logger.exception("Problem saving notebook.")
        return ret

    def _save_ipynb(self) -> bool:
        relpath = self.settings.x_jupyter_path
        logger.info("looking for notebook: %s", relpath)
        if relpath and os.path.exists(relpath):
            shutil.copy(
                relpath,
                os.path.join(self.settings._tmp_code_dir, os.path.basename(relpath)),
            )
            return True

        # TODO: likely only save if the code has changed
        colab_ipynb = attempt_colab_load_ipynb()
        if colab_ipynb:
            try:
                jupyter_metadata = (
                    notebook_metadata_from_jupyter_servers_and_kernel_id()
                )
                nb_name = jupyter_metadata["name"]
            except Exception:
                nb_name = "colab.ipynb"
            if not nb_name.endswith(".ipynb"):
                nb_name += ".ipynb"
            with open(
                os.path.join(
                    self.settings._tmp_code_dir,
                    nb_name,
                ),
                "w",
                encoding="utf-8",
            ) as f:
                f.write(json.dumps(colab_ipynb))
            return True

        kaggle_ipynb = attempt_kaggle_load_ipynb()
        if kaggle_ipynb and len(kaggle_ipynb["cells"]) > 0:
            with open(
                os.path.join(
                    self.settings._tmp_code_dir, kaggle_ipynb["metadata"]["name"]
                ),
                "w",
                encoding="utf-8",
            ) as f:
                f.write(json.dumps(kaggle_ipynb))
            return True

        return False

    def save_history(self, run: wandb.Run):
        """This saves all cell executions in the current session as a new notebook."""
        try:
            from nbformat import v4, validator, write  # type: ignore
        except ImportError:
            wandb.termerror(
                "The nbformat package was not found."
                " It is required to save notebook history."
            )
            return
        # TODO: some tests didn't patch ipython properly?
        if self.shell is None:
            return
        cells = []
        hist = list(self.shell.history_manager.get_range(output=True))
        if len(hist) <= 1 or not self.settings.save_code:
            logger.info("not saving jupyter history")
            return
        try:
            for _, execution_count, exc in hist:
                if exc[1]:
                    # TODO: capture stderr?
                    outputs = [
                        v4.new_output(output_type="stream", name="stdout", text=exc[1])
                    ]
                else:
                    outputs = []
                if self.outputs.get(execution_count):
                    for out in self.outputs[execution_count]:
                        outputs.append(
                            v4.new_output(
                                output_type="display_data",
                                data=out["data"],
                                metadata=out["metadata"] or {},
                            )
                        )
                cells.append(
                    v4.new_code_cell(
                        execution_count=execution_count, source=exc[0], outputs=outputs
                    )
                )
            if hasattr(self.shell, "kernel"):
                language_info = self.shell.kernel.language_info
            else:
                language_info = {"name": "python", "version": sys.version}
            logger.info("saving %i cells to _session_history.ipynb", len(cells))
            nb = v4.new_notebook(
                cells=cells,
                metadata={
                    "kernelspec": {
                        "display_name": f"Python {sys.version_info[0]}",
                        "name": f"python{sys.version_info[0]}",
                        "language": "python",
                    },
                    "language_info": language_info,
                },
            )
            state_path = os.path.join("code", "_session_history.ipynb")
            run._set_config_wandb("session_history", state_path)
            filesystem.mkdir_exists_ok(os.path.join(self.settings.files_dir, "code"))
            with open(
                os.path.join(self.settings._tmp_code_dir, "_session_history.ipynb"),
                "w",
                encoding="utf-8",
            ) as f:
                write(nb, f, version=4)
            with open(
                os.path.join(self.settings.files_dir, state_path),
                "w",
                encoding="utf-8",
            ) as f:
                write(nb, f, version=4)
        except (OSError, validator.NotebookValidationError):
            wandb.termerror("Unable to save notebook session history.")
            logger.exception("Unable to save notebook session history.")


def _load_ipython_extension(ipython):
    """Best-effort auto-registration of W&B magics in notebook contexts."""
    if ipython is None:
        return

    try:
        ipython.register_magics(WandBMagics)
    except Exception:
        logger.debug("Failed to register IPython magics.", exc_info=True)
        return