wandb_init.py 60 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611
  1. """Defines wandb.init() and associated classes and methods.
  2. `wandb.init()` indicates the beginning of a new run. In an ML training pipeline,
  3. you could add `wandb.init()` to the beginning of your training script as well as
  4. your evaluation script, and each step would be tracked as a run in W&B.
  5. For more on using `wandb.init()`, including code snippets, check out our
  6. [guide and FAQs](https://docs.wandb.ai/platform/launch).
  7. """
  8. from __future__ import annotations
  9. import contextlib
  10. import dataclasses
  11. import functools
  12. import json
  13. import logging
  14. import os
  15. import pathlib
  16. import platform
  17. import sys
  18. import tempfile
  19. import time
  20. from collections.abc import Iterable, Iterator, Sequence
  21. from typing_extensions import Any, Literal, Protocol, Self
  22. import wandb
  23. import wandb.env
  24. from wandb import env, trigger
  25. from wandb.analytics import get_sentry
  26. from wandb.errors import CommError, Error, UsageError
  27. from wandb.errors.links import url_registry
  28. from wandb.errors.util import ProtobufErrorHandler
  29. from wandb.integration import sagemaker, weave
  30. from wandb.proto.wandb_telemetry_pb2 import Deprecated
  31. from wandb.sdk.lib import ipython as wb_ipython
  32. from wandb.sdk.lib import progress, runid, wb_logging
  33. from wandb.sdk.lib.paths import StrPath
  34. from wandb.util import _is_artifact_representation
  35. from . import wandb_login, wandb_setup
  36. from .backend.backend import Backend
  37. from .lib import SummaryDisabled, filesystem, module, paths, printer, telemetry
  38. from .lib.deprecation import UNSET, DoNotSet, warn_and_record_deprecation
  39. from .mailbox import wait_with_progress
  40. from .wandb_helper import parse_config
  41. from .wandb_run import Run, TeardownHook, TeardownStage
  42. from .wandb_settings import Settings
  43. # Used to avoid printing the same notice repeatedly
  44. # for multiple runs in the same process.
  45. _shared_service_notice_shown = False
  46. def _huggingface_version() -> str | None:
  47. if "transformers" in sys.modules:
  48. trans = wandb.util.get_module("transformers")
  49. if hasattr(trans, "__version__"):
  50. return str(trans.__version__)
  51. return None
  52. def _handle_launch_config(settings: Settings) -> dict[str, Any]:
  53. launch_run_config: dict[str, Any] = {}
  54. if not settings.launch:
  55. return launch_run_config
  56. if os.environ.get("WANDB_CONFIG") is not None:
  57. try:
  58. launch_run_config = json.loads(os.environ.get("WANDB_CONFIG", "{}"))
  59. except (ValueError, SyntaxError):
  60. wandb.termwarn("Malformed WANDB_CONFIG, using original config")
  61. elif settings.launch_config_path and os.path.exists(settings.launch_config_path):
  62. with open(settings.launch_config_path) as fp:
  63. launch_config = json.loads(fp.read())
  64. launch_run_config = launch_config.get("overrides", {}).get("run_config")
  65. else:
  66. i = 0
  67. chunks = []
  68. while True:
  69. key = f"WANDB_CONFIG_{i}"
  70. if key in os.environ:
  71. chunks.append(os.environ[key])
  72. i += 1
  73. else:
  74. break
  75. if len(chunks) > 0:
  76. config_string = "".join(chunks)
  77. try:
  78. launch_run_config = json.loads(config_string)
  79. except (ValueError, SyntaxError):
  80. wandb.termwarn("Malformed WANDB_CONFIG, using original config")
  81. return launch_run_config
  82. @dataclasses.dataclass(frozen=True)
  83. class _ConfigParts:
  84. base_no_artifacts: dict[str, Any]
  85. """The run config passed to `init()` minus any artifact-valued keys."""
  86. sweep_no_artifacts: dict[str, Any]
  87. """The config loaded as part of a sweep minus any artifact-valued keys."""
  88. launch_no_artifacts: dict[str, Any]
  89. """The config loaded as part of Launch minus any artifact-valued keys."""
  90. artifacts: dict[str, Any]
  91. """Artifact keys removed from config dictionaries.
  92. Due to implementation details of how a Run is constructed,
  93. artifacts must be inserted into its config after initialization.
  94. """
  95. class _PrinterCallback(Protocol):
  96. """A callback for displaying messages after a printer is configured.
  97. This is used for a few messages that may be generated before run settings
  98. are computed, which are necessary for creating a printer.
  99. """
  100. def __call__(self, run_printer: printer.Printer) -> None:
  101. """Display information through the given printer."""
  102. def _noop_printer_callback() -> _PrinterCallback:
  103. """A printer callback that does not print anything."""
  104. return lambda _: None
  105. def _concat_printer_callbacks(
  106. cbs: Iterable[_PrinterCallback],
  107. ) -> _PrinterCallback:
  108. """Returns a printer callback that runs the given callbacks in order."""
  109. def do_callbacks(run_printer: printer.Printer) -> None:
  110. for cb in cbs:
  111. cb(run_printer)
  112. return do_callbacks
  113. class _WandbInit:
  114. def __init__(
  115. self,
  116. wl: wandb_setup._WandbSetup,
  117. telemetry: telemetry.TelemetryRecord,
  118. ) -> None:
  119. self._wl = wl
  120. self._telemetry = telemetry
  121. """Telemetry gathered before creating a run.
  122. After the run is created, `telemetry.context()` is used instead.
  123. """
  124. self.kwargs = None
  125. self.run: Run | None = None
  126. self.backend: Backend | None = None
  127. self._teardown_hooks: list[TeardownHook] = []
  128. self.notebook: wandb.jupyter.Notebook | None = None
  129. self.deprecated_features_used: list[tuple[Deprecated, str]] = []
  130. @property
  131. def _logger(self) -> wandb_setup.Logger:
  132. return self._wl._get_logger()
  133. def maybe_login(self, init_settings: Settings) -> None:
  134. """Log in if we are not creating an offline or disabled run.
  135. This may change the W&B singleton settings.
  136. Args:
  137. init_settings: Settings passed to `wandb.init()` or set via
  138. keyword arguments.
  139. """
  140. # Allow settings passed to init() to override inferred values.
  141. #
  142. # Calling login() may change settings on the singleton,
  143. # so these may not be the final run settings.
  144. run_settings = self._wl.settings.model_copy()
  145. run_settings.update_from_settings(init_settings)
  146. # NOTE: _noop or _offline can become true after _login().
  147. # _noop happens if _login hits a timeout.
  148. # _offline can be selected by the user at the login prompt.
  149. if run_settings._noop or run_settings._offline:
  150. return
  151. # Only pass an explicit key when the key was provided directly
  152. # to ensure correct messaging in _login().
  153. explicit_key = init_settings.api_key
  154. wandb_login._login(
  155. host=run_settings.base_url,
  156. force=run_settings.force,
  157. _silent=run_settings.quiet or run_settings.silent,
  158. key=explicit_key,
  159. update_api_key=explicit_key is None,
  160. )
  161. def warn_env_vars_change_after_setup(self) -> _PrinterCallback:
  162. """Warn if environment variables changed after `wandb.setup()`.
  163. Returns:
  164. A callback to print any generated warnings.
  165. """
  166. if not self._wl.did_environment_change():
  167. return _noop_printer_callback()
  168. def print_warning(run_printer: printer.Printer) -> None:
  169. line = (
  170. "Changes to your `wandb` environment variables will be ignored "
  171. "because your `wandb` session has already started. "
  172. "For more information on how to modify your settings with "
  173. "`wandb.init()` arguments, please refer to "
  174. f"{run_printer.link(url_registry.url('wandb-init'), 'the W&B docs')}."
  175. )
  176. run_printer.display(line, level="warn")
  177. return print_warning
  178. def clear_run_path_if_sweep_or_launch(
  179. self,
  180. init_settings: Settings,
  181. ) -> _PrinterCallback:
  182. """Clear project/entity/run_id keys if in a Sweep or a Launch context.
  183. Args:
  184. init_settings: Settings specified in the call to `wandb.init()`.
  185. Returns:
  186. A callback to print any generated warnings.
  187. """
  188. when_doing_thing = ""
  189. if self._wl.settings.sweep_id:
  190. when_doing_thing = "when running a sweep"
  191. elif self._wl.settings.launch:
  192. when_doing_thing = "when running from a wandb launch context"
  193. if not when_doing_thing:
  194. return _noop_printer_callback()
  195. warnings = []
  196. def warn(key: str, value: str) -> None:
  197. warnings.append(f"Ignoring {key} {value!r} {when_doing_thing}.")
  198. if init_settings.project is not None:
  199. warn("project", init_settings.project)
  200. init_settings.project = None
  201. if init_settings.entity is not None:
  202. warn("entity", init_settings.entity)
  203. init_settings.entity = None
  204. if init_settings.run_id is not None:
  205. warn("run_id", init_settings.run_id)
  206. init_settings.run_id = None
  207. def print_warnings(run_printer: printer.Printer) -> None:
  208. for warning in warnings:
  209. run_printer.display(warning, level="warn")
  210. return print_warnings
  211. def make_run_settings(
  212. self,
  213. init_settings: Settings,
  214. ) -> tuple[Settings, _PrinterCallback]:
  215. """Returns the run's settings and any warnings.
  216. Args:
  217. init_settings: Settings passed to `wandb.init()` or set via
  218. keyword arguments.
  219. """
  220. warning_callbacks: list[_PrinterCallback] = [
  221. self.warn_env_vars_change_after_setup(),
  222. self.clear_run_path_if_sweep_or_launch(init_settings),
  223. ]
  224. # Inherit global settings.
  225. settings = self._wl.settings.model_copy()
  226. # Apply settings from wandb.init() call.
  227. settings.update_from_settings(init_settings)
  228. # Infer the run ID from SageMaker.
  229. if (
  230. (not settings.sagemaker_disable)
  231. and sagemaker.is_using_sagemaker()
  232. and sagemaker.set_run_id(settings)
  233. ):
  234. self._logger.info("set run ID and group based on SageMaker")
  235. self._telemetry.feature.sagemaker = True
  236. # get status of code saving before applying user settings
  237. save_code_pre_user_settings = settings.save_code
  238. if not settings._offline and not settings._noop:
  239. user_settings = self._wl._load_user_settings()
  240. if user_settings is not None:
  241. settings.update_from_dict(user_settings)
  242. # ensure that user settings don't set saving to true
  243. # if user explicitly set these to false in UI
  244. if save_code_pre_user_settings is False:
  245. settings.save_code = False
  246. # TODO: remove this once we refactor the client. This is a temporary
  247. # fix to make sure that we use the same project name for wandb-core.
  248. # The reason this is not going through the settings object is to
  249. # avoid failure cases in other parts of the code that will be
  250. # removed with the switch to wandb-core.
  251. if settings.project is None:
  252. settings.project = wandb.util.auto_project_name(settings.program)
  253. settings.x_start_time = time.time()
  254. # In shared mode, generate a unique label if not provided.
  255. # The label is used to distinguish between system metrics and console logs
  256. # from different writers to the same run.
  257. if settings._shared and not settings.x_label:
  258. # TODO: If executed in a known distributed environment (e.g. Ray or SLURM),
  259. # use the env vars to generate a label (e.g. SLURM_JOB_ID or RANK)
  260. prefix = settings.host or ""
  261. label = runid.generate_id()
  262. settings.x_label = f"{prefix}-{label}" if prefix else label
  263. return settings, _concat_printer_callbacks(warning_callbacks)
  264. def _load_autoresume_run_id(self, resume_file: pathlib.Path) -> str | None:
  265. """Returns the run_id stored in the auto-resume file, if any.
  266. Returns `None` if the file does not exist or is not in a valid format.
  267. Args:
  268. resume_file: The file path to use for resume='auto' mode.
  269. """
  270. if not resume_file.exists():
  271. return None
  272. with resume_file.open() as f:
  273. try:
  274. return json.load(f)["run_id"]
  275. except json.JSONDecodeError as e:
  276. self._logger.exception(
  277. f"could not decode {resume_file}, ignoring",
  278. exc_info=e,
  279. )
  280. return None
  281. except KeyError:
  282. self._logger.exception(
  283. f"resume file at {resume_file} did not store a run_id"
  284. )
  285. return None
  286. def _save_autoresume_run_id(
  287. self,
  288. *,
  289. resume_file: pathlib.Path,
  290. run_id: str,
  291. ) -> None:
  292. """Write the run ID to the auto-resume file."""
  293. resume_file.parent.mkdir(exist_ok=True)
  294. with resume_file.open("w") as f:
  295. json.dump({"run_id": run_id}, f)
  296. def set_run_id(self, settings: Settings) -> None:
  297. """Set the run ID and possibly save it to the auto-resume file.
  298. After this, `settings.run_id` is guaranteed to be set.
  299. If a `resume_from` is provided and `run_id` is not set, initialize
  300. `run_id` with the `resume_from` run's `run_id`.
  301. Args:
  302. settings: The run's settings derived from the environment
  303. and explicit values passed to `wandb.init()`.
  304. """
  305. if settings.resume == "auto" and settings.resume_fname:
  306. resume_path = pathlib.Path(settings.resume_fname)
  307. else:
  308. resume_path = None
  309. if resume_path:
  310. previous_id = self._load_autoresume_run_id(resume_path)
  311. if not previous_id:
  312. pass
  313. elif settings.run_id is None:
  314. self._logger.info(f"loaded run ID from {resume_path}")
  315. settings.run_id = previous_id
  316. elif settings.run_id != previous_id:
  317. wandb.termwarn(
  318. f"Ignoring ID {previous_id} loaded due to resume='auto'"
  319. f" because the run ID is set to {settings.run_id}.",
  320. )
  321. # If no run ID was inferred, explicitly set, or loaded from an
  322. # auto-resume file, then we generate a new ID.
  323. if settings.run_id is None:
  324. # If resume_from is provided and run_id is not already set,
  325. # initialize run_id with the value from resume_from.
  326. if settings.resume_from:
  327. settings.run_id = settings.resume_from.run
  328. else:
  329. settings.run_id = runid.generate_id()
  330. if resume_path:
  331. self._save_autoresume_run_id(
  332. resume_file=resume_path,
  333. run_id=settings.run_id,
  334. )
  335. def set_sync_dir_suffix(self, settings: Settings) -> None:
  336. """Add a suffix to sync_dir if it already exists.
  337. The sync_dir uses a timestamp with second-level precision which can
  338. result in conflicts if a run with the same ID is initialized within the
  339. same second. This is most likely to happen in tests.
  340. This can't prevent conflicts from multiple processes attempting
  341. to create a wandb run simultaneously.
  342. Args:
  343. settings: Fully initialized settings other than the
  344. x_sync_dir_suffix setting which will be modified.
  345. """
  346. index = 1
  347. while pathlib.Path(settings.sync_dir).exists():
  348. settings.x_sync_dir_suffix = f"{index}"
  349. index += 1
  350. def make_run_config(
  351. self,
  352. settings: Settings,
  353. config: dict | str | None = None,
  354. config_exclude_keys: list[str] | None = None,
  355. config_include_keys: list[str] | None = None,
  356. ) -> _ConfigParts:
  357. """Construct the run's config.
  358. Args:
  359. settings: The run's finalized settings.
  360. config: The config passed to `init()`.
  361. config_exclude_keys: Deprecated. Keys to filter out from `config`.
  362. config_include_keys: Deprecated. Keys to include from `config`.
  363. Returns:
  364. Initial values for the run's config.
  365. """
  366. if config_exclude_keys:
  367. self.deprecated_features_used.append(
  368. (
  369. Deprecated(init__config_exclude_keys=True),
  370. "config_exclude_keys is deprecated. Use"
  371. " `config=wandb.helper.parse_config(config_object,"
  372. " exclude=('key',))` instead.",
  373. )
  374. )
  375. if config_include_keys:
  376. self.deprecated_features_used.append(
  377. (
  378. Deprecated(init__config_include_keys=True),
  379. "config_include_keys is deprecated. Use"
  380. " `config=wandb.helper.parse_config(config_object,"
  381. " include=('key',))` instead.",
  382. )
  383. )
  384. config = parse_config(
  385. config or dict(),
  386. include=config_include_keys,
  387. exclude=config_exclude_keys,
  388. )
  389. result = _ConfigParts(
  390. base_no_artifacts=dict(),
  391. sweep_no_artifacts=dict(),
  392. launch_no_artifacts=dict(),
  393. artifacts=dict(),
  394. )
  395. if not settings.sagemaker_disable and sagemaker.is_using_sagemaker():
  396. sagemaker_config = sagemaker.parse_sm_config()
  397. self._split_artifacts_from_config(
  398. sagemaker_config,
  399. config_target=result.base_no_artifacts,
  400. artifacts=result.artifacts,
  401. )
  402. self._telemetry.feature.sagemaker = True
  403. if self._wl.config:
  404. self._split_artifacts_from_config(
  405. self._wl.config,
  406. config_target=result.base_no_artifacts,
  407. artifacts=result.artifacts,
  408. )
  409. if config and isinstance(config, dict):
  410. self._split_artifacts_from_config(
  411. config,
  412. config_target=result.base_no_artifacts,
  413. artifacts=result.artifacts,
  414. )
  415. if self._wl._sweep_config:
  416. self._split_artifacts_from_config(
  417. self._wl._sweep_config,
  418. config_target=result.sweep_no_artifacts,
  419. artifacts=result.artifacts,
  420. )
  421. if launch_config := _handle_launch_config(settings):
  422. self._split_artifacts_from_config(
  423. launch_config,
  424. config_target=result.launch_no_artifacts,
  425. artifacts=result.artifacts,
  426. )
  427. wandb_internal = result.base_no_artifacts.setdefault("_wandb", dict())
  428. if settings.save_code and settings.program_relpath:
  429. wandb_internal["code_path"] = paths.LogicalPath(
  430. os.path.join("code", settings.program_relpath)
  431. )
  432. if settings.fork_from is not None:
  433. wandb_internal["branch_point"] = {
  434. "run_id": settings.fork_from.run,
  435. "step": settings.fork_from.value,
  436. }
  437. if settings.resume_from is not None:
  438. wandb_internal["branch_point"] = {
  439. "run_id": settings.resume_from.run,
  440. "step": settings.resume_from.value,
  441. }
  442. return result
  443. def teardown(self) -> None:
  444. # TODO: currently this is only called on failed wandb.init attempts
  445. # normally this happens on the run object
  446. self._logger.info("tearing down wandb.init")
  447. for hook in self._teardown_hooks:
  448. hook.call()
  449. def _split_artifacts_from_config(
  450. self,
  451. config_source: dict,
  452. config_target: dict,
  453. artifacts: dict,
  454. ) -> None:
  455. for k, v in config_source.items():
  456. if _is_artifact_representation(v):
  457. artifacts[k] = v
  458. else:
  459. config_target.setdefault(k, v)
  460. def _safe_symlink(
  461. self, base: str, target: str, name: str, delete: bool = False
  462. ) -> None:
  463. # TODO(jhr): do this with relpaths, but i can't figure it out on no sleep
  464. if not hasattr(os, "symlink"):
  465. return
  466. pid = os.getpid()
  467. tmp_name = os.path.join(base, f"{name}.{pid}")
  468. if delete:
  469. try:
  470. os.remove(os.path.join(base, name))
  471. except OSError:
  472. pass
  473. target = os.path.relpath(target, base)
  474. try:
  475. os.symlink(target, tmp_name)
  476. os.rename(tmp_name, os.path.join(base, name))
  477. except OSError:
  478. pass
  479. def _pre_run_cell_hook(self, *args, **kwargs) -> None:
  480. """Hook for the IPython pre_run_cell event.
  481. This pauses a run, preventing system metrics from being collected
  482. the run's runtime from increasing. It also uploads the notebook's code.
  483. """
  484. if not self.backend:
  485. return
  486. if self.notebook and self.notebook.save_ipynb():
  487. assert self.run is not None
  488. res = self.run.log_code(root=None)
  489. self._logger.info("saved code: %s", res)
  490. if self.backend.interface is not None:
  491. self._logger.info("pausing backend")
  492. self.backend.interface.publish_pause()
  493. def _post_run_cell_hook(self, *args, **kwargs) -> None:
  494. """Hook for the IPython post_run_cell event.
  495. Resumes collection of system metrics and the run's timer.
  496. """
  497. if self.backend is None or self.backend.interface is None:
  498. return
  499. self._logger.info("resuming backend")
  500. self.backend.interface.publish_resume()
  501. def _jupyter_teardown(self) -> None:
  502. """Teardown hooks and display saving, called with wandb.finish."""
  503. assert self.notebook
  504. ipython = self.notebook.shell
  505. if self.run:
  506. self.notebook.save_history(self.run)
  507. if self.notebook.save_ipynb():
  508. assert self.run is not None
  509. res = self.run.log_code(root=None)
  510. self._logger.info("saved code and history: %s", res)
  511. self._logger.info("cleaning up jupyter logic")
  512. ipython.events.unregister("pre_run_cell", self._pre_run_cell_hook)
  513. ipython.events.unregister("post_run_cell", self._post_run_cell_hook)
  514. ipython.display_pub.publish = ipython.display_pub._orig_publish
  515. del ipython.display_pub._orig_publish
  516. def monkeypatch_ipython(self, settings: Settings) -> None:
  517. """Add hooks, and session history saving."""
  518. self.notebook = wandb.jupyter.Notebook(settings)
  519. ipython = self.notebook.shell
  520. # Monkey patch ipython publish to capture displayed outputs
  521. if not hasattr(ipython.display_pub, "_orig_publish"):
  522. self._logger.info("configuring jupyter hooks %s", self)
  523. ipython.display_pub._orig_publish = ipython.display_pub.publish
  524. ipython.events.register("pre_run_cell", self._pre_run_cell_hook)
  525. ipython.events.register("post_run_cell", self._post_run_cell_hook)
  526. self._teardown_hooks.append(
  527. TeardownHook(self._jupyter_teardown, TeardownStage.EARLY)
  528. )
  529. def publish(data, metadata=None, **kwargs) -> None:
  530. ipython.display_pub._orig_publish(data, metadata=metadata, **kwargs)
  531. assert self.notebook is not None
  532. self.notebook.save_display(
  533. ipython.execution_count, {"data": data, "metadata": metadata}
  534. )
  535. ipython.display_pub.publish = publish
  536. @contextlib.contextmanager
  537. def setup_run_log_directory(self, settings: Settings) -> Iterator[None]:
  538. """Set up the run's log directory.
  539. This is a context manager that closes and unregisters the log handler
  540. in case of an uncaught exception, so that future logged messages do not
  541. modify this run's log file.
  542. """
  543. filesystem.mkdir_exists_ok(os.path.dirname(settings.log_user))
  544. filesystem.mkdir_exists_ok(os.path.dirname(settings.log_internal))
  545. filesystem.mkdir_exists_ok(os.path.dirname(settings.sync_file))
  546. filesystem.mkdir_exists_ok(settings.files_dir)
  547. filesystem.mkdir_exists_ok(settings._tmp_code_dir)
  548. if settings.symlink:
  549. self._safe_symlink(
  550. os.path.dirname(settings.sync_symlink_latest),
  551. os.path.dirname(settings.sync_file),
  552. os.path.basename(settings.sync_symlink_latest),
  553. delete=True,
  554. )
  555. self._safe_symlink(
  556. os.path.dirname(settings.log_symlink_user),
  557. settings.log_user,
  558. os.path.basename(settings.log_symlink_user),
  559. delete=True,
  560. )
  561. self._safe_symlink(
  562. os.path.dirname(settings.log_symlink_internal),
  563. settings.log_internal,
  564. os.path.basename(settings.log_symlink_internal),
  565. delete=True,
  566. )
  567. assert settings.run_id
  568. handler = wb_logging.add_file_handler(
  569. settings.run_id,
  570. pathlib.Path(settings.log_user),
  571. )
  572. if env.is_debug():
  573. handler.setLevel(logging.DEBUG)
  574. disposed = False
  575. def dispose_handler() -> None:
  576. nonlocal disposed
  577. if not disposed:
  578. disposed = True
  579. logging.getLogger("wandb").removeHandler(handler)
  580. handler.close()
  581. try:
  582. self._teardown_hooks.append(
  583. TeardownHook(
  584. call=dispose_handler,
  585. stage=TeardownStage.LATE,
  586. )
  587. )
  588. self._wl._early_logger_flush(logging.getLogger("wandb"))
  589. self._logger.info(f"Logging user logs to {settings.log_user}")
  590. self._logger.info(f"Logging internal logs to {settings.log_internal}")
  591. yield
  592. except Exception:
  593. dispose_handler()
  594. raise
  595. def make_disabled_run(self, config: _ConfigParts) -> Run:
  596. """Returns a Run-like object where all methods are no-ops.
  597. This method is used when the `mode` setting is set to "disabled", such as
  598. by wandb.init(mode="disabled") or by setting the WANDB_MODE environment
  599. variable to "disabled".
  600. It creates a Run object that mimics the behavior of a normal Run but doesn't
  601. communicate with the W&B servers.
  602. The returned Run object has all expected attributes and methods, but they
  603. are no-op versions that don't perform any actual logging or communication.
  604. """
  605. run_id = runid.generate_id()
  606. drun = Run(
  607. settings=Settings(
  608. mode="disabled",
  609. root_dir=tempfile.gettempdir(),
  610. run_id=run_id,
  611. run_tags=tuple(),
  612. run_notes=None,
  613. run_group=None,
  614. run_name=f"dummy-{run_id}",
  615. project="dummy",
  616. entity="dummy",
  617. )
  618. )
  619. # config, summary, and metadata objects
  620. drun._config = wandb.sdk.wandb_config.Config()
  621. drun._config.update(config.sweep_no_artifacts)
  622. drun._config.update(config.base_no_artifacts)
  623. drun.summary = SummaryDisabled() # type: ignore
  624. # methods
  625. drun.log = lambda data, *_, **__: drun.summary.update(data) # type: ignore[method-assign]
  626. drun.finish = lambda *_, **__: module.unset_globals() # type: ignore[method-assign]
  627. drun.join = drun.finish # type: ignore[method-assign]
  628. drun.define_metric = lambda *_, **__: wandb.sdk.wandb_metric.Metric("dummy") # type: ignore[method-assign]
  629. drun.save = lambda *_, **__: False # type: ignore[method-assign]
  630. for symbol in (
  631. "alert",
  632. "finish_artifact",
  633. "get_project_url",
  634. "get_sweep_url",
  635. "get_url",
  636. "link_artifact",
  637. "link_model",
  638. "use_artifact",
  639. "log_code",
  640. "log_model",
  641. "use_model",
  642. "mark_preempting",
  643. "restore",
  644. "status",
  645. "watch",
  646. "unwatch",
  647. "upsert_artifact",
  648. "_finish",
  649. ):
  650. setattr(drun, symbol, lambda *_, **__: None) # type: ignore
  651. # set properties to None
  652. for attr in ("url", "project_url", "sweep_url"):
  653. setattr(type(drun), attr, property(lambda _: None))
  654. class _ChainableNoOp:
  655. """An object that allows chaining arbitrary attributes and method calls."""
  656. def __getattr__(self, _: str) -> Self:
  657. return self
  658. def __call__(self, *_: Any, **__: Any) -> Self:
  659. return self
  660. class _ChainableNoOpField:
  661. # This is used to chain arbitrary attributes and method calls.
  662. # For example, `run.log_artifact().state` will work in disabled mode.
  663. def __init__(self) -> None:
  664. self._value = None
  665. def __set__(self, instance: Any, value: Any) -> None:
  666. self._value = value
  667. def __get__(self, instance: Any, owner: type) -> Any:
  668. return _ChainableNoOp() if (self._value is None) else self._value
  669. def __call__(self, *args: Any, **kwargs: Any) -> _ChainableNoOp:
  670. return _ChainableNoOp()
  671. drun.log_artifact = _ChainableNoOpField() # type: ignore
  672. # attributes
  673. drun._start_time = time.time()
  674. drun._starting_step = 0
  675. drun._step = 0
  676. drun._attach_id = None
  677. drun._backend = None
  678. # set the disabled run as the global run
  679. module.set_global(
  680. run=drun,
  681. config=drun.config,
  682. log=drun.log,
  683. summary=drun.summary,
  684. save=drun.save,
  685. use_artifact=drun.use_artifact,
  686. log_artifact=drun.log_artifact,
  687. define_metric=drun.define_metric,
  688. alert=drun.alert,
  689. watch=drun.watch,
  690. unwatch=drun.unwatch,
  691. )
  692. return drun
  693. def init( # noqa: C901
  694. self,
  695. settings: Settings,
  696. config: _ConfigParts,
  697. run_printer: printer.Printer,
  698. ) -> Run:
  699. self._logger.info("calling init triggers")
  700. trigger.call("on_init")
  701. assert self._wl is not None
  702. self._logger.info(
  703. f"wandb.init called with sweep_config: {config.sweep_no_artifacts}"
  704. f"\nconfig: {config.base_no_artifacts}"
  705. )
  706. if previous_run := self._wl.most_recent_active_run:
  707. if (
  708. settings.reinit in (True, "finish_previous")
  709. # calling wandb.init() in notebooks finishes previous runs
  710. # by default for user convenience.
  711. or (settings.reinit == "default" and wb_ipython.in_notebook())
  712. ):
  713. run_printer.display(
  714. "Finishing previous runs because reinit is set"
  715. f" to {settings.reinit!r}."
  716. )
  717. self._wl.finish_all_active_runs()
  718. elif settings.reinit == "create_new":
  719. self._logger.info(
  720. "wandb.init() called while a run is active,"
  721. " and reinit is set to 'create_new', so continuing"
  722. )
  723. elif settings.resume == "must":
  724. raise wandb.Error(
  725. "Cannot resume a run while another run is active."
  726. " You must either finish it using run.finish(),"
  727. " or use reinit='create_new' when calling wandb.init()."
  728. )
  729. else:
  730. run_printer.display(
  731. "wandb.init() called while a run is active and reinit is"
  732. f" set to {settings.reinit!r}, so returning the previous"
  733. " run."
  734. )
  735. with telemetry.context(run=previous_run) as tel:
  736. tel.feature.init_return_run = True
  737. return previous_run
  738. self._logger.info("starting backend")
  739. service = self._wl.ensure_service()
  740. global _shared_service_notice_shown
  741. if not service.owns_service:
  742. self._logger.info(
  743. "Connected to an existing wandb-core service via WANDB_SERVICE"
  744. )
  745. if not _shared_service_notice_shown:
  746. run_printer.display(
  747. "Using an existing wandb-core service via WANDB_SERVICE."
  748. )
  749. _shared_service_notice_shown = True
  750. self._logger.info("sending inform_init request")
  751. service.inform_init(
  752. settings=settings.to_proto(),
  753. run_id=settings.run_id, # type: ignore
  754. )
  755. backend = Backend(settings=settings, service=service)
  756. backend.ensure_launched()
  757. self._logger.info("backend started and connected")
  758. run = Run(
  759. config=config.base_no_artifacts,
  760. settings=settings,
  761. sweep_config=config.sweep_no_artifacts,
  762. launch_config=config.launch_no_artifacts,
  763. )
  764. # Populate initial telemetry
  765. with telemetry.context(run=run, obj=self._telemetry) as tel:
  766. tel.cli_version = wandb.__version__
  767. tel.python_version = platform.python_version()
  768. tel.platform = f"{platform.system()}-{platform.machine()}".lower()
  769. hf_version = _huggingface_version()
  770. if hf_version:
  771. tel.huggingface_version = hf_version
  772. if settings._jupyter:
  773. tel.env.jupyter = True
  774. if settings._ipython:
  775. tel.env.ipython = True
  776. if settings._colab:
  777. tel.env.colab = True
  778. if settings._kaggle:
  779. tel.env.kaggle = True
  780. if settings._windows:
  781. tel.env.windows = True
  782. if settings.launch:
  783. tel.feature.launch = True
  784. for module_name in telemetry.list_telemetry_imports(only_imported=True):
  785. setattr(tel.imports_init, module_name, True)
  786. if os.environ.get("PEX"):
  787. tel.env.pex = True
  788. if settings._aws_lambda:
  789. tel.env.aws_lambda = True
  790. if settings.x_flow_control_disabled:
  791. tel.feature.flow_control_disabled = True
  792. if settings.x_flow_control_custom:
  793. tel.feature.flow_control_custom = True
  794. if settings._shared:
  795. wandb.termwarn(
  796. "The `shared` mode feature is experimental and may change. "
  797. "Please contact support@wandb.com for guidance and to report any issues."
  798. )
  799. tel.feature.shared_mode = True
  800. if settings.x_label:
  801. tel.feature.user_provided_label = True
  802. if wandb.env.dcgm_profiling_enabled():
  803. tel.feature.dcgm_profiling_enabled = True
  804. if not settings.label_disable:
  805. if self.notebook:
  806. run._label_probe_notebook(self.notebook)
  807. else:
  808. run._label_probe_main()
  809. for deprecated_feature, msg in self.deprecated_features_used:
  810. warn_and_record_deprecation(
  811. feature=deprecated_feature,
  812. message=msg,
  813. run=run,
  814. )
  815. self._logger.info("updated telemetry")
  816. run._set_library(self._wl)
  817. run._set_backend(backend)
  818. run._set_teardown_hooks(self._teardown_hooks)
  819. assert backend.interface
  820. backend.interface.publish_header()
  821. # Using GitRepo() blocks & can be slow, depending on user's current git setup.
  822. # We don't want to block run initialization/start request, so populate run's git
  823. # info beforehand.
  824. if not (settings.disable_git or settings.x_disable_machine_info):
  825. run._populate_git_info()
  826. if settings._offline and settings.resume:
  827. wandb.termwarn(
  828. "`resume` will be ignored since W&B syncing is set to `offline`. "
  829. f"Starting a new run with run id {run.id}."
  830. )
  831. error: wandb.Error | None = None
  832. timeout = settings.init_timeout
  833. self._logger.info(
  834. f"communicating run to backend with {timeout} second timeout",
  835. )
  836. run_init_handle = backend.interface.deliver_run(run)
  837. try:
  838. with progress.progress_printer(
  839. run_printer,
  840. default_text="Waiting for wandb.init()...",
  841. ) as progress_printer:
  842. result = wait_with_progress(
  843. run_init_handle,
  844. timeout=timeout,
  845. display_progress=functools.partial(
  846. progress.loop_printing_operation_stats,
  847. progress_printer,
  848. backend.interface,
  849. ),
  850. )
  851. except TimeoutError:
  852. # This may either be an issue with the W&B server (a CommError)
  853. # or a bug in the SDK (an Error). We cannot distinguish between
  854. # the two causes here.
  855. raise CommError(
  856. f"Run initialization has timed out after {timeout} sec."
  857. + " Please try increasing the timeout with the `init_timeout`"
  858. + " setting: `wandb.init(settings=wandb.Settings(init_timeout=120))`."
  859. ) from None
  860. assert result.run_result
  861. if error := ProtobufErrorHandler.to_exception(result.run_result.error):
  862. raise error
  863. if not result.run_result.HasField("run"):
  864. raise Error("Assertion failed: run_result is missing the run field")
  865. if result.run_result.run.resumed:
  866. self._logger.info("run resumed")
  867. with telemetry.context(run=run) as tel:
  868. tel.feature.resumed = result.run_result.run.resumed
  869. run._set_run_obj(result.run_result.run)
  870. self._logger.info("starting run threads in backend")
  871. assert backend.interface
  872. run_start_handle = backend.interface.deliver_run_start(run)
  873. try:
  874. # TODO: add progress to let user know we are doing something
  875. run_start_handle.wait_or(timeout=30)
  876. except TimeoutError:
  877. pass
  878. backend.interface.publish_probe_system_info()
  879. assert self._wl is not None
  880. self.run = run
  881. run._handle_launch_artifact_overrides()
  882. if (
  883. settings.launch
  884. and settings.launch_config_path
  885. and os.path.exists(settings.launch_config_path)
  886. ):
  887. run.save(settings.launch_config_path)
  888. # put artifacts in run config here
  889. # since doing so earlier will cause an error
  890. # as the run is not upserted
  891. for k, v in config.artifacts.items():
  892. run.config.update({k: v}, allow_val_change=True)
  893. job_artifact = run._launch_artifact_mapping.get(
  894. wandb.util.LAUNCH_JOB_ARTIFACT_SLOT_NAME
  895. )
  896. if job_artifact:
  897. run.use_artifact(job_artifact)
  898. self.backend = backend
  899. if settings.reinit != "create_new":
  900. _set_global_run(run)
  901. run._on_start()
  902. self._logger.info("run started, returning control to user process")
  903. return run
  904. def _attach(
  905. attach_id: str | None = None,
  906. run_id: str | None = None,
  907. *,
  908. run: Run | None = None,
  909. ) -> Run | None:
  910. """Attach to a run currently executing in another process/thread.
  911. Args:
  912. attach_id: (str, optional) The id of the run or an attach identifier
  913. that maps to a run.
  914. run_id: (str, optional) The id of the run to attach to.
  915. run: (Run, optional) The run instance to attach
  916. """
  917. attach_id = attach_id or run_id
  918. if not ((attach_id is None) ^ (run is None)):
  919. raise UsageError("Either (`attach_id` or `run_id`) or `run` must be specified")
  920. attach_id = attach_id or (run._attach_id if run else None)
  921. if attach_id is None:
  922. raise UsageError(
  923. "Either `attach_id` or `run_id` must be specified or `run` must have `_attach_id`"
  924. )
  925. _wl = wandb_setup.singleton()
  926. logger = _wl._get_logger()
  927. service = _wl.ensure_service()
  928. try:
  929. attach_settings = service.inform_attach(attach_id=attach_id)
  930. except Exception as e:
  931. raise UsageError(f"Unable to attach to run {attach_id}") from e
  932. settings = _wl.settings.model_copy()
  933. settings.update_from_dict(
  934. {
  935. "run_id": attach_id,
  936. "x_start_time": attach_settings.x_start_time.value,
  937. "mode": attach_settings.mode.value,
  938. }
  939. )
  940. # TODO: consolidate this codepath with wandb.init()
  941. backend = Backend(settings=settings, service=service)
  942. backend.ensure_launched()
  943. logger.info("attach backend started and connected")
  944. if run is None:
  945. run = Run(settings=settings)
  946. else:
  947. run._init(settings=settings)
  948. run._set_library(_wl)
  949. run._set_backend(backend)
  950. assert backend.interface
  951. attach_handle = backend.interface.deliver_attach(attach_id)
  952. try:
  953. # TODO: add progress to let user know we are doing something
  954. attach_result = attach_handle.wait_or(timeout=30)
  955. except TimeoutError:
  956. raise UsageError("Timeout attaching to run")
  957. attach_response = attach_result.response.attach_response
  958. if attach_response.error and attach_response.error.message:
  959. raise UsageError(f"Failed to attach to run: {attach_response.error.message}")
  960. run._set_run_obj(attach_response.run)
  961. _set_global_run(run)
  962. run._on_attach()
  963. return run
  964. def _set_global_run(run: Run) -> None:
  965. """Set `wandb.run` and point some top-level functions to its methods.
  966. Args:
  967. run: The run to make global.
  968. """
  969. module.set_global(
  970. run=run,
  971. config=run.config,
  972. log=run.log,
  973. summary=run.summary,
  974. save=run.save,
  975. use_artifact=run.use_artifact,
  976. log_artifact=run.log_artifact,
  977. define_metric=run.define_metric,
  978. alert=run.alert,
  979. watch=run.watch,
  980. unwatch=run.unwatch,
  981. mark_preempting=run.mark_preempting,
  982. log_model=run.log_model,
  983. use_model=run.use_model,
  984. link_model=run.link_model,
  985. )
  986. def _monkeypatch_openai_gym() -> None:
  987. """Patch OpenAI gym to log to the global `wandb.run`."""
  988. if len(wandb.patched["gym"]) > 0:
  989. return
  990. from wandb.integration import gym
  991. gym.monitor()
  992. def _monkeypatch_tensorboard() -> None:
  993. """Patch TensorBoard to log to the global `wandb.run`."""
  994. if len(wandb.patched["tensorboard"]) > 0:
  995. return
  996. from wandb.integration import tensorboard as tb_module
  997. tb_module.patch()
  998. def try_create_root_dir(settings: Settings) -> None:
  999. """Try to create the root directory specified in settings.
  1000. If creation fails due to permissions or other errors,
  1001. falls back to using the system temp directory.
  1002. Args:
  1003. settings: The runs settings containing root_dir configuration.
  1004. This function may update the root_dir to a temporary directory
  1005. if the parent directory is not writable.
  1006. """
  1007. fallback_to_temp_dir = False
  1008. try:
  1009. os.makedirs(settings.root_dir, exist_ok=True)
  1010. except OSError:
  1011. wandb.termwarn(
  1012. f"Unable to create root directory {settings.root_dir}",
  1013. repeat=False,
  1014. )
  1015. fallback_to_temp_dir = True
  1016. else:
  1017. if not os.access(settings.root_dir, os.W_OK | os.R_OK):
  1018. wandb.termwarn(
  1019. f"Path {settings.root_dir} wasn't read/writable",
  1020. repeat=False,
  1021. )
  1022. fallback_to_temp_dir = True
  1023. if not fallback_to_temp_dir:
  1024. return
  1025. tmp_dir = tempfile.gettempdir()
  1026. if not os.access(tmp_dir, os.W_OK | os.R_OK):
  1027. raise ValueError(
  1028. f"System temp directory ({tmp_dir}) is not writable/readable, "
  1029. "please set the `dir` argument in `wandb.init()` to a writable/readable directory."
  1030. )
  1031. settings.root_dir = tmp_dir
  1032. wandb.termwarn(
  1033. f"Falling back to temporary directory {tmp_dir}.",
  1034. repeat=False,
  1035. )
  1036. os.makedirs(settings.root_dir, exist_ok=True)
  1037. def init( # noqa: C901
  1038. entity: str | None = None,
  1039. project: str | None = None,
  1040. dir: StrPath | None = None,
  1041. id: str | None = None,
  1042. name: str | None = None,
  1043. notes: str | None = None,
  1044. tags: Sequence[str] | None = None,
  1045. config: dict[str, Any] | str | None = None,
  1046. config_exclude_keys: list[str] | None = None,
  1047. config_include_keys: list[str] | None = None,
  1048. allow_val_change: bool | None = None,
  1049. group: str | None = None,
  1050. job_type: str | None = None,
  1051. mode: Literal["online", "offline", "disabled", "shared"] | None = None,
  1052. force: bool | None = None,
  1053. reinit: (
  1054. bool
  1055. | Literal[
  1056. None,
  1057. "default",
  1058. "return_previous",
  1059. "finish_previous",
  1060. "create_new",
  1061. ]
  1062. ) = None,
  1063. resume: bool | Literal["allow", "never", "must", "auto"] | None = None,
  1064. resume_from: str | None = None,
  1065. fork_from: str | None = None,
  1066. save_code: bool | None = None,
  1067. tensorboard: bool | None = None,
  1068. sync_tensorboard: bool | None = None,
  1069. monitor_gym: bool | None = None,
  1070. settings: Settings | dict[str, Any] | None = None,
  1071. anonymous: DoNotSet = UNSET,
  1072. ) -> Run:
  1073. r"""Start a new run to track and log to W&B.
  1074. In an ML training pipeline, you could add `wandb.init()` to the beginning of
  1075. your training script as well as your evaluation script, and each piece would
  1076. be tracked as a run in W&B.
  1077. `wandb.init()` spawns a new background process to log data to a run, and it
  1078. also syncs data to https://wandb.ai by default, so you can see your results
  1079. in real-time. When you're done logging data, call `wandb.Run.finish()` to end the run.
  1080. If you don't call `run.finish()`, the run will end when your script exits.
  1081. Run IDs must not contain any of the following special characters `/ \ # ? % :`
  1082. Args:
  1083. entity: The username or team name the runs are logged to.
  1084. The entity must already exist, so ensure you create your account
  1085. or team in the UI before starting to log runs. If not specified, the
  1086. run will default your default entity. To change the default entity,
  1087. go to your settings and update the
  1088. "Default location to create new projects" under "Default team".
  1089. project: The name of the project under which this run will be logged.
  1090. If not specified, we use a heuristic to infer the project name based
  1091. on the system, such as checking the git root or the current program
  1092. file. If we can't infer the project name, the project will default to
  1093. `"uncategorized"`.
  1094. dir: The absolute path to the directory where experiment logs and
  1095. metadata files are stored. If not specified, this defaults
  1096. to the `./wandb` directory. Note that this does not affect the
  1097. location where artifacts are stored when calling `download()`.
  1098. id: A unique identifier for this run, used for resuming. It must be unique
  1099. within the project and cannot be reused once a run is deleted. For
  1100. a short descriptive name, use the `name` field,
  1101. or for saving hyperparameters to compare across runs, use `config`.
  1102. name: A short display name for this run, which appears in the UI to help
  1103. you identify it. By default, we generate a random two-word name
  1104. allowing easy cross-reference runs from table to charts. Keeping these
  1105. run names brief enhances readability in chart legends and tables. For
  1106. saving hyperparameters, we recommend using the `config` field.
  1107. notes: A detailed description of the run, similar to a commit message in
  1108. Git. Use this argument to capture any context or details that may
  1109. help you recall the purpose or setup of this run in the future.
  1110. tags: A list of tags to label this run in the UI. Tags are helpful for
  1111. organizing runs or adding temporary identifiers like "baseline" or
  1112. "production." You can easily add, remove tags, or filter by tags in
  1113. the UI.
  1114. If resuming a run, the tags provided here will replace any existing
  1115. tags. To add tags to a resumed run without overwriting the current
  1116. tags, use `run.tags += ("new_tag",)` after calling `run = wandb.init()`.
  1117. config: Sets `wandb.config`, a dictionary-like object for storing input
  1118. parameters to your run, such as model hyperparameters or data
  1119. preprocessing settings.
  1120. The config appears in the UI in an overview page, allowing you to
  1121. group, filter, and sort runs based on these parameters.
  1122. Keys should not contain periods (`.`), and values should be
  1123. smaller than 10 MB.
  1124. If a dictionary, `argparse.Namespace`, or `absl.flags.FLAGS` is
  1125. provided, the key-value pairs will be loaded directly into
  1126. `wandb.config`.
  1127. If a string is provided, it is interpreted as a path to a YAML file,
  1128. from which configuration values will be loaded into `wandb.config`.
  1129. config_exclude_keys: A list of specific keys to exclude from `wandb.config`.
  1130. config_include_keys: A list of specific keys to include in `wandb.config`.
  1131. allow_val_change: Controls whether config values can be modified after their
  1132. initial set. By default, an exception is raised if a config value is
  1133. overwritten. For tracking variables that change during training, such as
  1134. a learning rate, consider using `wandb.log()` instead. By default, this
  1135. is `False` in scripts and `True` in Notebook environments.
  1136. group: Specify a group name to organize individual runs as part of a larger
  1137. experiment. This is useful for cases like cross-validation or running
  1138. multiple jobs that train and evaluate a model on different test sets.
  1139. Grouping allows you to manage related runs collectively in the UI,
  1140. making it easy to toggle and review results as a unified experiment.
  1141. job_type: Specify the type of run, especially helpful when organizing runs
  1142. within a group as part of a larger experiment. For example, in a group,
  1143. you might label runs with job types such as "train" and "eval".
  1144. Defining job types enables you to easily filter and group similar runs
  1145. in the UI, facilitating direct comparisons.
  1146. mode: Specifies how run data is managed, with the following options:
  1147. - `"online"` (default): Enables live syncing with W&B when a network
  1148. connection is available, with real-time updates to visualizations.
  1149. - `"offline"`: Suitable for air-gapped or offline environments; data
  1150. is saved locally and can be synced later. Ensure the run folder
  1151. is preserved to enable future syncing.
  1152. - `"disabled"`: Disables all W&B functionality, making the run’s methods
  1153. no-ops. Typically used in testing to bypass W&B operations.
  1154. - `"shared"`: (This is an experimental feature). Allows multiple processes,
  1155. possibly on different machines, to simultaneously log to the same run.
  1156. In this approach you use a primary node and one or more worker nodes
  1157. to log data to the same run. Within the primary node you
  1158. initialize a run. For each worker node, initialize a run
  1159. using the run ID used by the primary node.
  1160. force: Determines if a W&B login is required to run the script. If `True`,
  1161. the user must be logged in to W&B; otherwise, the script will not
  1162. proceed. If `False` (default), the script can proceed without a login,
  1163. switching to offline mode if the user is not logged in.
  1164. reinit: Shorthand for the "reinit" setting. Determines the behavior of
  1165. `wandb.init()` when a run is active.
  1166. resume: Controls the behavior when resuming a run with the specified `id`.
  1167. Available options are:
  1168. - `"allow"`: If a run with the specified `id` exists, it will resume
  1169. from the last step; otherwise, a new run will be created.
  1170. - `"never"`: If a run with the specified `id` exists, an error will
  1171. be raised. If no such run is found, a new run will be created.
  1172. - `"must"`: If a run with the specified `id` exists, it will resume
  1173. from the last step. If no run is found, an error will be raised.
  1174. - `"auto"`: Automatically resumes the previous run if it crashed on
  1175. this machine; otherwise, starts a new run.
  1176. - `True`: Deprecated. Use `"auto"` instead.
  1177. - `False`: Deprecated. Use the default behavior (leaving `resume`
  1178. unset) to always start a new run.
  1179. If `resume` is set, `fork_from` and `resume_from` cannot be
  1180. used. When `resume` is unset, the system will always start a new run.
  1181. resume_from: Specifies a moment in a previous run to resume a run from,
  1182. using the format `{run_id}?_step={step}`. This allows users to truncate
  1183. the history logged to a run at an intermediate step and resume logging
  1184. from that step. The target run must be in the same project.
  1185. If an `id` argument is also provided, the `resume_from` argument will
  1186. take precedence.
  1187. `resume`, `resume_from` and `fork_from` cannot be used together, only
  1188. one of them can be used at a time.
  1189. Note that this feature is in beta and may change in the future.
  1190. fork_from: Specifies a point in a previous run from which to fork a new
  1191. run, using the format `{id}?_step={step}`. This creates a new run that
  1192. resumes logging from the specified step in the target run’s history.
  1193. The target run must be part of the current project.
  1194. If an `id` argument is also provided, it must be different from the
  1195. `fork_from` argument, an error will be raised if they are the same.
  1196. `resume`, `resume_from` and `fork_from` cannot be used together, only
  1197. one of them can be used at a time.
  1198. Note that this feature is in beta and may change in the future.
  1199. save_code: Enables saving the main script or notebook to W&B, aiding in
  1200. experiment reproducibility and allowing code comparisons across runs in
  1201. the UI. By default, this is disabled, but you can change the default to
  1202. enable on your settings page.
  1203. tensorboard: Deprecated. Use `sync_tensorboard` instead.
  1204. sync_tensorboard: Enables automatic syncing of W&B logs from TensorBoard
  1205. or TensorBoardX, saving relevant event files for viewing in
  1206. the W&B UI.
  1207. monitor_gym: Enables automatic logging of videos of the environment when
  1208. using OpenAI Gym.
  1209. settings: Specifies a dictionary or `wandb.Settings` object with advanced
  1210. settings for the run.
  1211. Returns:
  1212. A `Run` object.
  1213. Raises:
  1214. Error: If some unknown or internal error happened during the run
  1215. initialization.
  1216. AuthenticationError: If the user failed to provide valid credentials.
  1217. CommError: If there was a problem communicating with the WandB server.
  1218. UsageError: If the user provided invalid arguments.
  1219. KeyboardInterrupt: If user interrupts the run.
  1220. Examples:
  1221. `wandb.init()` returns a `Run` object. Use the run object to log data,
  1222. save artifacts, and manage the run lifecycle.
  1223. ```python
  1224. import wandb
  1225. config = {"lr": 0.01, "batch_size": 32}
  1226. with wandb.init(config=config) as run:
  1227. # Log accuracy and loss to the run
  1228. acc = 0.95 # Example accuracy
  1229. loss = 0.05 # Example loss
  1230. run.log({"accuracy": acc, "loss": loss})
  1231. ```
  1232. """
  1233. init_telemetry = telemetry.TelemetryRecord()
  1234. init_settings = Settings()
  1235. if isinstance(settings, dict):
  1236. init_settings = Settings(**settings)
  1237. elif isinstance(settings, Settings):
  1238. init_settings = settings
  1239. # Explicit function arguments take precedence over settings
  1240. if job_type is not None:
  1241. init_settings.run_job_type = job_type
  1242. if dir is not None:
  1243. init_settings.root_dir = dir # type: ignore
  1244. if project is not None:
  1245. init_settings.project = project
  1246. if entity is not None:
  1247. init_settings.entity = entity
  1248. if reinit is not None:
  1249. init_settings.reinit = reinit
  1250. if tags is not None:
  1251. init_settings.run_tags = tuple(tags)
  1252. if group is not None:
  1253. init_settings.run_group = group
  1254. if name is not None:
  1255. init_settings.run_name = name
  1256. if notes is not None:
  1257. init_settings.run_notes = notes
  1258. if anonymous is not UNSET:
  1259. init_settings.anonymous = anonymous
  1260. if mode is not None:
  1261. init_settings.mode = mode # type: ignore
  1262. if resume is not None:
  1263. init_settings.resume = resume # type: ignore
  1264. if force is not None:
  1265. init_settings.force = force
  1266. # TODO: deprecate "tensorboard" in favor of "sync_tensorboard"
  1267. if tensorboard is not None:
  1268. init_settings.sync_tensorboard = tensorboard
  1269. if sync_tensorboard is not None:
  1270. init_settings.sync_tensorboard = sync_tensorboard
  1271. if save_code is not None:
  1272. init_settings.save_code = save_code
  1273. if id is not None:
  1274. init_settings.run_id = id
  1275. if fork_from is not None:
  1276. init_settings.fork_from = fork_from # type: ignore
  1277. if resume_from is not None:
  1278. init_settings.resume_from = resume_from # type: ignore
  1279. if config is not None:
  1280. init_telemetry.feature.set_init_config = True
  1281. wl: wandb_setup._WandbSetup | None = None
  1282. try:
  1283. wl = wandb_setup.singleton()
  1284. wi = _WandbInit(wl, init_telemetry)
  1285. wi.maybe_login(init_settings)
  1286. run_settings, show_warnings = wi.make_run_settings(init_settings)
  1287. if isinstance(run_settings.reinit, bool):
  1288. wi.deprecated_features_used.append(
  1289. (
  1290. Deprecated(run__reinit_bool=True),
  1291. "Using a boolean value for 'reinit' is deprecated."
  1292. " Use 'return_previous' or 'finish_previous' instead.",
  1293. )
  1294. )
  1295. if run_settings.run_id is not None:
  1296. init_telemetry.feature.set_init_id = True
  1297. if run_settings.run_name is not None:
  1298. init_telemetry.feature.set_init_name = True
  1299. if run_settings.run_tags is not None:
  1300. init_telemetry.feature.set_init_tags = True
  1301. if run_settings._offline:
  1302. init_telemetry.feature.offline = True
  1303. if run_settings.fork_from is not None:
  1304. init_telemetry.feature.fork_mode = True
  1305. if run_settings.resume_from is not None:
  1306. init_telemetry.feature.rewind_mode = True
  1307. wi.set_run_id(run_settings)
  1308. wi.set_sync_dir_suffix(run_settings)
  1309. run_printer = printer.new_printer(run_settings)
  1310. show_warnings(run_printer)
  1311. with contextlib.ExitStack() as exit_stack:
  1312. exit_stack.enter_context(wb_logging.log_to_run(run_settings.run_id))
  1313. run_config = wi.make_run_config(
  1314. settings=run_settings,
  1315. config=config,
  1316. config_exclude_keys=config_exclude_keys,
  1317. config_include_keys=config_include_keys,
  1318. )
  1319. if run_settings._noop:
  1320. return wi.make_disabled_run(run_config)
  1321. try_create_root_dir(run_settings)
  1322. exit_stack.enter_context(wi.setup_run_log_directory(run_settings))
  1323. if run_settings._jupyter:
  1324. wi.monkeypatch_ipython(run_settings)
  1325. if monitor_gym:
  1326. _monkeypatch_openai_gym()
  1327. if wandb.patched["tensorboard"]:
  1328. # NOTE: The user may have called the patch function directly.
  1329. init_telemetry.feature.tensorboard_patch = True
  1330. if run_settings.sync_tensorboard:
  1331. _monkeypatch_tensorboard()
  1332. init_telemetry.feature.tensorboard_sync = True
  1333. if run_settings.x_server_side_derived_summary:
  1334. init_telemetry.feature.server_side_derived_summary = True
  1335. run = wi.init(run_settings, run_config, run_printer)
  1336. # Set up automatic Weave integration if Weave is installed
  1337. weave.setup(run_settings.entity, run_settings.project)
  1338. return run
  1339. except KeyboardInterrupt as e:
  1340. if wl:
  1341. wl._get_logger().warning("interrupted", exc_info=e)
  1342. raise
  1343. except Exception as e:
  1344. if wl:
  1345. wl._get_logger().exception("error in wandb.init()", exc_info=e)
  1346. get_sentry().reraise(e)