loader.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179
  1. """A simple configuration system."""
  2. # Copyright (c) IPython Development Team.
  3. # Distributed under the terms of the Modified BSD License.
  4. from __future__ import annotations
  5. import argparse
  6. import copy
  7. import functools
  8. import json
  9. import os
  10. import re
  11. import sys
  12. import typing as t
  13. from logging import Logger
  14. from traitlets.traitlets import Any, Container, Dict, HasTraits, List, TraitType, Undefined
  15. from ..utils import cast_unicode, filefind, warnings
  16. # -----------------------------------------------------------------------------
  17. # Exceptions
  18. # -----------------------------------------------------------------------------
  19. class ConfigError(Exception):
  20. pass
  21. class ConfigLoaderError(ConfigError):
  22. pass
  23. class ConfigFileNotFound(ConfigError):
  24. pass
  25. class ArgumentError(ConfigLoaderError):
  26. pass
  27. # -----------------------------------------------------------------------------
  28. # Argparse fix
  29. # -----------------------------------------------------------------------------
  30. # Unfortunately argparse by default prints help messages to stderr instead of
  31. # stdout. This makes it annoying to capture long help screens at the command
  32. # line, since one must know how to pipe stderr, which many users don't know how
  33. # to do. So we override the print_help method with one that defaults to
  34. # stdout and use our class instead.
  35. class _Sentinel:
  36. def __repr__(self) -> str:
  37. return "<Sentinel deprecated>"
  38. def __str__(self) -> str:
  39. return "<deprecated>"
  40. _deprecated = _Sentinel()
  41. class ArgumentParser(argparse.ArgumentParser):
  42. """Simple argparse subclass that prints help to stdout by default."""
  43. def print_help(self, file: t.Any = None) -> None:
  44. if file is None:
  45. file = sys.stdout
  46. return super().print_help(file)
  47. print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
  48. # -----------------------------------------------------------------------------
  49. # Config class for holding config information
  50. # -----------------------------------------------------------------------------
  51. def execfile(fname: str, glob: dict[str, Any]) -> None:
  52. with open(fname, "rb") as f:
  53. exec(compile(f.read(), fname, "exec"), glob, glob) # noqa: S102
  54. class LazyConfigValue(HasTraits):
  55. """Proxy object for exposing methods on configurable containers
  56. These methods allow appending/extending/updating
  57. to add to non-empty defaults instead of clobbering them.
  58. Exposes:
  59. - append, extend, insert on lists
  60. - update on dicts
  61. - update, add on sets
  62. """
  63. _value = None
  64. # list methods
  65. _extend: List[t.Any] = List()
  66. _prepend: List[t.Any] = List()
  67. _inserts: List[t.Any] = List()
  68. def append(self, obj: t.Any) -> None:
  69. """Append an item to a List"""
  70. self._extend.append(obj)
  71. def extend(self, other: t.Any) -> None:
  72. """Extend a list"""
  73. self._extend.extend(other)
  74. def prepend(self, other: t.Any) -> None:
  75. """like list.extend, but for the front"""
  76. self._prepend[:0] = other
  77. def merge_into(self, other: t.Any) -> t.Any:
  78. """
  79. Merge with another earlier LazyConfigValue or an earlier container.
  80. This is useful when having global system-wide configuration files.
  81. Self is expected to have higher precedence.
  82. Parameters
  83. ----------
  84. other : LazyConfigValue or container
  85. Returns
  86. -------
  87. LazyConfigValue
  88. if ``other`` is also lazy, a reified container otherwise.
  89. """
  90. if isinstance(other, LazyConfigValue):
  91. other._extend.extend(self._extend)
  92. self._extend = other._extend
  93. self._prepend.extend(other._prepend)
  94. other._inserts.extend(self._inserts)
  95. self._inserts = other._inserts
  96. if self._update:
  97. other.update(self._update)
  98. self._update = other._update
  99. return self
  100. else:
  101. # other is a container, reify now.
  102. return self.get_value(other)
  103. def insert(self, index: int, other: t.Any) -> None:
  104. if not isinstance(index, int):
  105. raise TypeError("An integer is required")
  106. self._inserts.append((index, other))
  107. # dict methods
  108. # update is used for both dict and set
  109. _update = Any()
  110. def update(self, other: t.Any) -> None:
  111. """Update either a set or dict"""
  112. if self._update is None:
  113. if isinstance(other, dict):
  114. self._update = {}
  115. else:
  116. self._update = set()
  117. self._update.update(other)
  118. # set methods
  119. def add(self, obj: t.Any) -> None:
  120. """Add an item to a set"""
  121. self.update({obj})
  122. def get_value(self, initial: t.Any) -> t.Any:
  123. """construct the value from the initial one
  124. after applying any insert / extend / update changes
  125. """
  126. if self._value is not None:
  127. return self._value # type:ignore[unreachable]
  128. value = copy.deepcopy(initial)
  129. if isinstance(value, list):
  130. for idx, obj in self._inserts:
  131. value.insert(idx, obj)
  132. value[:0] = self._prepend
  133. value.extend(self._extend)
  134. elif isinstance(value, dict):
  135. if self._update:
  136. value.update(self._update)
  137. elif isinstance(value, set):
  138. if self._update:
  139. value.update(self._update)
  140. self._value = value
  141. return value
  142. def to_dict(self) -> dict[str, t.Any]:
  143. """return JSONable dict form of my data
  144. Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples.
  145. """
  146. d = {}
  147. if self._update:
  148. d["update"] = self._update
  149. if self._extend:
  150. d["extend"] = self._extend
  151. if self._prepend:
  152. d["prepend"] = self._prepend
  153. elif self._inserts:
  154. d["inserts"] = self._inserts
  155. return d
  156. def __repr__(self) -> str:
  157. if self._value is not None:
  158. return f"<{self.__class__.__name__} value={self._value!r}>"
  159. else:
  160. return f"<{self.__class__.__name__} {self.to_dict()!r}>"
  161. def _is_section_key(key: str) -> bool:
  162. """Is a Config key a section name (does it start with a capital)?"""
  163. return bool(key and key[0].upper() == key[0] and not key.startswith("_"))
  164. class Config(dict): # type:ignore[type-arg]
  165. """An attribute-based dict that can do smart merges.
  166. Accessing a field on a config object for the first time populates the key
  167. with either a nested Config object for keys starting with capitals
  168. or :class:`.LazyConfigValue` for lowercase keys,
  169. allowing quick assignments such as::
  170. c = Config()
  171. c.Class.int_trait = 5
  172. c.Class.list_trait.append("x")
  173. """
  174. def __init__(self, *args: t.Any, **kwds: t.Any) -> None:
  175. dict.__init__(self, *args, **kwds)
  176. self._ensure_subconfig()
  177. def _ensure_subconfig(self) -> None:
  178. """ensure that sub-dicts that should be Config objects are
  179. casts dicts that are under section keys to Config objects,
  180. which is necessary for constructing Config objects from dict literals.
  181. """
  182. for key in self:
  183. obj = self[key]
  184. if _is_section_key(key) and isinstance(obj, dict) and not isinstance(obj, Config):
  185. setattr(self, key, Config(obj))
  186. def _merge(self, other: t.Any) -> None:
  187. """deprecated alias, use Config.merge()"""
  188. self.merge(other)
  189. def merge(self, other: t.Any) -> None:
  190. """merge another config object into this one"""
  191. to_update = {}
  192. for k, v in other.items():
  193. if k not in self:
  194. to_update[k] = v
  195. else: # I have this key
  196. if isinstance(v, Config) and isinstance(self[k], Config):
  197. # Recursively merge common sub Configs
  198. self[k].merge(v)
  199. elif isinstance(v, LazyConfigValue):
  200. self[k] = v.merge_into(self[k])
  201. else:
  202. # Plain updates for non-Configs
  203. to_update[k] = v
  204. self.update(to_update)
  205. def collisions(self, other: Config) -> dict[str, t.Any]:
  206. """Check for collisions between two config objects.
  207. Returns a dict of the form {"Class": {"trait": "collision message"}}`,
  208. indicating which values have been ignored.
  209. An empty dict indicates no collisions.
  210. """
  211. collisions: dict[str, t.Any] = {}
  212. for section in self:
  213. if section not in other:
  214. continue
  215. mine = self[section]
  216. theirs = other[section]
  217. for key in mine:
  218. if key in theirs and mine[key] != theirs[key]:
  219. collisions.setdefault(section, {})
  220. collisions[section][key] = f"{mine[key]!r} ignored, using {theirs[key]!r}"
  221. return collisions
  222. def __contains__(self, key: t.Any) -> bool:
  223. # allow nested contains of the form `"Section.key" in config`
  224. if "." in key:
  225. first, remainder = key.split(".", 1)
  226. if first not in self:
  227. return False
  228. return remainder in self[first]
  229. return super().__contains__(key)
  230. # .has_key is deprecated for dictionaries.
  231. has_key = __contains__
  232. def _has_section(self, key: str) -> bool:
  233. return _is_section_key(key) and key in self
  234. def copy(self) -> dict[str, t.Any]:
  235. return type(self)(dict.copy(self))
  236. def __copy__(self) -> dict[str, t.Any]:
  237. return self.copy()
  238. def __deepcopy__(self, memo: t.Any) -> Config:
  239. new_config = type(self)()
  240. for key, value in self.items():
  241. if isinstance(value, (Config, LazyConfigValue)):
  242. # deep copy config objects
  243. value = copy.deepcopy(value, memo)
  244. elif type(value) in {dict, list, set, tuple}:
  245. # shallow copy plain container traits
  246. value = copy.copy(value)
  247. new_config[key] = value
  248. return new_config
  249. def __getitem__(self, key: str) -> t.Any:
  250. try:
  251. return dict.__getitem__(self, key)
  252. except KeyError:
  253. if _is_section_key(key):
  254. c = Config()
  255. dict.__setitem__(self, key, c)
  256. return c
  257. elif not key.startswith("_"):
  258. # undefined, create lazy value, used for container methods
  259. v = LazyConfigValue()
  260. dict.__setitem__(self, key, v)
  261. return v
  262. else:
  263. raise
  264. def __setitem__(self, key: str, value: t.Any) -> None:
  265. if _is_section_key(key):
  266. if not isinstance(value, Config):
  267. raise ValueError(
  268. "values whose keys begin with an uppercase "
  269. f"char must be Config instances: {key!r}, {value!r}"
  270. )
  271. dict.__setitem__(self, key, value)
  272. def __getattr__(self, key: str) -> t.Any:
  273. if key.startswith("__"):
  274. return dict.__getattr__(self, key) # type:ignore[attr-defined]
  275. try:
  276. return self.__getitem__(key)
  277. except KeyError as e:
  278. raise AttributeError(e) from e
  279. def __setattr__(self, key: str, value: t.Any) -> None:
  280. if key.startswith("__"):
  281. return dict.__setattr__(self, key, value)
  282. try:
  283. self.__setitem__(key, value)
  284. except KeyError as e:
  285. raise AttributeError(e) from e
  286. def __delattr__(self, key: str) -> None:
  287. if key.startswith("__"):
  288. return dict.__delattr__(self, key)
  289. try:
  290. dict.__delitem__(self, key)
  291. except KeyError as e:
  292. raise AttributeError(e) from e
  293. class DeferredConfig:
  294. """Class for deferred-evaluation of config from CLI"""
  295. def get_value(self, trait: TraitType[t.Any, t.Any]) -> t.Any:
  296. raise NotImplementedError("Implement in subclasses")
  297. def _super_repr(self) -> str:
  298. # explicitly call super on direct parent
  299. return super(self.__class__, self).__repr__()
  300. class DeferredConfigString(str, DeferredConfig):
  301. """Config value for loading config from a string
  302. Interpretation is deferred until it is loaded into the trait.
  303. Subclass of str for backward compatibility.
  304. This class is only used for values that are not listed
  305. in the configurable classes.
  306. When config is loaded, `trait.from_string` will be used.
  307. If an error is raised in `.from_string`,
  308. the original string is returned.
  309. .. versionadded:: 5.0
  310. """
  311. def get_value(self, trait: TraitType[t.Any, t.Any]) -> t.Any:
  312. """Get the value stored in this string"""
  313. s = str(self)
  314. try:
  315. return trait.from_string(s)
  316. except Exception:
  317. # exception casting from string,
  318. # let the original string lie.
  319. # this will raise a more informative error when config is loaded.
  320. return s
  321. def __repr__(self) -> str:
  322. return f"{self.__class__.__name__}({self._super_repr()})"
  323. class DeferredConfigList(t.List[t.Any], DeferredConfig):
  324. """Config value for loading config from a list of strings
  325. Interpretation is deferred until it is loaded into the trait.
  326. This class is only used for values that are not listed
  327. in the configurable classes.
  328. When config is loaded, `trait.from_string_list` will be used.
  329. If an error is raised in `.from_string_list`,
  330. the original string list is returned.
  331. .. versionadded:: 5.0
  332. """
  333. def get_value(self, trait: TraitType[t.Any, t.Any]) -> t.Any:
  334. """Get the value stored in this string"""
  335. if hasattr(trait, "from_string_list"):
  336. src = list(self)
  337. cast = trait.from_string_list
  338. else:
  339. # only allow one item
  340. if len(self) > 1:
  341. raise ValueError(
  342. f"{trait.name} only accepts one value, got {len(self)}: {list(self)}"
  343. )
  344. src = self[0]
  345. cast = trait.from_string
  346. try:
  347. return cast(src)
  348. except Exception:
  349. # exception casting from string,
  350. # let the original value lie.
  351. # this will raise a more informative error when config is loaded.
  352. return src
  353. def __repr__(self) -> str:
  354. return f"{self.__class__.__name__}({self._super_repr()})"
  355. # -----------------------------------------------------------------------------
  356. # Config loading classes
  357. # -----------------------------------------------------------------------------
  358. class ConfigLoader:
  359. """A object for loading configurations from just about anywhere.
  360. The resulting configuration is packaged as a :class:`Config`.
  361. Notes
  362. -----
  363. A :class:`ConfigLoader` does one thing: load a config from a source
  364. (file, command line arguments) and returns the data as a :class:`Config` object.
  365. There are lots of things that :class:`ConfigLoader` does not do. It does
  366. not implement complex logic for finding config files. It does not handle
  367. default values or merge multiple configs. These things need to be
  368. handled elsewhere.
  369. """
  370. def _log_default(self) -> Logger:
  371. from traitlets.log import get_logger
  372. return t.cast(Logger, get_logger())
  373. def __init__(self, log: Logger | None = None) -> None:
  374. """A base class for config loaders.
  375. log : instance of :class:`logging.Logger` to use.
  376. By default logger of :meth:`traitlets.config.application.Application.instance()`
  377. will be used
  378. Examples
  379. --------
  380. >>> cl = ConfigLoader()
  381. >>> config = cl.load_config()
  382. >>> config
  383. {}
  384. """
  385. self.clear()
  386. if log is None:
  387. self.log = self._log_default()
  388. self.log.debug("Using default logger")
  389. else:
  390. self.log = log
  391. def clear(self) -> None:
  392. self.config = Config()
  393. def load_config(self) -> Config:
  394. """Load a config from somewhere, return a :class:`Config` instance.
  395. Usually, this will cause self.config to be set and then returned.
  396. However, in most cases, :meth:`ConfigLoader.clear` should be called
  397. to erase any previous state.
  398. """
  399. self.clear()
  400. return self.config
  401. class FileConfigLoader(ConfigLoader):
  402. """A base class for file based configurations.
  403. As we add more file based config loaders, the common logic should go
  404. here.
  405. """
  406. def __init__(self, filename: str, path: str | None = None, **kw: t.Any) -> None:
  407. """Build a config loader for a filename and path.
  408. Parameters
  409. ----------
  410. filename : str
  411. The file name of the config file.
  412. path : str, list, tuple
  413. The path to search for the config file on, or a sequence of
  414. paths to try in order.
  415. """
  416. super().__init__(**kw)
  417. self.filename = filename
  418. self.path = path
  419. self.full_filename = ""
  420. def _find_file(self) -> None:
  421. """Try to find the file by searching the paths."""
  422. self.full_filename = filefind(self.filename, self.path)
  423. class JSONFileConfigLoader(FileConfigLoader):
  424. """A JSON file loader for config
  425. Can also act as a context manager that rewrite the configuration file to disk on exit.
  426. Example::
  427. with JSONFileConfigLoader('myapp.json','/home/jupyter/configurations/') as c:
  428. c.MyNewConfigurable.new_value = 'Updated'
  429. """
  430. def load_config(self) -> Config:
  431. """Load the config from a file and return it as a Config object."""
  432. self.clear()
  433. try:
  434. self._find_file()
  435. except OSError as e:
  436. raise ConfigFileNotFound(str(e)) from e
  437. dct = self._read_file_as_dict()
  438. self.config = self._convert_to_config(dct)
  439. return self.config
  440. def _read_file_as_dict(self) -> dict[str, t.Any]:
  441. with open(self.full_filename) as f:
  442. return t.cast("dict[str, t.Any]", json.load(f))
  443. def _convert_to_config(self, dictionary: dict[str, t.Any]) -> Config:
  444. if "version" in dictionary:
  445. version = dictionary.pop("version")
  446. else:
  447. version = 1
  448. if version == 1:
  449. return Config(dictionary)
  450. else:
  451. raise ValueError(f"Unknown version of JSON config file: {version}")
  452. def __enter__(self) -> Config:
  453. self.load_config()
  454. return self.config
  455. def __exit__(self, exc_type: object, exc_value: object, traceback: object) -> None:
  456. """
  457. Exit the context manager but do not handle any errors.
  458. In case of any error, we do not want to write the potentially broken
  459. configuration to disk.
  460. """
  461. self.config.version = 1
  462. json_config = json.dumps(self.config, indent=2)
  463. with open(self.full_filename, "w") as f:
  464. f.write(json_config)
  465. class PyFileConfigLoader(FileConfigLoader):
  466. """A config loader for pure python files.
  467. This is responsible for locating a Python config file by filename and
  468. path, then executing it to construct a Config object.
  469. """
  470. def load_config(self) -> Config:
  471. """Load the config from a file and return it as a Config object."""
  472. self.clear()
  473. try:
  474. self._find_file()
  475. except OSError as e:
  476. raise ConfigFileNotFound(str(e)) from e
  477. self._read_file_as_dict()
  478. return self.config
  479. def load_subconfig(self, fname: str, path: str | None = None) -> None:
  480. """Injected into config file namespace as load_subconfig"""
  481. if path is None:
  482. path = self.path
  483. loader = self.__class__(fname, path)
  484. try:
  485. sub_config = loader.load_config()
  486. except ConfigFileNotFound:
  487. # Pass silently if the sub config is not there,
  488. # treat it as an empty config file.
  489. pass
  490. else:
  491. self.config.merge(sub_config)
  492. def _read_file_as_dict(self) -> None:
  493. """Load the config file into self.config, with recursive loading."""
  494. def get_config() -> Config:
  495. """Unnecessary now, but a deprecation warning is more trouble than it's worth."""
  496. return self.config
  497. namespace = dict( # noqa: C408
  498. c=self.config,
  499. load_subconfig=self.load_subconfig,
  500. get_config=get_config,
  501. __file__=self.full_filename,
  502. )
  503. conf_filename = self.full_filename
  504. with open(conf_filename, "rb") as f:
  505. exec(compile(f.read(), conf_filename, "exec"), namespace, namespace) # noqa: S102
  506. class CommandLineConfigLoader(ConfigLoader):
  507. """A config loader for command line arguments.
  508. As we add more command line based loaders, the common logic should go
  509. here.
  510. """
  511. def _exec_config_str(
  512. self, lhs: t.Any, rhs: t.Any, trait: TraitType[t.Any, t.Any] | None = None
  513. ) -> None:
  514. """execute self.config.<lhs> = <rhs>
  515. * expands ~ with expanduser
  516. * interprets value with trait if available
  517. """
  518. value = rhs
  519. if isinstance(value, DeferredConfig):
  520. if trait:
  521. # trait available, reify config immediately
  522. value = value.get_value(trait)
  523. elif isinstance(rhs, DeferredConfigList) and len(rhs) == 1:
  524. # single item, make it a deferred str
  525. value = DeferredConfigString(os.path.expanduser(rhs[0]))
  526. else:
  527. if trait:
  528. value = trait.from_string(value)
  529. else:
  530. value = DeferredConfigString(value)
  531. *path, key = lhs.split(".")
  532. section = self.config
  533. for part in path:
  534. section = section[part]
  535. section[key] = value
  536. return
  537. def _load_flag(self, cfg: t.Any) -> None:
  538. """update self.config from a flag, which can be a dict or Config"""
  539. if isinstance(cfg, (dict, Config)):
  540. # don't clobber whole config sections, update
  541. # each section from config:
  542. for sec, c in cfg.items():
  543. self.config[sec].update(c)
  544. else:
  545. raise TypeError("Invalid flag: %r" % cfg)
  546. # match --Class.trait keys for argparse
  547. # matches:
  548. # --Class.trait
  549. # --x
  550. # -x
  551. class_trait_opt_pattern = re.compile(r"^\-?\-[A-Za-z][\w]*(\.[\w]+)*$")
  552. _DOT_REPLACEMENT = "__DOT__"
  553. _DASH_REPLACEMENT = "__DASH__"
  554. class _KVAction(argparse.Action):
  555. """Custom argparse action for handling --Class.trait=x
  556. Always
  557. """
  558. def __call__( # type:ignore[override]
  559. self,
  560. parser: argparse.ArgumentParser,
  561. namespace: dict[str, t.Any],
  562. values: t.Sequence[t.Any],
  563. option_string: str | None = None,
  564. ) -> None:
  565. if isinstance(values, str):
  566. values = [values]
  567. values = ["-" if v is _DASH_REPLACEMENT else v for v in values]
  568. items = getattr(namespace, self.dest, None)
  569. if items is None:
  570. items = DeferredConfigList()
  571. else:
  572. items = DeferredConfigList(items)
  573. items.extend(values)
  574. setattr(namespace, self.dest, items)
  575. class _DefaultOptionDict(dict): # type:ignore[type-arg]
  576. """Like the default options dict
  577. but acts as if all --Class.trait options are predefined
  578. """
  579. def _add_kv_action(self, key: str) -> None:
  580. self[key] = _KVAction(
  581. option_strings=[key],
  582. dest=key.lstrip("-").replace(".", _DOT_REPLACEMENT),
  583. # use metavar for display purposes
  584. metavar=key.lstrip("-"),
  585. )
  586. def __contains__(self, key: t.Any) -> bool:
  587. if "=" in key:
  588. return False
  589. if super().__contains__(key):
  590. return True
  591. if key.startswith("-") and class_trait_opt_pattern.match(key):
  592. self._add_kv_action(key)
  593. return True
  594. return False
  595. def __getitem__(self, key: str) -> t.Any:
  596. if key in self:
  597. return super().__getitem__(key)
  598. else:
  599. raise KeyError(key)
  600. def get(self, key: str, default: t.Any = None) -> t.Any:
  601. try:
  602. return self[key]
  603. except KeyError:
  604. return default
  605. class _KVArgParser(argparse.ArgumentParser):
  606. """subclass of ArgumentParser where any --Class.trait option is implicitly defined"""
  607. def parse_known_args( # type:ignore[override]
  608. self, args: t.Sequence[str] | None = None, namespace: argparse.Namespace | None = None
  609. ) -> tuple[argparse.Namespace | None, list[str]]:
  610. # must be done immediately prior to parsing because if we do it in init,
  611. # registration of explicit actions via parser.add_option will fail during setup
  612. for container in (self, self._optionals):
  613. container._option_string_actions = _DefaultOptionDict(container._option_string_actions)
  614. return super().parse_known_args(args, namespace)
  615. # type aliases
  616. SubcommandsDict = t.Dict[str, t.Any]
  617. class ArgParseConfigLoader(CommandLineConfigLoader):
  618. """A loader that uses the argparse module to load from the command line."""
  619. parser_class = ArgumentParser
  620. def __init__(
  621. self,
  622. argv: list[str] | None = None,
  623. aliases: dict[str, str] | None = None,
  624. flags: dict[str, str] | None = None,
  625. log: t.Any = None,
  626. classes: list[type[t.Any]] | None = None,
  627. subcommands: SubcommandsDict | None = None,
  628. *parser_args: t.Any,
  629. **parser_kw: t.Any,
  630. ) -> None:
  631. """Create a config loader for use with argparse.
  632. Parameters
  633. ----------
  634. classes : optional, list
  635. The classes to scan for *container* config-traits and decide
  636. for their "multiplicity" when adding them as *argparse* arguments.
  637. argv : optional, list
  638. If given, used to read command-line arguments from, otherwise
  639. sys.argv[1:] is used.
  640. *parser_args : tuple
  641. A tuple of positional arguments that will be passed to the
  642. constructor of :class:`argparse.ArgumentParser`.
  643. **parser_kw : dict
  644. A tuple of keyword arguments that will be passed to the
  645. constructor of :class:`argparse.ArgumentParser`.
  646. aliases : dict of str to str
  647. Dict of aliases to full traitlets names for CLI parsing
  648. flags : dict of str to str
  649. Dict of flags to full traitlets names for CLI parsing
  650. log
  651. Passed to `ConfigLoader`
  652. Returns
  653. -------
  654. config : Config
  655. The resulting Config object.
  656. """
  657. classes = classes or []
  658. super(CommandLineConfigLoader, self).__init__(log=log)
  659. self.clear()
  660. if argv is None:
  661. argv = sys.argv[1:]
  662. self.argv = argv
  663. self.aliases = aliases or {}
  664. self.flags = flags or {}
  665. self.classes = classes
  666. self.subcommands = subcommands # only used for argcomplete currently
  667. self.parser_args = parser_args
  668. self.version = parser_kw.pop("version", None)
  669. kwargs = dict(argument_default=argparse.SUPPRESS) # noqa: C408
  670. kwargs.update(parser_kw)
  671. self.parser_kw = kwargs
  672. def load_config(
  673. self,
  674. argv: list[str] | None = None,
  675. aliases: t.Any = None,
  676. flags: t.Any = _deprecated,
  677. classes: t.Any = None,
  678. ) -> Config:
  679. """Parse command line arguments and return as a Config object.
  680. Parameters
  681. ----------
  682. argv : optional, list
  683. If given, a list with the structure of sys.argv[1:] to parse
  684. arguments from. If not given, the instance's self.argv attribute
  685. (given at construction time) is used.
  686. flags
  687. Deprecated in traitlets 5.0, instantiate the config loader with the flags.
  688. """
  689. if flags is not _deprecated:
  690. warnings.warn(
  691. "The `flag` argument to load_config is deprecated since Traitlets "
  692. f"5.0 and will be ignored, pass flags the `{type(self)}` constructor.",
  693. DeprecationWarning,
  694. stacklevel=2,
  695. )
  696. self.clear()
  697. if argv is None:
  698. argv = self.argv
  699. if aliases is not None:
  700. self.aliases = aliases
  701. if classes is not None:
  702. self.classes = classes
  703. self._create_parser()
  704. self._argcomplete(self.classes, self.subcommands)
  705. self._parse_args(argv)
  706. self._convert_to_config()
  707. return self.config
  708. def get_extra_args(self) -> list[str]:
  709. if hasattr(self, "extra_args"):
  710. return self.extra_args
  711. else:
  712. return []
  713. def _create_parser(self) -> None:
  714. self.parser = self.parser_class(
  715. *self.parser_args,
  716. **self.parser_kw, # type:ignore[arg-type]
  717. )
  718. self._add_arguments(self.aliases, self.flags, self.classes)
  719. def _add_arguments(self, aliases: t.Any, flags: t.Any, classes: t.Any) -> None:
  720. raise NotImplementedError("subclasses must implement _add_arguments")
  721. def _argcomplete(self, classes: list[t.Any], subcommands: SubcommandsDict | None) -> None:
  722. """If argcomplete is enabled, allow triggering command-line autocompletion"""
  723. def _parse_args(self, args: t.Any) -> t.Any:
  724. """self.parser->self.parsed_data"""
  725. uargs = [cast_unicode(a) for a in args]
  726. unpacked_aliases: dict[str, str] = {}
  727. if self.aliases:
  728. unpacked_aliases = {}
  729. for alias, alias_target in self.aliases.items():
  730. if alias in self.flags:
  731. continue
  732. if not isinstance(alias, tuple): # type:ignore[unreachable]
  733. alias = (alias,) # type:ignore[assignment]
  734. for al in alias:
  735. if len(al) == 1:
  736. unpacked_aliases["-" + al] = "--" + alias_target
  737. unpacked_aliases["--" + al] = "--" + alias_target
  738. def _replace(arg: str) -> str:
  739. if arg == "-":
  740. return _DASH_REPLACEMENT
  741. for k, v in unpacked_aliases.items():
  742. if arg == k:
  743. return v
  744. if arg.startswith(k + "="):
  745. return v + "=" + arg[len(k) + 1 :]
  746. return arg
  747. if "--" in uargs:
  748. idx = uargs.index("--")
  749. extra_args = uargs[idx + 1 :]
  750. to_parse = uargs[:idx]
  751. else:
  752. extra_args = []
  753. to_parse = uargs
  754. to_parse = [_replace(a) for a in to_parse]
  755. self.parsed_data = self.parser.parse_args(to_parse)
  756. self.extra_args = extra_args
  757. def _convert_to_config(self) -> None:
  758. """self.parsed_data->self.config"""
  759. for k, v in vars(self.parsed_data).items():
  760. *path, key = k.split(".")
  761. section = self.config
  762. for p in path:
  763. section = section[p]
  764. setattr(section, key, v)
  765. class _FlagAction(argparse.Action):
  766. """ArgParse action to handle a flag"""
  767. def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
  768. self.flag = kwargs.pop("flag")
  769. self.alias = kwargs.pop("alias", None)
  770. kwargs["const"] = Undefined
  771. if not self.alias:
  772. kwargs["nargs"] = 0
  773. super().__init__(*args, **kwargs)
  774. def __call__(
  775. self, parser: t.Any, namespace: t.Any, values: t.Any, option_string: str | None = None
  776. ) -> None:
  777. if self.nargs == 0 or values is Undefined:
  778. if not hasattr(namespace, "_flags"):
  779. namespace._flags = []
  780. namespace._flags.append(self.flag)
  781. else:
  782. setattr(namespace, self.alias, values)
  783. class KVArgParseConfigLoader(ArgParseConfigLoader):
  784. """A config loader that loads aliases and flags with argparse,
  785. as well as arbitrary --Class.trait value
  786. """
  787. parser_class = _KVArgParser # type:ignore[assignment]
  788. def _add_arguments(self, aliases: t.Any, flags: t.Any, classes: t.Any) -> None:
  789. alias_flags: dict[str, t.Any] = {}
  790. argparse_kwds: dict[str, t.Any]
  791. argparse_traits: dict[str, t.Any]
  792. paa = self.parser.add_argument
  793. self.parser.set_defaults(_flags=[])
  794. paa("extra_args", nargs="*")
  795. # An index of all container traits collected::
  796. #
  797. # { <traitname>: (<trait>, <argparse-kwds>) }
  798. #
  799. # Used to add the correct type into the `config` tree.
  800. # Used also for aliases, not to re-collect them.
  801. self.argparse_traits = argparse_traits = {}
  802. for cls in classes:
  803. for traitname, trait in cls.class_traits(config=True).items():
  804. argname = f"{cls.__name__}.{traitname}"
  805. argparse_kwds = {"type": str}
  806. if isinstance(trait, (Container, Dict)):
  807. multiplicity = trait.metadata.get("multiplicity", "append")
  808. if multiplicity == "append":
  809. argparse_kwds["action"] = multiplicity
  810. else:
  811. argparse_kwds["nargs"] = multiplicity
  812. argparse_traits[argname] = (trait, argparse_kwds)
  813. for keys, (value, fhelp) in flags.items():
  814. if not isinstance(keys, tuple):
  815. keys = (keys,)
  816. for key in keys:
  817. if key in aliases:
  818. alias_flags[aliases[key]] = value
  819. continue
  820. keys = ("-" + key, "--" + key) if len(key) == 1 else ("--" + key,)
  821. paa(*keys, action=_FlagAction, flag=value, help=fhelp)
  822. for keys, traitname in aliases.items():
  823. if not isinstance(keys, tuple):
  824. keys = (keys,)
  825. for key in keys:
  826. argparse_kwds = {
  827. "type": str,
  828. "dest": traitname.replace(".", _DOT_REPLACEMENT),
  829. "metavar": traitname,
  830. }
  831. argcompleter = None
  832. if traitname in argparse_traits:
  833. trait, kwds = argparse_traits[traitname]
  834. argparse_kwds.update(kwds)
  835. if "action" in argparse_kwds and traitname in alias_flags:
  836. # flag sets 'action', so can't have flag & alias with custom action
  837. # on the same name
  838. raise ArgumentError(
  839. f"The alias `{key}` for the 'append' sequence "
  840. f"config-trait `{traitname}` cannot be also a flag!'"
  841. )
  842. # For argcomplete, check if any either an argcompleter metadata tag or method
  843. # is available. If so, it should be a callable which takes the command-line key
  844. # string as an argument and other kwargs passed by argcomplete,
  845. # and returns the a list of string completions.
  846. argcompleter = trait.metadata.get("argcompleter") or getattr(
  847. trait, "argcompleter", None
  848. )
  849. if traitname in alias_flags:
  850. # alias and flag.
  851. # when called with 0 args: flag
  852. # when called with >= 1: alias
  853. argparse_kwds.setdefault("nargs", "?")
  854. argparse_kwds["action"] = _FlagAction
  855. argparse_kwds["flag"] = alias_flags[traitname]
  856. argparse_kwds["alias"] = traitname
  857. keys = ("-" + key, "--" + key) if len(key) == 1 else ("--" + key,)
  858. action = paa(*keys, **argparse_kwds)
  859. if argcompleter is not None:
  860. # argcomplete's completers are callables returning list of completion strings
  861. action.completer = functools.partial( # type:ignore[attr-defined]
  862. argcompleter, key=key
  863. )
  864. def _convert_to_config(self) -> None:
  865. """self.parsed_data->self.config, parse unrecognized extra args via KVLoader."""
  866. extra_args = self.extra_args
  867. for lhs, rhs in vars(self.parsed_data).items():
  868. if lhs == "extra_args":
  869. self.extra_args = ["-" if a == _DASH_REPLACEMENT else a for a in rhs] + extra_args
  870. continue
  871. if lhs == "_flags":
  872. # _flags will be handled later
  873. continue
  874. lhs = lhs.replace(_DOT_REPLACEMENT, ".")
  875. if "." not in lhs:
  876. self._handle_unrecognized_alias(lhs)
  877. trait = None
  878. if isinstance(rhs, list):
  879. rhs = DeferredConfigList(rhs)
  880. elif isinstance(rhs, str):
  881. rhs = DeferredConfigString(rhs)
  882. trait = self.argparse_traits.get(lhs)
  883. if trait:
  884. trait = trait[0]
  885. # eval the KV assignment
  886. try:
  887. self._exec_config_str(lhs, rhs, trait)
  888. except Exception as e:
  889. # cast deferred to nicer repr for the error
  890. # DeferredList->list, etc
  891. if isinstance(rhs, DeferredConfig):
  892. rhs = rhs._super_repr()
  893. raise ArgumentError(f"Error loading argument {lhs}={rhs}, {e}") from e
  894. for subc in self.parsed_data._flags:
  895. self._load_flag(subc)
  896. def _handle_unrecognized_alias(self, arg: str) -> None:
  897. """Handling for unrecognized alias arguments
  898. Probably a mistyped alias. By default just log a warning,
  899. but users can override this to raise an error instead, e.g.
  900. self.parser.error("Unrecognized alias: '%s'" % arg)
  901. """
  902. self.log.warning("Unrecognized alias: '%s', it will have no effect.", arg)
  903. def _argcomplete(self, classes: list[t.Any], subcommands: SubcommandsDict | None) -> None:
  904. """If argcomplete is enabled, allow triggering command-line autocompletion"""
  905. try:
  906. import argcomplete # noqa: F401
  907. except ImportError:
  908. return
  909. from . import argcomplete_config
  910. finder = argcomplete_config.ExtendedCompletionFinder() # type:ignore[no-untyped-call]
  911. finder.config_classes = classes
  912. finder.subcommands = list(subcommands or [])
  913. # for ease of testing, pass through self._argcomplete_kwargs if set
  914. finder(self.parser, **getattr(self, "_argcomplete_kwargs", {}))
  915. class KeyValueConfigLoader(KVArgParseConfigLoader):
  916. """Deprecated in traitlets 5.0
  917. Use KVArgParseConfigLoader
  918. """
  919. def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
  920. warnings.warn(
  921. "KeyValueConfigLoader is deprecated since Traitlets 5.0."
  922. " Use KVArgParseConfigLoader instead.",
  923. DeprecationWarning,
  924. stacklevel=2,
  925. )
  926. super().__init__(*args, **kwargs)
  927. def load_pyconfig_files(config_files: list[str], path: str) -> Config:
  928. """Load multiple Python config files, merging each of them in turn.
  929. Parameters
  930. ----------
  931. config_files : list of str
  932. List of config files names to load and merge into the config.
  933. path : unicode
  934. The full path to the location of the config files.
  935. """
  936. config = Config()
  937. for cf in config_files:
  938. loader = PyFileConfigLoader(cf, path=path)
  939. try:
  940. next_config = loader.load_config()
  941. except ConfigFileNotFound:
  942. pass
  943. except Exception:
  944. raise
  945. else:
  946. config.merge(next_config)
  947. return config