autoreload.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915
  1. """IPython extension to reload modules before executing user code.
  2. ``autoreload`` reloads modules automatically before entering the execution of
  3. code typed at the IPython prompt.
  4. This makes for example the following workflow possible:
  5. .. sourcecode:: ipython
  6. In [1]: %load_ext autoreload
  7. In [2]: %autoreload 2
  8. In [3]: from foo import some_function
  9. In [4]: some_function()
  10. Out[4]: 42
  11. In [5]: # open foo.py in an editor and change some_function to return 43
  12. In [6]: some_function()
  13. Out[6]: 43
  14. The module was reloaded without reloading it explicitly, and the object
  15. imported with ``from foo import ...`` was also updated.
  16. Usage
  17. =====
  18. The following magic commands are provided:
  19. ``%autoreload``, ``%autoreload now``
  20. Reload all modules (except those excluded by ``%aimport``)
  21. automatically now.
  22. ``%autoreload 0``, ``%autoreload off``
  23. Disable automatic reloading.
  24. ``%autoreload 1``, ``%autoreload explicit``
  25. Reload all modules imported with ``%aimport`` every time before
  26. executing the Python code typed.
  27. ``%autoreload 2``, ``%autoreload all``
  28. Reload all modules (except those excluded by ``%aimport``) every
  29. time before executing the Python code typed.
  30. ``%autoreload 3``, ``%autoreload complete``
  31. Same as 2/all, but also adds any new objects in the module. See
  32. unit test at IPython/extensions/tests/test_autoreload.py::test_autoload_newly_added_objects
  33. Adding ``--print`` or ``-p`` to the ``%autoreload`` line will print autoreload activity to
  34. standard out. ``--log`` or ``-l`` will do it to the log at INFO level; both can be used
  35. simultaneously.
  36. ``%aimport``
  37. List modules which are to be automatically imported or not to be imported.
  38. ``%aimport foo``
  39. Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
  40. ``%aimport foo, bar``
  41. Import modules 'foo', 'bar' and mark them to be autoreloaded for ``%autoreload 1``
  42. ``%aimport -foo``
  43. Mark module 'foo' to not be autoreloaded.
  44. Import Conflict Resolution
  45. ==========================
  46. In ``%autoreload 3`` mode, the extension tracks ``from X import Y`` style imports
  47. and intelligently resolves conflicts when the same name is imported multiple ways.
  48. Import tracking occurs after successful code execution, ensuring that only valid
  49. imports are tracked. This approach handles edge cases such as:
  50. - Importing a name that doesn't initially exist in a module, then adding that name
  51. to the module and importing it again
  52. - Conflicts between aliased imports (``from X import Y as Z``) and direct imports
  53. (``from X import Z``)
  54. When conflicts occur:
  55. - If you first do ``from X import Y as Z`` then later ``from X import Z``,
  56. the extension will switch to reloading ``Z`` instead of ``Y`` under the name ``Z``.
  57. - Similarly, if you first do ``from X import Z`` then later ``from X import Y as Z``,
  58. the extension will switch to reloading ``Y`` as ``Z`` instead of the original ``Z``.
  59. - The most recent successful import always takes precedence in conflict resolution.
  60. Caveats
  61. =======
  62. Reloading Python modules in a reliable way is in general difficult,
  63. and unexpected things may occur. ``%autoreload`` tries to work around
  64. common pitfalls by replacing function code objects and parts of
  65. classes previously in the module with new versions. This makes the
  66. following things to work:
  67. - Functions and classes imported via 'from xxx import foo' are upgraded
  68. to new versions when 'xxx' is reloaded.
  69. - Methods and properties of classes are upgraded on reload, so that
  70. calling 'c.foo()' on an object 'c' created before the reload causes
  71. the new code for 'foo' to be executed.
  72. Some of the known remaining caveats are:
  73. - Replacing code objects does not always succeed: changing a @property
  74. in a class to an ordinary method or a method to a member variable
  75. can cause problems (but in old objects only).
  76. - Functions that are removed (eg. via monkey-patching) from a module
  77. before it is reloaded are not upgraded.
  78. - C extension modules cannot be reloaded, and so cannot be autoreloaded.
  79. - While comparing Enum and Flag, the 'is' Identity Operator is used (even in the case '==' has been used (Similar to the 'None' keyword)).
  80. - Reloading a module, or importing the same module by a different name, creates new Enums. These may look the same, but are not.
  81. """
  82. from IPython.core import magic_arguments
  83. from IPython.core.magic import Magics, magics_class, line_magic
  84. from IPython.extensions.deduperreload.deduperreload import DeduperReloader
  85. __skip_doctest__ = True
  86. # -----------------------------------------------------------------------------
  87. # Copyright (C) 2000 Thomas Heller
  88. # Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
  89. # Copyright (C) 2012 The IPython Development Team
  90. #
  91. # Distributed under the terms of the BSD License. The full license is in
  92. # the file COPYING, distributed as part of this software.
  93. # -----------------------------------------------------------------------------
  94. #
  95. # This IPython module is written by Pauli Virtanen, based on the autoreload
  96. # code by Thomas Heller.
  97. # -----------------------------------------------------------------------------
  98. # Imports
  99. # -----------------------------------------------------------------------------
  100. import ast
  101. import os
  102. import sys
  103. import traceback
  104. import types
  105. import weakref
  106. import gc
  107. import logging
  108. from importlib import import_module, reload
  109. from importlib.util import source_from_cache
  110. # ------------------------------------------------------------------------------
  111. # Autoreload functionality
  112. # ------------------------------------------------------------------------------
  113. class ModuleReloader:
  114. enabled = False
  115. """Whether this reloader is enabled"""
  116. check_all = True
  117. """Autoreload all modules, not just those listed in 'modules'"""
  118. autoload_obj = False
  119. """Autoreload all modules AND autoload all new objects"""
  120. def __init__(self, shell=None):
  121. # Modules that failed to reload: {module: mtime-on-failed-reload, ...}
  122. self.failed = {}
  123. # Modules specially marked as autoreloadable.
  124. self.modules = {}
  125. # Modules specially marked as not autoreloadable.
  126. self.skip_modules = {}
  127. # (module-name, name) -> weakref, for replacing old code objects
  128. self.old_objects = {}
  129. # Module modification timestamps
  130. self.modules_mtimes = {}
  131. self.shell = shell
  132. # Reporting callable for verbosity
  133. self._report = lambda msg: None # by default, be quiet.
  134. # Deduper reloader
  135. self.deduper_reloader = DeduperReloader()
  136. # Persistent import tracker for from-imports
  137. self.import_from_tracker = ImportFromTracker({}, {})
  138. # Cache module modification times
  139. self.check(check_all=True, do_reload=False)
  140. # To hide autoreload errors
  141. self.hide_errors = False
  142. def mark_module_skipped(self, module_name):
  143. """Skip reloading the named module in the future"""
  144. try:
  145. del self.modules[module_name]
  146. except KeyError:
  147. pass
  148. self.skip_modules[module_name] = True
  149. def mark_module_reloadable(self, module_name):
  150. """Reload the named module in the future (if it is imported)"""
  151. try:
  152. del self.skip_modules[module_name]
  153. except KeyError:
  154. pass
  155. self.modules[module_name] = True
  156. def clear_import_tracker(self):
  157. """Clear the persistent import tracker state"""
  158. self.import_from_tracker = ImportFromTracker({}, {})
  159. def aimport_module(self, module_name):
  160. """Import a module, and mark it reloadable
  161. Returns
  162. -------
  163. top_module : module
  164. The imported module if it is top-level, or the top-level
  165. top_name : module
  166. Name of top_module
  167. """
  168. self.mark_module_reloadable(module_name)
  169. import_module(module_name)
  170. top_name = module_name.split(".")[0]
  171. top_module = sys.modules[top_name]
  172. return top_module, top_name
  173. def filename_and_mtime(self, module):
  174. if not hasattr(module, "__file__") or module.__file__ is None:
  175. return None, None
  176. if getattr(module, "__name__", None) in [None, "__mp_main__", "__main__"]:
  177. # we cannot reload(__main__) or reload(__mp_main__)
  178. return None, None
  179. filename = module.__file__
  180. path, ext = os.path.splitext(filename)
  181. if ext.lower() == ".py":
  182. py_filename = filename
  183. else:
  184. try:
  185. py_filename = source_from_cache(filename)
  186. except ValueError:
  187. return None, None
  188. try:
  189. pymtime = os.stat(py_filename).st_mtime
  190. except OSError:
  191. return None, None
  192. return py_filename, pymtime
  193. def check(self, check_all=False, do_reload=True, execution_info=None):
  194. """Check whether some modules need to be reloaded."""
  195. if not self.enabled and not check_all:
  196. return
  197. if check_all or self.check_all:
  198. modules = list(sys.modules.keys())
  199. else:
  200. modules = list(self.modules.keys())
  201. # Use the persistent import_from_tracker
  202. import_from_tracker = (
  203. self.import_from_tracker if self.import_from_tracker.imports_froms else None
  204. )
  205. for modname in modules:
  206. m = sys.modules.get(modname, None)
  207. if modname in self.skip_modules:
  208. continue
  209. py_filename, pymtime = self.filename_and_mtime(m)
  210. if py_filename is None:
  211. continue
  212. try:
  213. if pymtime <= self.modules_mtimes[modname]:
  214. continue
  215. except KeyError:
  216. self.modules_mtimes[modname] = pymtime
  217. continue
  218. else:
  219. if self.failed.get(py_filename, None) == pymtime:
  220. continue
  221. self.modules_mtimes[modname] = pymtime
  222. # If we've reached this point, we should try to reload the module
  223. if do_reload:
  224. self._report(f"Reloading '{modname}'.")
  225. try:
  226. if self.autoload_obj:
  227. superreload(
  228. m,
  229. reload,
  230. self.old_objects,
  231. self.shell,
  232. import_from_tracker=import_from_tracker,
  233. )
  234. # if not using autoload, check if deduperreload is viable for this module
  235. elif self.deduper_reloader.maybe_reload_module(m):
  236. pass
  237. else:
  238. superreload(m, reload, self.old_objects)
  239. if py_filename in self.failed:
  240. del self.failed[py_filename]
  241. except:
  242. if not self.hide_errors:
  243. logger = logging.getLogger("autoreload")
  244. logger.exception(
  245. f"Failed to reload module '{modname}' from file '{py_filename}'"
  246. )
  247. print(
  248. "[autoreload of {} failed: {}]".format(
  249. modname, traceback.format_exc(10)
  250. ),
  251. file=sys.stderr,
  252. )
  253. self.failed[py_filename] = pymtime
  254. self.deduper_reloader.update_sources()
  255. # ------------------------------------------------------------------------------
  256. # superreload
  257. # ------------------------------------------------------------------------------
  258. func_attrs = [
  259. "__code__",
  260. "__defaults__",
  261. "__doc__",
  262. "__closure__",
  263. "__globals__",
  264. "__dict__",
  265. ]
  266. def update_function(old, new):
  267. """Upgrade the code object of a function"""
  268. for name in func_attrs:
  269. try:
  270. setattr(old, name, getattr(new, name))
  271. except (AttributeError, TypeError):
  272. pass
  273. def update_instances(old, new):
  274. """Use garbage collector to find all instances that refer to the old
  275. class definition and update their __class__ to point to the new class
  276. definition"""
  277. refs = gc.get_referrers(old)
  278. for ref in refs:
  279. if type(ref) is old:
  280. object.__setattr__(ref, "__class__", new)
  281. def update_class(old, new):
  282. """Replace stuff in the __dict__ of a class, and upgrade
  283. method code objects, and add new methods, if any"""
  284. for key in list(old.__dict__.keys()):
  285. old_obj = getattr(old, key)
  286. try:
  287. new_obj = getattr(new, key)
  288. # explicitly checking that comparison returns True to handle
  289. # cases where `==` doesn't return a boolean.
  290. if (old_obj == new_obj) is True:
  291. continue
  292. except AttributeError:
  293. # obsolete attribute: remove it
  294. try:
  295. delattr(old, key)
  296. except (AttributeError, TypeError):
  297. pass
  298. continue
  299. except ValueError:
  300. # can't compare nested structures containing
  301. # numpy arrays using `==`
  302. pass
  303. if update_generic(old_obj, new_obj):
  304. continue
  305. try:
  306. setattr(old, key, getattr(new, key))
  307. except (AttributeError, TypeError):
  308. pass # skip non-writable attributes
  309. for key in list(new.__dict__.keys()):
  310. if key not in list(old.__dict__.keys()):
  311. try:
  312. setattr(old, key, getattr(new, key))
  313. except (AttributeError, TypeError):
  314. pass # skip non-writable attributes
  315. # update all instances of class
  316. update_instances(old, new)
  317. def update_property(old, new):
  318. """Replace get/set/del functions of a property"""
  319. update_generic(old.fdel, new.fdel)
  320. update_generic(old.fget, new.fget)
  321. update_generic(old.fset, new.fset)
  322. def isinstance2(a, b, typ):
  323. return isinstance(a, typ) and isinstance(b, typ)
  324. UPDATE_RULES = [
  325. (lambda a, b: isinstance2(a, b, type), update_class),
  326. (lambda a, b: isinstance2(a, b, types.FunctionType), update_function),
  327. (lambda a, b: isinstance2(a, b, property), update_property),
  328. ]
  329. UPDATE_RULES.extend(
  330. [
  331. (
  332. lambda a, b: isinstance2(a, b, types.MethodType),
  333. lambda a, b: update_function(a.__func__, b.__func__),
  334. ),
  335. ]
  336. )
  337. def update_generic(a, b):
  338. for type_check, update in UPDATE_RULES:
  339. if type_check(a, b):
  340. update(a, b)
  341. return True
  342. return False
  343. class StrongRef:
  344. def __init__(self, obj):
  345. self.obj = obj
  346. def __call__(self):
  347. return self.obj
  348. mod_attrs = [
  349. "__name__",
  350. "__doc__",
  351. "__package__",
  352. "__loader__",
  353. "__spec__",
  354. "__file__",
  355. "__cached__",
  356. "__builtins__",
  357. ]
  358. class ImportFromTracker:
  359. def __init__(self, imports_froms: dict, symbol_map: dict):
  360. self.imports_froms = imports_froms
  361. # symbol_map maps original_name -> list of resolved_names
  362. self.symbol_map = {}
  363. if symbol_map:
  364. for module_name, mappings in symbol_map.items():
  365. self.symbol_map[module_name] = {}
  366. for original_name, resolved_names in mappings.items():
  367. if isinstance(resolved_names, list):
  368. self.symbol_map[module_name][original_name] = resolved_names[:]
  369. else:
  370. self.symbol_map[module_name][original_name] = [resolved_names]
  371. else:
  372. self.symbol_map = symbol_map or {}
  373. def add_import(
  374. self, module_name: str, original_name: str, resolved_name: str
  375. ) -> None:
  376. """Add an import, handling conflicts with existing imports.
  377. This method is called after successful code execution, so we know the import is valid.
  378. """
  379. if module_name not in self.imports_froms:
  380. self.imports_froms[module_name] = []
  381. if module_name not in self.symbol_map:
  382. self.symbol_map[module_name] = {}
  383. # Check if there's already a different mapping for the same resolved_name from a different original_name
  384. # We need to remove any conflicting mappings
  385. for orig_name, res_names in list(self.symbol_map[module_name].items()):
  386. if resolved_name in res_names and orig_name != original_name:
  387. # Remove the conflicting resolved_name from the other original_name's list
  388. res_names.remove(resolved_name)
  389. if (
  390. not res_names
  391. ): # If the list is now empty, remove the original_name entirely
  392. if orig_name in self.imports_froms[module_name]:
  393. self.imports_froms[module_name].remove(orig_name)
  394. del self.symbol_map[module_name][orig_name]
  395. # Add the new mapping
  396. if original_name not in self.imports_froms[module_name]:
  397. self.imports_froms[module_name].append(original_name)
  398. if original_name not in self.symbol_map[module_name]:
  399. self.symbol_map[module_name][original_name] = []
  400. # Add the resolved_name if it's not already in the list
  401. if resolved_name not in self.symbol_map[module_name][original_name]:
  402. self.symbol_map[module_name][original_name].append(resolved_name)
  403. def append_obj(module, d, name, obj, autoload=False):
  404. in_module = hasattr(obj, "__module__") and obj.__module__ == module.__name__
  405. if autoload:
  406. # check needed for module global built-ins
  407. if not in_module and name in mod_attrs:
  408. return False
  409. else:
  410. if not in_module:
  411. return False
  412. key = (module.__name__, name)
  413. try:
  414. d.setdefault(key, []).append(weakref.ref(obj))
  415. except TypeError:
  416. pass
  417. return True
  418. def superreload(
  419. module, reload=reload, old_objects=None, shell=None, import_from_tracker=None
  420. ):
  421. """Enhanced version of the builtin reload function.
  422. superreload remembers objects previously in the module, and
  423. - upgrades the class dictionary of every old class in the module
  424. - upgrades the code object of every old function and method
  425. - clears the module's namespace before reloading
  426. """
  427. if old_objects is None:
  428. old_objects = {}
  429. # collect old objects in the module
  430. for name, obj in list(module.__dict__.items()):
  431. if not append_obj(module, old_objects, name, obj):
  432. continue
  433. key = (module.__name__, name)
  434. try:
  435. old_objects.setdefault(key, []).append(weakref.ref(obj))
  436. except TypeError:
  437. pass
  438. # reload module
  439. try:
  440. # clear namespace first from old cruft
  441. old_dict = module.__dict__.copy()
  442. old_name = module.__name__
  443. module.__dict__.clear()
  444. module.__dict__["__name__"] = old_name
  445. module.__dict__["__loader__"] = old_dict["__loader__"]
  446. except (TypeError, AttributeError, KeyError):
  447. pass
  448. try:
  449. module = reload(module)
  450. except:
  451. # restore module dictionary on failed reload
  452. module.__dict__.update(old_dict)
  453. raise
  454. for name, new_obj in list(module.__dict__.items()):
  455. key = (module.__name__, name)
  456. if key not in old_objects:
  457. # here 'shell' acts both as a flag and as an output var
  458. imports_froms = (
  459. import_from_tracker.imports_froms if import_from_tracker else None
  460. )
  461. symbol_map = import_from_tracker.symbol_map if import_from_tracker else None
  462. if (
  463. shell is None
  464. or name == "Enum"
  465. or not append_obj(module, old_objects, name, new_obj, True)
  466. or (
  467. imports_froms
  468. and module.__name__ in imports_froms
  469. and "*" not in imports_froms[module.__name__]
  470. and name not in imports_froms[module.__name__]
  471. )
  472. ):
  473. continue
  474. # Handle symbol mapping - now supporting multiple resolved names per original name
  475. if symbol_map and name in symbol_map.get(module.__name__, {}):
  476. resolved_names = symbol_map.get(module.__name__, {})[name]
  477. for resolved_name in resolved_names:
  478. shell.user_ns[resolved_name] = new_obj
  479. else:
  480. shell.user_ns[name] = new_obj
  481. new_refs = []
  482. for old_ref in old_objects[key]:
  483. old_obj = old_ref()
  484. if old_obj is None:
  485. continue
  486. new_refs.append(old_ref)
  487. update_generic(old_obj, new_obj)
  488. if new_refs:
  489. old_objects[key] = new_refs
  490. else:
  491. del old_objects[key]
  492. return module
  493. # ------------------------------------------------------------------------------
  494. # IPython connectivity
  495. # ------------------------------------------------------------------------------
  496. @magics_class
  497. class AutoreloadMagics(Magics):
  498. def __init__(self, *a, **kw):
  499. super().__init__(*a, **kw)
  500. self._reloader = ModuleReloader(self.shell)
  501. self._reloader.check_all = False
  502. self._reloader.autoload_obj = False
  503. self.loaded_modules = set(sys.modules)
  504. @line_magic
  505. @magic_arguments.magic_arguments()
  506. @magic_arguments.argument(
  507. "mode",
  508. type=str,
  509. default="now",
  510. nargs="?",
  511. help="""blank or 'now' - Reload all modules (except those excluded by %%aimport)
  512. automatically now.
  513. '0' or 'off' - Disable automatic reloading.
  514. '1' or 'explicit' - Reload only modules imported with %%aimport every
  515. time before executing the Python code typed.
  516. '2' or 'all' - Reload all modules (except those excluded by %%aimport)
  517. every time before executing the Python code typed.
  518. '3' or 'complete' - Same as 2/all, but also adds any new
  519. objects in the module.
  520. By default, a newer autoreload algorithm that diffs the module's source code
  521. with the previous version and only reloads changed parts is applied for modes
  522. 2 and below. To use the original algorithm, add the `-` suffix to the mode,
  523. e.g. '%autoreload 2-', or pass in --full.
  524. """,
  525. )
  526. @magic_arguments.argument(
  527. "-p",
  528. "--print",
  529. action="store_true",
  530. default=False,
  531. help="Show autoreload activity using `print` statements",
  532. )
  533. @magic_arguments.argument(
  534. "-l",
  535. "--log",
  536. action="store_true",
  537. default=False,
  538. help="Show autoreload activity using the logger",
  539. )
  540. @magic_arguments.argument(
  541. "--hide-errors",
  542. action="store_true",
  543. default=False,
  544. help="Hide autoreload errors",
  545. )
  546. @magic_arguments.argument(
  547. "--full",
  548. action="store_true",
  549. default=False,
  550. help="Don't ever use new diffing algorithm",
  551. )
  552. def autoreload(self, line=""):
  553. r"""%autoreload => Reload modules automatically
  554. %autoreload or %autoreload now
  555. Reload all modules (except those excluded by %aimport) automatically
  556. now.
  557. %autoreload 0 or %autoreload off
  558. Disable automatic reloading.
  559. %autoreload 1 or %autoreload explicit
  560. Reload only modules imported with %aimport every time before executing
  561. the Python code typed.
  562. %autoreload 2 or %autoreload all
  563. Reload all modules (except those excluded by %aimport) every time
  564. before executing the Python code typed.
  565. %autoreload 3 or %autoreload complete
  566. Same as 2/all, but also but also adds any new objects in the module. See
  567. unit test at IPython/extensions/tests/test_autoreload.py::test_autoload_newly_added_objects
  568. The optional arguments --print and --log control display of autoreload activity. The default
  569. is to act silently; --print (or -p) will print out the names of modules that are being
  570. reloaded, and --log (or -l) outputs them to the log at INFO level.
  571. The optional argument --hide-errors hides any errors that can happen when trying to
  572. reload code.
  573. Reloading Python modules in a reliable way is in general
  574. difficult, and unexpected things may occur. %autoreload tries to
  575. work around common pitfalls by replacing function code objects and
  576. parts of classes previously in the module with new versions. This
  577. makes the following things to work:
  578. - Functions and classes imported via 'from xxx import foo' are upgraded
  579. to new versions when 'xxx' is reloaded.
  580. - Methods and properties of classes are upgraded on reload, so that
  581. calling 'c.foo()' on an object 'c' created before the reload causes
  582. the new code for 'foo' to be executed.
  583. Some of the known remaining caveats are:
  584. - Replacing code objects does not always succeed: changing a @property
  585. in a class to an ordinary method or a method to a member variable
  586. can cause problems (but in old objects only).
  587. - Functions that are removed (eg. via monkey-patching) from a module
  588. before it is reloaded are not upgraded.
  589. - C extension modules cannot be reloaded, and so cannot be
  590. autoreloaded.
  591. """
  592. args = magic_arguments.parse_argstring(self.autoreload, line)
  593. mode = args.mode.lower()
  594. enable_deduperreload = not args.full
  595. if mode.endswith("-"):
  596. enable_deduperreload = False
  597. mode = mode[:-1]
  598. self._reloader.deduper_reloader.enabled = enable_deduperreload
  599. p = print
  600. logger = logging.getLogger("autoreload")
  601. l = logger.info
  602. def pl(msg):
  603. p(msg)
  604. l(msg)
  605. if args.print is False and args.log is False:
  606. self._reloader._report = lambda msg: None
  607. elif args.print is True:
  608. if args.log is True:
  609. self._reloader._report = pl
  610. else:
  611. self._reloader._report = p
  612. elif args.log is True:
  613. self._reloader._report = l
  614. self._reloader.hide_errors = args.hide_errors
  615. if mode == "" or mode == "now":
  616. self._reloader.check(True)
  617. elif mode == "0" or mode == "off":
  618. self._reloader.enabled = False
  619. elif mode == "1" or mode == "explicit":
  620. self._reloader.enabled = True
  621. self._reloader.check_all = False
  622. self._reloader.autoload_obj = False
  623. elif mode == "2" or mode == "all":
  624. self._reloader.enabled = True
  625. self._reloader.check_all = True
  626. self._reloader.autoload_obj = False
  627. elif mode == "3" or mode == "complete":
  628. self._reloader.enabled = True
  629. self._reloader.check_all = True
  630. self._reloader.autoload_obj = True
  631. else:
  632. raise ValueError(f'Unrecognized autoreload mode "{mode}".')
  633. @line_magic
  634. def aimport(self, parameter_s="", stream=None):
  635. """%aimport => Import modules for automatic reloading.
  636. %aimport
  637. List modules to automatically import and not to import.
  638. %aimport foo
  639. Import module 'foo' and mark it to be autoreloaded for %autoreload explicit
  640. %aimport foo, bar
  641. Import modules 'foo', 'bar' and mark them to be autoreloaded for %autoreload explicit
  642. %aimport -foo, bar
  643. Mark module 'foo' to not be autoreloaded for %autoreload explicit, all, or complete, and 'bar'
  644. to be autoreloaded for mode explicit.
  645. """
  646. modname = parameter_s
  647. if not modname:
  648. to_reload = sorted(self._reloader.modules.keys())
  649. to_skip = sorted(self._reloader.skip_modules.keys())
  650. if stream is None:
  651. stream = sys.stdout
  652. if self._reloader.check_all:
  653. stream.write("Modules to reload:\nall-except-skipped\n")
  654. else:
  655. stream.write("Modules to reload:\n%s\n" % " ".join(to_reload))
  656. stream.write("\nModules to skip:\n%s\n" % " ".join(to_skip))
  657. else:
  658. for _module in [_.strip() for _ in modname.split(",")]:
  659. if _module.startswith("-"):
  660. _module = _module[1:].strip()
  661. self._reloader.mark_module_skipped(_module)
  662. else:
  663. top_module, top_name = self._reloader.aimport_module(_module)
  664. # Inject module to user namespace
  665. self.shell.push({top_name: top_module})
  666. def pre_run_cell(self, info):
  667. # Store the execution info for later use in post_execute_hook
  668. self._last_execution_info = info
  669. if self._reloader.enabled:
  670. try:
  671. self._reloader.check()
  672. except:
  673. pass
  674. def post_execute_hook(self):
  675. """Cache the modification times of any modules imported in this execution and track imports"""
  676. # Track imports from the recently executed code if autoreload 3 is enabled
  677. if self._reloader.enabled and self._reloader.autoload_obj:
  678. # Use the stored execution info
  679. if (
  680. hasattr(self, "_last_execution_info")
  681. and self._last_execution_info
  682. and self._last_execution_info.transformed_cell
  683. ):
  684. self._track_imports_from_code(
  685. self._last_execution_info.transformed_cell
  686. )
  687. newly_loaded_modules = set(sys.modules) - self.loaded_modules
  688. for modname in newly_loaded_modules:
  689. _, pymtime = self._reloader.filename_and_mtime(sys.modules[modname])
  690. if pymtime is not None:
  691. self._reloader.modules_mtimes[modname] = pymtime
  692. self.loaded_modules.update(newly_loaded_modules)
  693. def _track_imports_from_code(self, code: str) -> None:
  694. """Track import statements from executed code"""
  695. try:
  696. tree = ast.parse(code)
  697. for node in ast.walk(tree):
  698. # Handle "from X import Y" style imports
  699. if isinstance(node, ast.ImportFrom):
  700. mod = node.module
  701. # Skip relative imports that don't have a module name
  702. if mod is None:
  703. continue
  704. for name in node.names:
  705. # name.name is going to be actual name that we want to import from module
  706. # name.asname is Z in the case of from X import Y as Z
  707. # we should update Z in the shell in this situation, so track it too.
  708. original_name = name.name
  709. resolved_name = name.asname if name.asname else name.name
  710. # Since the code executed successfully, we know this import is valid
  711. self._reloader.import_from_tracker.add_import(
  712. mod, original_name, resolved_name
  713. )
  714. except (SyntaxError, ValueError):
  715. # If there's a syntax error, skip import tracking
  716. # (though this shouldn't happen since the code already executed successfully)
  717. pass
  718. def load_ipython_extension(ip):
  719. """Load the extension in IPython."""
  720. auto_reload = AutoreloadMagics(ip)
  721. ip.register_magics(auto_reload)
  722. ip.events.register("pre_run_cell", auto_reload.pre_run_cell)
  723. ip.events.register("post_execute", auto_reload.post_execute_hook)