restarter.py 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103
  1. """A basic in process kernel monitor with autorestarting.
  2. This watches a kernel's state using KernelManager.is_alive and auto
  3. restarts the kernel if it dies.
  4. """
  5. # Copyright (c) Jupyter Development Team.
  6. # Distributed under the terms of the Modified BSD License.
  7. import time
  8. import warnings
  9. from typing import Any
  10. from traitlets import Instance
  11. from ..restarter import KernelRestarter
  12. class IOLoopKernelRestarter(KernelRestarter):
  13. """Monitor and autorestart a kernel."""
  14. loop = Instance("tornado.ioloop.IOLoop")
  15. def _loop_default(self) -> Any:
  16. warnings.warn(
  17. "IOLoopKernelRestarter.loop is deprecated in jupyter-client 5.2",
  18. DeprecationWarning,
  19. stacklevel=4,
  20. )
  21. from tornado import ioloop
  22. return ioloop.IOLoop.current()
  23. _pcallback = None
  24. def start(self) -> None:
  25. """Start the polling of the kernel."""
  26. if self._pcallback is None:
  27. from tornado.ioloop import PeriodicCallback
  28. self._pcallback = PeriodicCallback(
  29. self.poll,
  30. 1000 * self.time_to_dead,
  31. )
  32. self._pcallback.start()
  33. def stop(self) -> None:
  34. """Stop the kernel polling."""
  35. if self._pcallback is not None:
  36. self._pcallback.stop()
  37. self._pcallback = None
  38. class AsyncIOLoopKernelRestarter(IOLoopKernelRestarter):
  39. """An async io loop kernel restarter."""
  40. async def poll(self) -> None: # type:ignore[override]
  41. """Poll the kernel."""
  42. if self.debug:
  43. self.log.debug("Polling kernel...")
  44. is_alive = await self.kernel_manager.is_alive()
  45. now = time.time()
  46. if not is_alive:
  47. self._last_dead = now
  48. if self._restarting:
  49. self._restart_count += 1
  50. else:
  51. self._restart_count = 1
  52. if self._restart_count > self.restart_limit:
  53. self.log.warning("AsyncIOLoopKernelRestarter: restart failed")
  54. self._fire_callbacks("dead")
  55. self._restarting = False
  56. self._restart_count = 0
  57. self.stop()
  58. else:
  59. newports = self.random_ports_until_alive and self._initial_startup
  60. self.log.info(
  61. "AsyncIOLoopKernelRestarter: restarting kernel (%i/%i), %s random ports",
  62. self._restart_count,
  63. self.restart_limit,
  64. "new" if newports else "keep",
  65. )
  66. self._fire_callbacks("restart")
  67. await self.kernel_manager.restart_kernel(now=True, newports=newports)
  68. self._restarting = True
  69. else:
  70. # Since `is_alive` only tests that the kernel process is alive, it does not
  71. # indicate that the kernel has successfully completed startup. To solve this
  72. # correctly, we would need to wait for a kernel info reply, but it is not
  73. # necessarily appropriate to start a kernel client + channels in the
  74. # restarter. Therefore, we use "has been alive continuously for X time" as a
  75. # heuristic for a stable start up.
  76. # See https://github.com/jupyter/jupyter_client/pull/717 for details.
  77. stable_start_time = self.stable_start_time
  78. if self.kernel_manager.provisioner:
  79. stable_start_time = self.kernel_manager.provisioner.get_stable_start_time(
  80. recommended=stable_start_time
  81. )
  82. if self._initial_startup and now - self._last_dead >= stable_start_time:
  83. self._initial_startup = False
  84. if self._restarting and now - self._last_dead >= stable_start_time:
  85. self.log.debug("AsyncIOLoopKernelRestarter: restart apparently succeeded")
  86. self._restarting = False