editable_wheel.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914
  1. """
  2. Create a wheel that, when installed, will make the source package 'editable'
  3. (add it to the interpreter's path, including metadata) per PEP 660. Replaces
  4. 'setup.py develop'.
  5. .. note::
  6. One of the mechanisms briefly mentioned in PEP 660 to implement editable installs is
  7. to create a separated directory inside ``build`` and use a .pth file to point to that
  8. directory. In the context of this file such directory is referred as
  9. *auxiliary build directory* or ``auxiliary_dir``.
  10. """
  11. from __future__ import annotations
  12. import io
  13. import logging
  14. import operator
  15. import os
  16. import shutil
  17. import traceback
  18. from collections.abc import Iterable, Iterator, Mapping
  19. from contextlib import suppress
  20. from enum import Enum
  21. from inspect import cleandoc
  22. from itertools import chain, starmap
  23. from pathlib import Path
  24. from tempfile import TemporaryDirectory
  25. from types import TracebackType
  26. from typing import TYPE_CHECKING, Protocol, TypeVar, cast
  27. from .. import Command, _normalization, _path, _shutil, errors, namespaces
  28. from .._path import StrPath
  29. from ..compat import py310, py312
  30. from ..discovery import find_package_path
  31. from ..dist import Distribution
  32. from ..warnings import InformationOnly, SetuptoolsDeprecationWarning
  33. from .build import build as build_cls
  34. from .build_py import build_py as build_py_cls
  35. from .dist_info import dist_info as dist_info_cls
  36. from .egg_info import egg_info as egg_info_cls
  37. from .install import install as install_cls
  38. from .install_scripts import install_scripts as install_scripts_cls
  39. if TYPE_CHECKING:
  40. from typing_extensions import Self
  41. from .._vendor.wheel.wheelfile import WheelFile
  42. _P = TypeVar("_P", bound=StrPath)
  43. _logger = logging.getLogger(__name__)
  44. class _EditableMode(Enum):
  45. """
  46. Possible editable installation modes:
  47. `lenient` (new files automatically added to the package - DEFAULT);
  48. `strict` (requires a new installation when files are added/removed); or
  49. `compat` (attempts to emulate `python setup.py develop` - DEPRECATED).
  50. """
  51. STRICT = "strict"
  52. LENIENT = "lenient"
  53. COMPAT = "compat" # TODO: Remove `compat` after Dec/2022.
  54. @classmethod
  55. def convert(cls, mode: str | None) -> _EditableMode:
  56. if not mode:
  57. return _EditableMode.LENIENT # default
  58. _mode = mode.upper()
  59. if _mode not in _EditableMode.__members__:
  60. raise errors.OptionError(f"Invalid editable mode: {mode!r}. Try: 'strict'.")
  61. if _mode == "COMPAT":
  62. SetuptoolsDeprecationWarning.emit(
  63. "Compat editable installs",
  64. """
  65. The 'compat' editable mode is transitional and will be removed
  66. in future versions of `setuptools`.
  67. Please adapt your code accordingly to use either the 'strict' or the
  68. 'lenient' modes.
  69. """,
  70. see_docs="userguide/development_mode.html",
  71. # TODO: define due_date
  72. # There is a series of shortcomings with the available editable install
  73. # methods, and they are very controversial. This is something that still
  74. # needs work.
  75. # Moreover, `pip` is still hiding this warning, so users are not aware.
  76. )
  77. return _EditableMode[_mode]
  78. _STRICT_WARNING = """
  79. New or renamed files may not be automatically picked up without a new installation.
  80. """
  81. _LENIENT_WARNING = """
  82. Options like `package-data`, `include/exclude-package-data` or
  83. `packages.find.exclude/include` may have no effect.
  84. """
  85. class editable_wheel(Command):
  86. """Build 'editable' wheel for development.
  87. This command is private and reserved for internal use of setuptools,
  88. users should rely on ``setuptools.build_meta`` APIs.
  89. """
  90. description = "DO NOT CALL DIRECTLY, INTERNAL ONLY: create PEP 660 editable wheel"
  91. user_options = [
  92. ("dist-dir=", "d", "directory to put final built distributions in"),
  93. ("dist-info-dir=", "I", "path to a pre-build .dist-info directory"),
  94. ("mode=", None, cleandoc(_EditableMode.__doc__ or "")),
  95. ]
  96. def initialize_options(self):
  97. self.dist_dir = None
  98. self.dist_info_dir = None
  99. self.project_dir = None
  100. self.mode = None
  101. def finalize_options(self) -> None:
  102. dist = self.distribution
  103. self.project_dir = dist.src_root or os.curdir
  104. self.package_dir = dist.package_dir or {}
  105. self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist"))
  106. def run(self) -> None:
  107. try:
  108. self.dist_dir.mkdir(exist_ok=True)
  109. self._ensure_dist_info()
  110. # Add missing dist_info files
  111. self.reinitialize_command("bdist_wheel")
  112. bdist_wheel = self.get_finalized_command("bdist_wheel")
  113. bdist_wheel.write_wheelfile(self.dist_info_dir)
  114. self._create_wheel_file(bdist_wheel)
  115. except Exception as ex:
  116. project = self.distribution.name or self.distribution.get_name()
  117. py310.add_note(
  118. ex,
  119. f"An error occurred when building editable wheel for {project}.\n"
  120. "See debugging tips in: "
  121. "https://setuptools.pypa.io/en/latest/userguide/development_mode.html#debugging-tips",
  122. )
  123. raise
  124. def _ensure_dist_info(self):
  125. if self.dist_info_dir is None:
  126. dist_info = cast(dist_info_cls, self.reinitialize_command("dist_info"))
  127. dist_info.output_dir = self.dist_dir
  128. dist_info.ensure_finalized()
  129. dist_info.run()
  130. self.dist_info_dir = dist_info.dist_info_dir
  131. else:
  132. assert str(self.dist_info_dir).endswith(".dist-info")
  133. assert Path(self.dist_info_dir, "METADATA").exists()
  134. def _install_namespaces(self, installation_dir, pth_prefix):
  135. # XXX: Only required to support the deprecated namespace practice
  136. dist = self.distribution
  137. if not dist.namespace_packages:
  138. return
  139. src_root = Path(self.project_dir, self.package_dir.get("", ".")).resolve()
  140. installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, src_root)
  141. installer.install_namespaces()
  142. def _find_egg_info_dir(self) -> str | None:
  143. parent_dir = Path(self.dist_info_dir).parent if self.dist_info_dir else Path()
  144. candidates = map(str, parent_dir.glob("*.egg-info"))
  145. return next(candidates, None)
  146. def _configure_build(
  147. self, name: str, unpacked_wheel: StrPath, build_lib: StrPath, tmp_dir: StrPath
  148. ):
  149. """Configure commands to behave in the following ways:
  150. - Build commands can write to ``build_lib`` if they really want to...
  151. (but this folder is expected to be ignored and modules are expected to live
  152. in the project directory...)
  153. - Binary extensions should be built in-place (editable_mode = True)
  154. - Data/header/script files are not part of the "editable" specification
  155. so they are written directly to the unpacked_wheel directory.
  156. """
  157. # Non-editable files (data, headers, scripts) are written directly to the
  158. # unpacked_wheel
  159. dist = self.distribution
  160. wheel = str(unpacked_wheel)
  161. build_lib = str(build_lib)
  162. data = str(Path(unpacked_wheel, f"{name}.data", "data"))
  163. headers = str(Path(unpacked_wheel, f"{name}.data", "headers"))
  164. scripts = str(Path(unpacked_wheel, f"{name}.data", "scripts"))
  165. # egg-info may be generated again to create a manifest (used for package data)
  166. egg_info = cast(
  167. egg_info_cls, dist.reinitialize_command("egg_info", reinit_subcommands=True)
  168. )
  169. egg_info.egg_base = str(tmp_dir)
  170. egg_info.ignore_egg_info_in_manifest = True
  171. build = cast(
  172. build_cls, dist.reinitialize_command("build", reinit_subcommands=True)
  173. )
  174. install = cast(
  175. install_cls, dist.reinitialize_command("install", reinit_subcommands=True)
  176. )
  177. build.build_platlib = build.build_purelib = build.build_lib = build_lib
  178. install.install_purelib = install.install_platlib = install.install_lib = wheel
  179. install.install_scripts = build.build_scripts = scripts
  180. install.install_headers = headers
  181. install.install_data = data
  182. # For portability, ensure scripts are built with #!python shebang
  183. # pypa/setuptools#4863
  184. build_scripts = dist.get_command_obj("build_scripts")
  185. build_scripts.executable = 'python'
  186. install_scripts = cast(
  187. install_scripts_cls, dist.get_command_obj("install_scripts")
  188. )
  189. install_scripts.no_ep = True
  190. build.build_temp = str(tmp_dir)
  191. build_py = cast(build_py_cls, dist.get_command_obj("build_py"))
  192. build_py.compile = False
  193. build_py.existing_egg_info_dir = self._find_egg_info_dir()
  194. self._set_editable_mode()
  195. build.ensure_finalized()
  196. install.ensure_finalized()
  197. def _set_editable_mode(self):
  198. """Set the ``editable_mode`` flag in the build sub-commands"""
  199. dist = self.distribution
  200. build = dist.get_command_obj("build")
  201. for cmd_name in build.get_sub_commands():
  202. cmd = dist.get_command_obj(cmd_name)
  203. if hasattr(cmd, "editable_mode"):
  204. cmd.editable_mode = True
  205. elif hasattr(cmd, "inplace"):
  206. cmd.inplace = True # backward compatibility with distutils
  207. def _collect_build_outputs(self) -> tuple[list[str], dict[str, str]]:
  208. files: list[str] = []
  209. mapping: dict[str, str] = {}
  210. build = self.get_finalized_command("build")
  211. for cmd_name in build.get_sub_commands():
  212. cmd = self.get_finalized_command(cmd_name)
  213. if hasattr(cmd, "get_outputs"):
  214. files.extend(cmd.get_outputs() or [])
  215. if hasattr(cmd, "get_output_mapping"):
  216. mapping.update(cmd.get_output_mapping() or {})
  217. return files, mapping
  218. def _run_build_commands(
  219. self,
  220. dist_name: str,
  221. unpacked_wheel: StrPath,
  222. build_lib: StrPath,
  223. tmp_dir: StrPath,
  224. ) -> tuple[list[str], dict[str, str]]:
  225. self._configure_build(dist_name, unpacked_wheel, build_lib, tmp_dir)
  226. self._run_build_subcommands()
  227. files, mapping = self._collect_build_outputs()
  228. self._run_install("headers")
  229. self._run_install("scripts")
  230. self._run_install("data")
  231. return files, mapping
  232. def _run_build_subcommands(self) -> None:
  233. """
  234. Issue #3501 indicates that some plugins/customizations might rely on:
  235. 1. ``build_py`` not running
  236. 2. ``build_py`` always copying files to ``build_lib``
  237. However both these assumptions may be false in editable_wheel.
  238. This method implements a temporary workaround to support the ecosystem
  239. while the implementations catch up.
  240. """
  241. # TODO: Once plugins/customizations had the chance to catch up, replace
  242. # `self._run_build_subcommands()` with `self.run_command("build")`.
  243. # Also remove _safely_run, TestCustomBuildPy. Suggested date: Aug/2023.
  244. build = self.get_finalized_command("build")
  245. for name in build.get_sub_commands():
  246. cmd = self.get_finalized_command(name)
  247. if name == "build_py" and type(cmd) is not build_py_cls:
  248. self._safely_run(name)
  249. else:
  250. self.run_command(name)
  251. def _safely_run(self, cmd_name: str):
  252. try:
  253. return self.run_command(cmd_name)
  254. except Exception:
  255. SetuptoolsDeprecationWarning.emit(
  256. "Customization incompatible with editable install",
  257. f"""
  258. {traceback.format_exc()}
  259. If you are seeing this warning it is very likely that a setuptools
  260. plugin or customization overrides the `{cmd_name}` command, without
  261. taking into consideration how editable installs run build steps
  262. starting from setuptools v64.0.0.
  263. Plugin authors and developers relying on custom build steps are
  264. encouraged to update their `{cmd_name}` implementation considering the
  265. information about editable installs in
  266. https://setuptools.pypa.io/en/latest/userguide/extension.html.
  267. For the time being `setuptools` will silence this error and ignore
  268. the faulty command, but this behavior will change in future versions.
  269. """,
  270. # TODO: define due_date
  271. # There is a series of shortcomings with the available editable install
  272. # methods, and they are very controversial. This is something that still
  273. # needs work.
  274. )
  275. def _create_wheel_file(self, bdist_wheel):
  276. from wheel.wheelfile import WheelFile
  277. dist_info = self.get_finalized_command("dist_info")
  278. dist_name = dist_info.name
  279. tag = "-".join(bdist_wheel.get_tag())
  280. build_tag = "0.editable" # According to PEP 427 needs to start with digit
  281. archive_name = f"{dist_name}-{build_tag}-{tag}.whl"
  282. wheel_path = Path(self.dist_dir, archive_name)
  283. if wheel_path.exists():
  284. wheel_path.unlink()
  285. unpacked_wheel = TemporaryDirectory(suffix=archive_name)
  286. build_lib = TemporaryDirectory(suffix=".build-lib")
  287. build_tmp = TemporaryDirectory(suffix=".build-temp")
  288. with unpacked_wheel as unpacked, build_lib as lib, build_tmp as tmp:
  289. unpacked_dist_info = Path(unpacked, Path(self.dist_info_dir).name)
  290. shutil.copytree(self.dist_info_dir, unpacked_dist_info)
  291. self._install_namespaces(unpacked, dist_name)
  292. files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp)
  293. strategy = self._select_strategy(dist_name, tag, lib)
  294. with strategy, WheelFile(wheel_path, "w") as wheel_obj:
  295. strategy(wheel_obj, files, mapping)
  296. wheel_obj.write_files(unpacked)
  297. return wheel_path
  298. def _run_install(self, category: str):
  299. has_category = getattr(self.distribution, f"has_{category}", None)
  300. if has_category and has_category():
  301. _logger.info(f"Installing {category} as non editable")
  302. self.run_command(f"install_{category}")
  303. def _select_strategy(
  304. self,
  305. name: str,
  306. tag: str,
  307. build_lib: StrPath,
  308. ) -> EditableStrategy:
  309. """Decides which strategy to use to implement an editable installation."""
  310. build_name = f"__editable__.{name}-{tag}"
  311. project_dir = Path(self.project_dir)
  312. mode = _EditableMode.convert(self.mode)
  313. if mode is _EditableMode.STRICT:
  314. auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name))
  315. return _LinkTree(self.distribution, name, auxiliary_dir, build_lib)
  316. packages = _find_packages(self.distribution)
  317. has_simple_layout = _simple_layout(packages, self.package_dir, project_dir)
  318. is_compat_mode = mode is _EditableMode.COMPAT
  319. if set(self.package_dir) == {""} and has_simple_layout or is_compat_mode:
  320. # src-layout(ish) is relatively safe for a simple pth file
  321. src_dir = self.package_dir.get("", ".")
  322. return _StaticPth(self.distribution, name, [Path(project_dir, src_dir)])
  323. # Use a MetaPathFinder to avoid adding accidental top-level packages/modules
  324. return _TopLevelFinder(self.distribution, name)
  325. class EditableStrategy(Protocol):
  326. def __call__(
  327. self, wheel: WheelFile, files: list[str], mapping: Mapping[str, str]
  328. ) -> object: ...
  329. def __enter__(self) -> Self: ...
  330. def __exit__(
  331. self,
  332. _exc_type: type[BaseException] | None,
  333. _exc_value: BaseException | None,
  334. _traceback: TracebackType | None,
  335. ) -> object: ...
  336. class _StaticPth:
  337. def __init__(self, dist: Distribution, name: str, path_entries: list[Path]) -> None:
  338. self.dist = dist
  339. self.name = name
  340. self.path_entries = path_entries
  341. def __call__(
  342. self, wheel: WheelFile, files: list[str], mapping: Mapping[str, str]
  343. ) -> None:
  344. entries = "\n".join(str(p.resolve()) for p in self.path_entries)
  345. contents = _encode_pth(f"{entries}\n")
  346. wheel.writestr(f"__editable__.{self.name}.pth", contents)
  347. def __enter__(self) -> Self:
  348. msg = f"""
  349. Editable install will be performed using .pth file to extend `sys.path` with:
  350. {list(map(os.fspath, self.path_entries))!r}
  351. """
  352. _logger.warning(msg + _LENIENT_WARNING)
  353. return self
  354. def __exit__(
  355. self,
  356. _exc_type: object,
  357. _exc_value: object,
  358. _traceback: object,
  359. ) -> None:
  360. pass
  361. class _LinkTree(_StaticPth):
  362. """
  363. Creates a ``.pth`` file that points to a link tree in the ``auxiliary_dir``.
  364. This strategy will only link files (not dirs), so it can be implemented in
  365. any OS, even if that means using hardlinks instead of symlinks.
  366. By collocating ``auxiliary_dir`` and the original source code, limitations
  367. with hardlinks should be avoided.
  368. """
  369. def __init__(
  370. self,
  371. dist: Distribution,
  372. name: str,
  373. auxiliary_dir: StrPath,
  374. build_lib: StrPath,
  375. ) -> None:
  376. self.auxiliary_dir = Path(auxiliary_dir)
  377. self.build_lib = Path(build_lib).resolve()
  378. self._file = dist.get_command_obj("build_py").copy_file
  379. super().__init__(dist, name, [self.auxiliary_dir])
  380. def __call__(
  381. self, wheel: WheelFile, files: list[str], mapping: Mapping[str, str]
  382. ) -> None:
  383. self._create_links(files, mapping)
  384. super().__call__(wheel, files, mapping)
  385. def _normalize_output(self, file: str) -> str | None:
  386. # Files relative to build_lib will be normalized to None
  387. with suppress(ValueError):
  388. path = Path(file).resolve().relative_to(self.build_lib)
  389. return str(path).replace(os.sep, '/')
  390. return None
  391. def _create_file(self, relative_output: str, src_file: str, link=None):
  392. dest = self.auxiliary_dir / relative_output
  393. if not dest.parent.is_dir():
  394. dest.parent.mkdir(parents=True)
  395. self._file(src_file, dest, link=link)
  396. def _create_links(self, outputs, output_mapping: Mapping[str, str]):
  397. self.auxiliary_dir.mkdir(parents=True, exist_ok=True)
  398. link_type = "sym" if _can_symlink_files(self.auxiliary_dir) else "hard"
  399. normalised = ((self._normalize_output(k), v) for k, v in output_mapping.items())
  400. # remove files that are not relative to build_lib
  401. mappings = {k: v for k, v in normalised if k is not None}
  402. for output in outputs:
  403. relative = self._normalize_output(output)
  404. if relative and relative not in mappings:
  405. self._create_file(relative, output)
  406. for relative, src in mappings.items():
  407. self._create_file(relative, src, link=link_type)
  408. def __enter__(self) -> Self:
  409. msg = "Strict editable install will be performed using a link tree.\n"
  410. _logger.warning(msg + _STRICT_WARNING)
  411. return self
  412. def __exit__(
  413. self,
  414. _exc_type: object,
  415. _exc_value: object,
  416. _traceback: object,
  417. ) -> None:
  418. msg = f"""\n
  419. Strict editable installation performed using the auxiliary directory:
  420. {self.auxiliary_dir}
  421. Please be careful to not remove this directory, otherwise you might not be able
  422. to import/use your package.
  423. """
  424. InformationOnly.emit("Editable installation.", msg)
  425. class _TopLevelFinder:
  426. def __init__(self, dist: Distribution, name: str) -> None:
  427. self.dist = dist
  428. self.name = name
  429. def template_vars(self) -> tuple[str, str, dict[str, str], dict[str, list[str]]]:
  430. src_root = self.dist.src_root or os.curdir
  431. top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist))
  432. package_dir = self.dist.package_dir or {}
  433. roots = _find_package_roots(top_level, package_dir, src_root)
  434. namespaces_ = dict(
  435. chain(
  436. _find_namespaces(self.dist.packages or [], roots),
  437. ((ns, []) for ns in _find_virtual_namespaces(roots)),
  438. )
  439. )
  440. legacy_namespaces = {
  441. pkg: find_package_path(pkg, roots, self.dist.src_root or "")
  442. for pkg in self.dist.namespace_packages or []
  443. }
  444. mapping = {**roots, **legacy_namespaces}
  445. # ^-- We need to explicitly add the legacy_namespaces to the mapping to be
  446. # able to import their modules even if another package sharing the same
  447. # namespace is installed in a conventional (non-editable) way.
  448. name = f"__editable__.{self.name}.finder"
  449. finder = _normalization.safe_identifier(name)
  450. return finder, name, mapping, namespaces_
  451. def get_implementation(self) -> Iterator[tuple[str, bytes]]:
  452. finder, name, mapping, namespaces_ = self.template_vars()
  453. content = bytes(_finder_template(name, mapping, namespaces_), "utf-8")
  454. yield (f"{finder}.py", content)
  455. content = _encode_pth(f"import {finder}; {finder}.install()")
  456. yield (f"__editable__.{self.name}.pth", content)
  457. def __call__(
  458. self, wheel: WheelFile, files: list[str], mapping: Mapping[str, str]
  459. ) -> None:
  460. for file, content in self.get_implementation():
  461. wheel.writestr(file, content)
  462. def __enter__(self) -> Self:
  463. msg = "Editable install will be performed using a meta path finder.\n"
  464. _logger.warning(msg + _LENIENT_WARNING)
  465. return self
  466. def __exit__(
  467. self,
  468. _exc_type: object,
  469. _exc_value: object,
  470. _traceback: object,
  471. ) -> None:
  472. msg = """\n
  473. Please be careful with folders in your working directory with the same
  474. name as your package as they may take precedence during imports.
  475. """
  476. InformationOnly.emit("Editable installation.", msg)
  477. def _encode_pth(content: str) -> bytes:
  478. """
  479. Prior to Python 3.13 (see https://github.com/python/cpython/issues/77102),
  480. .pth files are always read with 'locale' encoding, the recommendation
  481. from the cpython core developers is to write them as ``open(path, "w")``
  482. and ignore warnings (see python/cpython#77102, pypa/setuptools#3937).
  483. This function tries to simulate this behavior without having to create an
  484. actual file, in a way that supports a range of active Python versions.
  485. (There seems to be some variety in the way different version of Python handle
  486. ``encoding=None``, not all of them use ``locale.getpreferredencoding(False)``
  487. or ``locale.getencoding()``).
  488. """
  489. with io.BytesIO() as buffer:
  490. wrapper = io.TextIOWrapper(buffer, encoding=py312.PTH_ENCODING)
  491. # TODO: Python 3.13 replace the whole function with `bytes(content, "utf-8")`
  492. wrapper.write(content)
  493. wrapper.flush()
  494. buffer.seek(0)
  495. return buffer.read()
  496. def _can_symlink_files(base_dir: Path) -> bool:
  497. with TemporaryDirectory(dir=str(base_dir.resolve())) as tmp:
  498. path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt")
  499. path1.write_text("file1", encoding="utf-8")
  500. with suppress(AttributeError, NotImplementedError, OSError):
  501. os.symlink(path1, path2)
  502. if path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1":
  503. return True
  504. try:
  505. os.link(path1, path2) # Ensure hard links can be created
  506. except Exception as ex:
  507. msg = (
  508. "File system does not seem to support either symlinks or hard links. "
  509. "Strict editable installs require one of them to be supported."
  510. )
  511. raise LinksNotSupported(msg) from ex
  512. return False
  513. def _simple_layout(
  514. packages: Iterable[str], package_dir: dict[str, str], project_dir: StrPath
  515. ) -> bool:
  516. """Return ``True`` if:
  517. - all packages are contained by the same parent directory, **and**
  518. - all packages become importable if the parent directory is added to ``sys.path``.
  519. >>> _simple_layout(['a'], {"": "src"}, "/tmp/myproj")
  520. True
  521. >>> _simple_layout(['a', 'a.b'], {"": "src"}, "/tmp/myproj")
  522. True
  523. >>> _simple_layout(['a', 'a.b'], {}, "/tmp/myproj")
  524. True
  525. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"": "src"}, "/tmp/myproj")
  526. True
  527. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "a", "b": "b"}, ".")
  528. True
  529. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a", "b": "_b"}, ".")
  530. False
  531. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a"}, "/tmp/myproj")
  532. False
  533. >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a.a1.a2": "_a2"}, ".")
  534. False
  535. >>> _simple_layout(['a', 'a.b'], {"": "src", "a.b": "_ab"}, "/tmp/myproj")
  536. False
  537. >>> # Special cases, no packages yet:
  538. >>> _simple_layout([], {"": "src"}, "/tmp/myproj")
  539. True
  540. >>> _simple_layout([], {"a": "_a", "": "src"}, "/tmp/myproj")
  541. False
  542. """
  543. layout = {pkg: find_package_path(pkg, package_dir, project_dir) for pkg in packages}
  544. if not layout:
  545. return set(package_dir) in ({}, {""})
  546. parent = os.path.commonpath(starmap(_parent_path, layout.items()))
  547. return all(
  548. _path.same_path(Path(parent, *key.split('.')), value)
  549. for key, value in layout.items()
  550. )
  551. def _parent_path(pkg, pkg_path):
  552. """Infer the parent path containing a package, that if added to ``sys.path`` would
  553. allow importing that package.
  554. When ``pkg`` is directly mapped into a directory with a different name, return its
  555. own path.
  556. >>> _parent_path("a", "src/a")
  557. 'src'
  558. >>> _parent_path("b", "src/c")
  559. 'src/c'
  560. """
  561. parent = pkg_path.removesuffix(pkg)
  562. return parent.rstrip("/" + os.sep)
  563. def _find_packages(dist: Distribution) -> Iterator[str]:
  564. yield from iter(dist.packages or [])
  565. py_modules = dist.py_modules or []
  566. nested_modules = [mod for mod in py_modules if "." in mod]
  567. if dist.ext_package:
  568. yield dist.ext_package
  569. else:
  570. ext_modules = dist.ext_modules or []
  571. nested_modules += [x.name for x in ext_modules if "." in x.name]
  572. for module in nested_modules:
  573. package, _, _ = module.rpartition(".")
  574. yield package
  575. def _find_top_level_modules(dist: Distribution) -> Iterator[str]:
  576. py_modules = dist.py_modules or []
  577. yield from (mod for mod in py_modules if "." not in mod)
  578. if not dist.ext_package:
  579. ext_modules = dist.ext_modules or []
  580. yield from (x.name for x in ext_modules if "." not in x.name)
  581. def _find_package_roots(
  582. packages: Iterable[str],
  583. package_dir: Mapping[str, str],
  584. src_root: StrPath,
  585. ) -> dict[str, str]:
  586. pkg_roots: dict[str, str] = {
  587. pkg: _absolute_root(find_package_path(pkg, package_dir, src_root))
  588. for pkg in sorted(packages)
  589. }
  590. return _remove_nested(pkg_roots)
  591. def _absolute_root(path: StrPath) -> str:
  592. """Works for packages and top-level modules"""
  593. path_ = Path(path)
  594. parent = path_.parent
  595. if path_.exists():
  596. return str(path_.resolve())
  597. else:
  598. return str(parent.resolve() / path_.name)
  599. def _find_virtual_namespaces(pkg_roots: dict[str, str]) -> Iterator[str]:
  600. """By carefully designing ``package_dir``, it is possible to implement the logical
  601. structure of PEP 420 in a package without the corresponding directories.
  602. Moreover a parent package can be purposefully/accidentally skipped in the discovery
  603. phase (e.g. ``find_packages(include=["mypkg.*"])``, when ``mypkg.foo`` is included
  604. by ``mypkg`` itself is not).
  605. We consider this case to also be a virtual namespace (ignoring the original
  606. directory) to emulate a non-editable installation.
  607. This function will try to find these kinds of namespaces.
  608. """
  609. for pkg in pkg_roots:
  610. if "." not in pkg:
  611. continue
  612. parts = pkg.split(".")
  613. for i in range(len(parts) - 1, 0, -1):
  614. partial_name = ".".join(parts[:i])
  615. path = Path(find_package_path(partial_name, pkg_roots, ""))
  616. if not path.exists() or partial_name not in pkg_roots:
  617. # partial_name not in pkg_roots ==> purposefully/accidentally skipped
  618. yield partial_name
  619. def _find_namespaces(
  620. packages: list[str], pkg_roots: dict[str, str]
  621. ) -> Iterator[tuple[str, list[str]]]:
  622. for pkg in packages:
  623. path = find_package_path(pkg, pkg_roots, "")
  624. if Path(path).exists() and not Path(path, "__init__.py").exists():
  625. yield (pkg, [path])
  626. def _remove_nested(pkg_roots: dict[str, str]) -> dict[str, str]:
  627. output = dict(pkg_roots.copy())
  628. for pkg, path in reversed(list(pkg_roots.items())):
  629. if any(
  630. pkg != other and _is_nested(pkg, path, other, other_path)
  631. for other, other_path in pkg_roots.items()
  632. ):
  633. output.pop(pkg)
  634. return output
  635. def _is_nested(pkg: str, pkg_path: str, parent: str, parent_path: str) -> bool:
  636. """
  637. Return ``True`` if ``pkg`` is nested inside ``parent`` both logically and in the
  638. file system.
  639. >>> _is_nested("a.b", "path/a/b", "a", "path/a")
  640. True
  641. >>> _is_nested("a.b", "path/a/b", "a", "otherpath/a")
  642. False
  643. >>> _is_nested("a.b", "path/a/b", "c", "path/c")
  644. False
  645. >>> _is_nested("a.a", "path/a/a", "a", "path/a")
  646. True
  647. >>> _is_nested("b.a", "path/b/a", "a", "path/a")
  648. False
  649. """
  650. norm_pkg_path = _path.normpath(pkg_path)
  651. rest = pkg.replace(parent, "", 1).strip(".").split(".")
  652. return pkg.startswith(parent) and norm_pkg_path == _path.normpath(
  653. Path(parent_path, *rest)
  654. )
  655. def _empty_dir(dir_: _P) -> _P:
  656. """Create a directory ensured to be empty. Existing files may be removed."""
  657. _shutil.rmtree(dir_, ignore_errors=True)
  658. os.makedirs(dir_)
  659. return dir_
  660. class _NamespaceInstaller(namespaces.Installer):
  661. def __init__(self, distribution, installation_dir, editable_name, src_root) -> None:
  662. self.distribution = distribution
  663. self.src_root = src_root
  664. self.installation_dir = installation_dir
  665. self.editable_name = editable_name
  666. self.outputs: list[str] = []
  667. def _get_nspkg_file(self):
  668. """Installation target."""
  669. return os.path.join(self.installation_dir, self.editable_name + self.nspkg_ext)
  670. def _get_root(self):
  671. """Where the modules/packages should be loaded from."""
  672. return repr(str(self.src_root))
  673. _FINDER_TEMPLATE = """\
  674. from __future__ import annotations
  675. import sys
  676. from importlib.machinery import ModuleSpec, PathFinder
  677. from importlib.machinery import all_suffixes as module_suffixes
  678. from importlib.util import spec_from_file_location
  679. from itertools import chain
  680. from pathlib import Path
  681. MAPPING: dict[str, str] = {mapping!r}
  682. NAMESPACES: dict[str, list[str]] = {namespaces!r}
  683. PATH_PLACEHOLDER = {name!r} + ".__path_hook__"
  684. class _EditableFinder: # MetaPathFinder
  685. @classmethod
  686. def find_spec(cls, fullname: str, path=None, target=None) -> ModuleSpec | None: # type: ignore
  687. # Top-level packages and modules (we know these exist in the FS)
  688. if fullname in MAPPING:
  689. pkg_path = MAPPING[fullname]
  690. return cls._find_spec(fullname, Path(pkg_path))
  691. # Handle immediate children modules (required for namespaces to work)
  692. # To avoid problems with case sensitivity in the file system we delegate
  693. # to the importlib.machinery implementation.
  694. parent, _, child = fullname.rpartition(".")
  695. if parent and parent in MAPPING:
  696. return PathFinder.find_spec(fullname, path=[MAPPING[parent]])
  697. # Other levels of nesting should be handled automatically by importlib
  698. # using the parent path.
  699. return None
  700. @classmethod
  701. def _find_spec(cls, fullname: str, candidate_path: Path) -> ModuleSpec | None:
  702. init = candidate_path / "__init__.py"
  703. candidates = (candidate_path.with_suffix(x) for x in module_suffixes())
  704. for candidate in chain([init], candidates):
  705. if candidate.exists():
  706. return spec_from_file_location(fullname, candidate)
  707. return None
  708. class _EditableNamespaceFinder: # PathEntryFinder
  709. @classmethod
  710. def _path_hook(cls, path) -> type[_EditableNamespaceFinder]:
  711. if path == PATH_PLACEHOLDER:
  712. return cls
  713. raise ImportError
  714. @classmethod
  715. def _paths(cls, fullname: str) -> list[str]:
  716. paths = NAMESPACES[fullname]
  717. if not paths and fullname in MAPPING:
  718. paths = [MAPPING[fullname]]
  719. # Always add placeholder, for 2 reasons:
  720. # 1. __path__ cannot be empty for the spec to be considered namespace.
  721. # 2. In the case of nested namespaces, we need to force
  722. # import machinery to query _EditableNamespaceFinder again.
  723. return [*paths, PATH_PLACEHOLDER]
  724. @classmethod
  725. def find_spec(cls, fullname: str, target=None) -> ModuleSpec | None: # type: ignore
  726. if fullname in NAMESPACES:
  727. spec = ModuleSpec(fullname, None, is_package=True)
  728. spec.submodule_search_locations = cls._paths(fullname)
  729. return spec
  730. return None
  731. @classmethod
  732. def find_module(cls, _fullname) -> None:
  733. return None
  734. def install():
  735. if not any(finder == _EditableFinder for finder in sys.meta_path):
  736. sys.meta_path.append(_EditableFinder)
  737. if not NAMESPACES:
  738. return
  739. if not any(hook == _EditableNamespaceFinder._path_hook for hook in sys.path_hooks):
  740. # PathEntryFinder is needed to create NamespaceSpec without private APIS
  741. sys.path_hooks.append(_EditableNamespaceFinder._path_hook)
  742. if PATH_PLACEHOLDER not in sys.path:
  743. sys.path.append(PATH_PLACEHOLDER) # Used just to trigger the path hook
  744. """
  745. def _finder_template(
  746. name: str, mapping: Mapping[str, str], namespaces: dict[str, list[str]]
  747. ) -> str:
  748. """Create a string containing the code for the``MetaPathFinder`` and
  749. ``PathEntryFinder``.
  750. """
  751. mapping = dict(sorted(mapping.items(), key=operator.itemgetter(0)))
  752. return _FINDER_TEMPLATE.format(name=name, mapping=mapping, namespaces=namespaces)
  753. class LinksNotSupported(errors.FileError):
  754. """File system does not seem to support either symlinks or hard links."""