wheel_builder.py 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261
  1. """Orchestrator for building wheels from InstallRequirements."""
  2. from __future__ import annotations
  3. import logging
  4. import os.path
  5. import re
  6. from collections.abc import Iterable
  7. from tempfile import TemporaryDirectory
  8. from pip._vendor.packaging.utils import canonicalize_name, canonicalize_version
  9. from pip._vendor.packaging.version import InvalidVersion, Version
  10. from pip._internal.cache import WheelCache
  11. from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel
  12. from pip._internal.metadata import FilesystemWheel, get_wheel_distribution
  13. from pip._internal.models.link import Link
  14. from pip._internal.models.wheel import Wheel
  15. from pip._internal.operations.build.wheel import build_wheel_pep517
  16. from pip._internal.operations.build.wheel_editable import build_wheel_editable
  17. from pip._internal.req.req_install import InstallRequirement
  18. from pip._internal.utils.logging import indent_log
  19. from pip._internal.utils.misc import ensure_dir, hash_file
  20. from pip._internal.utils.urls import path_to_url
  21. from pip._internal.vcs import vcs
  22. logger = logging.getLogger(__name__)
  23. _egg_info_re = re.compile(r"([a-z0-9_.]+)-([a-z0-9_.!+-]+)", re.IGNORECASE)
  24. BuildResult = tuple[list[InstallRequirement], list[InstallRequirement]]
  25. def _contains_egg_info(s: str) -> bool:
  26. """Determine whether the string looks like an egg_info.
  27. :param s: The string to parse. E.g. foo-2.1
  28. """
  29. return bool(_egg_info_re.search(s))
  30. def _should_cache(
  31. req: InstallRequirement,
  32. ) -> bool | None:
  33. """
  34. Return whether a built InstallRequirement can be stored in the persistent
  35. wheel cache, assuming the wheel cache is available.
  36. """
  37. if req.editable or not req.source_dir:
  38. # never cache editable requirements
  39. return False
  40. if req.link and req.link.is_vcs:
  41. # VCS checkout. Do not cache
  42. # unless it points to an immutable commit hash.
  43. assert not req.editable
  44. assert req.source_dir
  45. vcs_backend = vcs.get_backend_for_scheme(req.link.scheme)
  46. assert vcs_backend
  47. if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir):
  48. return True
  49. return False
  50. assert req.link
  51. base, ext = req.link.splitext()
  52. if _contains_egg_info(base):
  53. return True
  54. # Otherwise, do not cache.
  55. return False
  56. def _get_cache_dir(
  57. req: InstallRequirement,
  58. wheel_cache: WheelCache,
  59. ) -> str:
  60. """Return the persistent or temporary cache directory where the built
  61. wheel need to be stored.
  62. """
  63. cache_available = bool(wheel_cache.cache_dir)
  64. assert req.link
  65. if cache_available and _should_cache(req):
  66. cache_dir = wheel_cache.get_path_for_link(req.link)
  67. else:
  68. cache_dir = wheel_cache.get_ephem_path_for_link(req.link)
  69. return cache_dir
  70. def _verify_one(req: InstallRequirement, wheel_path: str) -> None:
  71. canonical_name = canonicalize_name(req.name or "")
  72. w = Wheel(os.path.basename(wheel_path))
  73. if w.name != canonical_name:
  74. raise InvalidWheelFilename(
  75. f"Wheel has unexpected file name: expected {canonical_name!r}, "
  76. f"got {w.name!r}",
  77. )
  78. dist = get_wheel_distribution(FilesystemWheel(wheel_path), canonical_name)
  79. dist_verstr = str(dist.version)
  80. if canonicalize_version(dist_verstr) != canonicalize_version(w.version):
  81. raise InvalidWheelFilename(
  82. f"Wheel has unexpected file name: expected {dist_verstr!r}, "
  83. f"got {w.version!r}",
  84. )
  85. metadata_version_value = dist.metadata_version
  86. if metadata_version_value is None:
  87. raise UnsupportedWheel("Missing Metadata-Version")
  88. try:
  89. metadata_version = Version(metadata_version_value)
  90. except InvalidVersion:
  91. msg = f"Invalid Metadata-Version: {metadata_version_value}"
  92. raise UnsupportedWheel(msg)
  93. if metadata_version >= Version("1.2") and not isinstance(dist.version, Version):
  94. raise UnsupportedWheel(
  95. f"Metadata 1.2 mandates PEP 440 version, but {dist_verstr!r} is not"
  96. )
  97. def _build_one(
  98. req: InstallRequirement,
  99. output_dir: str,
  100. verify: bool,
  101. editable: bool,
  102. ) -> str | None:
  103. """Build one wheel.
  104. :return: The filename of the built wheel, or None if the build failed.
  105. """
  106. artifact = "editable" if editable else "wheel"
  107. try:
  108. ensure_dir(output_dir)
  109. except OSError as e:
  110. logger.warning(
  111. "Building %s for %s failed: %s",
  112. artifact,
  113. req.name,
  114. e,
  115. )
  116. return None
  117. # Install build deps into temporary directory (PEP 518)
  118. with req.build_env:
  119. wheel_path = _build_one_inside_env(req, output_dir, editable)
  120. if wheel_path and verify:
  121. try:
  122. _verify_one(req, wheel_path)
  123. except (InvalidWheelFilename, UnsupportedWheel) as e:
  124. logger.warning("Built %s for %s is invalid: %s", artifact, req.name, e)
  125. return None
  126. return wheel_path
  127. def _build_one_inside_env(
  128. req: InstallRequirement,
  129. output_dir: str,
  130. editable: bool,
  131. ) -> str | None:
  132. with TemporaryDirectory(dir=output_dir) as wheel_directory:
  133. assert req.name
  134. assert req.metadata_directory
  135. assert req.pep517_backend
  136. if editable:
  137. wheel_path = build_wheel_editable(
  138. name=req.name,
  139. backend=req.pep517_backend,
  140. metadata_directory=req.metadata_directory,
  141. wheel_directory=wheel_directory,
  142. )
  143. else:
  144. wheel_path = build_wheel_pep517(
  145. name=req.name,
  146. backend=req.pep517_backend,
  147. metadata_directory=req.metadata_directory,
  148. wheel_directory=wheel_directory,
  149. )
  150. if wheel_path is not None:
  151. wheel_name = os.path.basename(wheel_path)
  152. dest_path = os.path.join(output_dir, wheel_name)
  153. try:
  154. wheel_hash, length = hash_file(wheel_path)
  155. # We can do a replace here because wheel_path is guaranteed to
  156. # be in the same filesystem as output_dir. This will perform an
  157. # atomic rename, which is necessary to avoid concurrency issues
  158. # when populating the cache.
  159. os.replace(wheel_path, dest_path)
  160. logger.info(
  161. "Created wheel for %s: filename=%s size=%d sha256=%s",
  162. req.name,
  163. wheel_name,
  164. length,
  165. wheel_hash.hexdigest(),
  166. )
  167. logger.info("Stored in directory: %s", output_dir)
  168. return dest_path
  169. except Exception as e:
  170. logger.warning(
  171. "Building wheel for %s failed: %s",
  172. req.name,
  173. e,
  174. )
  175. return None
  176. def build(
  177. requirements: Iterable[InstallRequirement],
  178. wheel_cache: WheelCache,
  179. verify: bool,
  180. ) -> BuildResult:
  181. """Build wheels.
  182. :return: The list of InstallRequirement that succeeded to build and
  183. the list of InstallRequirement that failed to build.
  184. """
  185. if not requirements:
  186. return [], []
  187. # Build the wheels.
  188. logger.info(
  189. "Building wheels for collected packages: %s",
  190. ", ".join(req.name for req in requirements), # type: ignore
  191. )
  192. with indent_log():
  193. build_successes, build_failures = [], []
  194. for req in requirements:
  195. assert req.name
  196. cache_dir = _get_cache_dir(req, wheel_cache)
  197. wheel_file = _build_one(
  198. req,
  199. cache_dir,
  200. verify,
  201. req.editable and req.permit_editable_wheels,
  202. )
  203. if wheel_file:
  204. # Record the download origin in the cache
  205. if req.download_info is not None:
  206. # download_info is guaranteed to be set because when we build an
  207. # InstallRequirement it has been through the preparer before, but
  208. # let's be cautious.
  209. wheel_cache.record_download_origin(cache_dir, req.download_info)
  210. # Update the link for this.
  211. req.link = Link(path_to_url(wheel_file))
  212. req.local_file_path = req.link.file_path
  213. assert req.link.is_wheel
  214. build_successes.append(req)
  215. else:
  216. build_failures.append(req)
  217. # notify success/failure
  218. if build_successes:
  219. logger.info(
  220. "Successfully built %s",
  221. " ".join([req.name for req in build_successes]), # type: ignore
  222. )
  223. if build_failures:
  224. logger.info(
  225. "Failed to build %s",
  226. " ".join([req.name for req in build_failures]), # type: ignore
  227. )
  228. # Return a list of requirements that failed to build
  229. return build_successes, build_failures