wheel.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. """Wheels support."""
  2. import contextlib
  3. import email
  4. import functools
  5. import itertools
  6. import os
  7. import posixpath
  8. import re
  9. import zipfile
  10. from collections.abc import Iterator
  11. from packaging.requirements import Requirement
  12. from packaging.tags import sys_tags
  13. from packaging.utils import canonicalize_name
  14. from packaging.version import Version as parse_version
  15. import setuptools
  16. from setuptools.archive_util import _unpack_zipfile_obj
  17. from setuptools.command.egg_info import _egg_basename, write_requirements
  18. from ._discovery import extras_from_deps
  19. from ._importlib import metadata
  20. from .unicode_utils import _read_utf8_with_fallback
  21. from distutils.util import get_platform
  22. WHEEL_NAME = re.compile(
  23. r"""^(?P<project_name>.+?)-(?P<version>\d.*?)
  24. ((-(?P<build>\d.*?))?-(?P<py_version>.+?)-(?P<abi>.+?)-(?P<platform>.+?)
  25. )\.whl$""",
  26. re.VERBOSE,
  27. ).match
  28. NAMESPACE_PACKAGE_INIT = "__import__('pkg_resources').declare_namespace(__name__)\n"
  29. @functools.cache
  30. def _get_supported_tags():
  31. # We calculate the supported tags only once, otherwise calling
  32. # this method on thousands of wheels takes seconds instead of
  33. # milliseconds.
  34. return {(t.interpreter, t.abi, t.platform) for t in sys_tags()}
  35. def unpack(src_dir, dst_dir) -> None:
  36. """Move everything under `src_dir` to `dst_dir`, and delete the former."""
  37. for dirpath, dirnames, filenames in os.walk(src_dir):
  38. subdir = os.path.relpath(dirpath, src_dir)
  39. for f in filenames:
  40. src = os.path.join(dirpath, f)
  41. dst = os.path.join(dst_dir, subdir, f)
  42. os.renames(src, dst)
  43. for n, d in reversed(list(enumerate(dirnames))):
  44. src = os.path.join(dirpath, d)
  45. dst = os.path.join(dst_dir, subdir, d)
  46. if not os.path.exists(dst):
  47. # Directory does not exist in destination,
  48. # rename it and prune it from os.walk list.
  49. os.renames(src, dst)
  50. del dirnames[n]
  51. # Cleanup.
  52. for dirpath, dirnames, filenames in os.walk(src_dir, topdown=True):
  53. assert not filenames
  54. os.rmdir(dirpath)
  55. @contextlib.contextmanager
  56. def disable_info_traces() -> Iterator[None]:
  57. """
  58. Temporarily disable info traces.
  59. """
  60. from distutils import log
  61. saved = log.set_threshold(log.WARN)
  62. try:
  63. yield
  64. finally:
  65. log.set_threshold(saved)
  66. class Wheel:
  67. def __init__(self, filename) -> None:
  68. match = WHEEL_NAME(os.path.basename(filename))
  69. if match is None:
  70. raise ValueError(f'invalid wheel name: {filename!r}')
  71. self.filename = filename
  72. for k, v in match.groupdict().items():
  73. setattr(self, k, v)
  74. def tags(self):
  75. """List tags (py_version, abi, platform) supported by this wheel."""
  76. return itertools.product(
  77. self.py_version.split('.'),
  78. self.abi.split('.'),
  79. self.platform.split('.'),
  80. )
  81. def is_compatible(self):
  82. """Is the wheel compatible with the current platform?"""
  83. return next((True for t in self.tags() if t in _get_supported_tags()), False)
  84. def egg_name(self):
  85. return (
  86. _egg_basename(
  87. self.project_name,
  88. self.version,
  89. platform=(None if self.platform == 'any' else get_platform()),
  90. )
  91. + ".egg"
  92. )
  93. def get_dist_info(self, zf):
  94. # find the correct name of the .dist-info dir in the wheel file
  95. for member in zf.namelist():
  96. dirname = posixpath.dirname(member)
  97. if dirname.endswith('.dist-info') and canonicalize_name(dirname).startswith(
  98. canonicalize_name(self.project_name)
  99. ):
  100. return dirname
  101. raise ValueError("unsupported wheel format. .dist-info not found")
  102. def install_as_egg(self, destination_eggdir) -> None:
  103. """Install wheel as an egg directory."""
  104. with zipfile.ZipFile(self.filename) as zf:
  105. self._install_as_egg(destination_eggdir, zf)
  106. def _install_as_egg(self, destination_eggdir, zf):
  107. dist_basename = f'{self.project_name}-{self.version}'
  108. dist_info = self.get_dist_info(zf)
  109. dist_data = f'{dist_basename}.data'
  110. egg_info = os.path.join(destination_eggdir, 'EGG-INFO')
  111. self._convert_metadata(zf, destination_eggdir, dist_info, egg_info)
  112. self._move_data_entries(destination_eggdir, dist_data)
  113. self._fix_namespace_packages(egg_info, destination_eggdir)
  114. @staticmethod
  115. def _convert_metadata(zf, destination_eggdir, dist_info, egg_info):
  116. def get_metadata(name):
  117. with zf.open(posixpath.join(dist_info, name)) as fp:
  118. value = fp.read().decode('utf-8')
  119. return email.parser.Parser().parsestr(value)
  120. wheel_metadata = get_metadata('WHEEL')
  121. # Check wheel format version is supported.
  122. wheel_version = parse_version(wheel_metadata.get('Wheel-Version'))
  123. wheel_v1 = parse_version('1.0') <= wheel_version < parse_version('2.0dev0')
  124. if not wheel_v1:
  125. raise ValueError(f'unsupported wheel format version: {wheel_version}')
  126. # Extract to target directory.
  127. _unpack_zipfile_obj(zf, destination_eggdir)
  128. dist_info = os.path.join(destination_eggdir, dist_info)
  129. install_requires, extras_require = Wheel._convert_requires(
  130. destination_eggdir, dist_info
  131. )
  132. os.rename(dist_info, egg_info)
  133. os.rename(
  134. os.path.join(egg_info, 'METADATA'),
  135. os.path.join(egg_info, 'PKG-INFO'),
  136. )
  137. setup_dist = setuptools.Distribution(
  138. attrs=dict(
  139. install_requires=install_requires,
  140. extras_require=extras_require,
  141. ),
  142. )
  143. with disable_info_traces():
  144. write_requirements(
  145. setup_dist.get_command_obj('egg_info'),
  146. None,
  147. os.path.join(egg_info, 'requires.txt'),
  148. )
  149. @staticmethod
  150. def _convert_requires(destination_eggdir, dist_info):
  151. md = metadata.Distribution.at(dist_info).metadata
  152. deps = md.get_all('Requires-Dist') or []
  153. reqs = list(map(Requirement, deps))
  154. extras = extras_from_deps(deps)
  155. # Note: Evaluate and strip markers now,
  156. # as it's difficult to convert back from the syntax:
  157. # foobar; "linux" in sys_platform and extra == 'test'
  158. def raw_req(req):
  159. req = Requirement(str(req))
  160. req.marker = None
  161. return str(req)
  162. def eval(req, **env):
  163. return not req.marker or req.marker.evaluate(env)
  164. def for_extra(req):
  165. try:
  166. markers = req.marker._markers
  167. except AttributeError:
  168. markers = ()
  169. return set(
  170. marker[2].value
  171. for marker in markers
  172. if isinstance(marker, tuple) and marker[0].value == 'extra'
  173. )
  174. install_requires = list(
  175. map(raw_req, filter(eval, itertools.filterfalse(for_extra, reqs)))
  176. )
  177. extras_require = {
  178. extra: list(
  179. map(
  180. raw_req,
  181. (req for req in reqs if for_extra(req) and eval(req, extra=extra)),
  182. )
  183. )
  184. for extra in extras
  185. }
  186. return install_requires, extras_require
  187. @staticmethod
  188. def _move_data_entries(destination_eggdir, dist_data):
  189. """Move data entries to their correct location."""
  190. dist_data = os.path.join(destination_eggdir, dist_data)
  191. dist_data_scripts = os.path.join(dist_data, 'scripts')
  192. if os.path.exists(dist_data_scripts):
  193. egg_info_scripts = os.path.join(destination_eggdir, 'EGG-INFO', 'scripts')
  194. os.mkdir(egg_info_scripts)
  195. for entry in os.listdir(dist_data_scripts):
  196. # Remove bytecode, as it's not properly handled
  197. # during easy_install scripts install phase.
  198. if entry.endswith('.pyc'):
  199. os.unlink(os.path.join(dist_data_scripts, entry))
  200. else:
  201. os.rename(
  202. os.path.join(dist_data_scripts, entry),
  203. os.path.join(egg_info_scripts, entry),
  204. )
  205. os.rmdir(dist_data_scripts)
  206. for subdir in filter(
  207. os.path.exists,
  208. (
  209. os.path.join(dist_data, d)
  210. for d in ('data', 'headers', 'purelib', 'platlib')
  211. ),
  212. ):
  213. unpack(subdir, destination_eggdir)
  214. if os.path.exists(dist_data):
  215. os.rmdir(dist_data)
  216. @staticmethod
  217. def _fix_namespace_packages(egg_info, destination_eggdir):
  218. namespace_packages = os.path.join(egg_info, 'namespace_packages.txt')
  219. if os.path.exists(namespace_packages):
  220. namespace_packages = _read_utf8_with_fallback(namespace_packages).split()
  221. for mod in namespace_packages:
  222. mod_dir = os.path.join(destination_eggdir, *mod.split('.'))
  223. mod_init = os.path.join(mod_dir, '__init__.py')
  224. if not os.path.exists(mod_dir):
  225. os.mkdir(mod_dir)
  226. if not os.path.exists(mod_init):
  227. with open(mod_init, 'w', encoding="utf-8") as fp:
  228. fp.write(NAMESPACE_PACKAGE_INIT)