wheel.py 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2013-2023 Vinay Sajip.
  4. # Licensed to the Python Software Foundation under a contributor agreement.
  5. # See LICENSE.txt and CONTRIBUTORS.txt.
  6. #
  7. from __future__ import unicode_literals
  8. import base64
  9. import codecs
  10. import datetime
  11. from email import message_from_file
  12. import hashlib
  13. import json
  14. import logging
  15. import os
  16. import posixpath
  17. import re
  18. import shutil
  19. import sys
  20. import tempfile
  21. import zipfile
  22. from . import __version__, DistlibException
  23. from .compat import sysconfig, ZipFile, fsdecode, text_type, filter
  24. from .database import InstalledDistribution
  25. from .metadata import Metadata, WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME
  26. from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache, cached_property, get_cache_base,
  27. read_exports, tempdir, get_platform)
  28. from .version import NormalizedVersion, UnsupportedVersionError
  29. logger = logging.getLogger(__name__)
  30. cache = None # created when needed
  31. if hasattr(sys, 'pypy_version_info'): # pragma: no cover
  32. IMP_PREFIX = 'pp'
  33. elif sys.platform.startswith('java'): # pragma: no cover
  34. IMP_PREFIX = 'jy'
  35. elif sys.platform == 'cli': # pragma: no cover
  36. IMP_PREFIX = 'ip'
  37. else:
  38. IMP_PREFIX = 'cp'
  39. VER_SUFFIX = sysconfig.get_config_var('py_version_nodot')
  40. if not VER_SUFFIX: # pragma: no cover
  41. VER_SUFFIX = '%s%s' % sys.version_info[:2]
  42. PYVER = 'py' + VER_SUFFIX
  43. IMPVER = IMP_PREFIX + VER_SUFFIX
  44. ARCH = get_platform().replace('-', '_').replace('.', '_')
  45. ABI = sysconfig.get_config_var('SOABI')
  46. if ABI and ABI.startswith('cpython-'):
  47. ABI = ABI.replace('cpython-', 'cp').split('-')[0]
  48. else:
  49. def _derive_abi():
  50. parts = ['cp', VER_SUFFIX]
  51. if sysconfig.get_config_var('Py_DEBUG'):
  52. parts.append('d')
  53. if IMP_PREFIX == 'cp':
  54. vi = sys.version_info[:2]
  55. if vi < (3, 8):
  56. wpm = sysconfig.get_config_var('WITH_PYMALLOC')
  57. if wpm is None:
  58. wpm = True
  59. if wpm:
  60. parts.append('m')
  61. if vi < (3, 3):
  62. us = sysconfig.get_config_var('Py_UNICODE_SIZE')
  63. if us == 4 or (us is None and sys.maxunicode == 0x10FFFF):
  64. parts.append('u')
  65. if bool(sysconfig.get_config_var("Py_GIL_DISABLED")):
  66. parts.append('t')
  67. return ''.join(parts)
  68. ABI = _derive_abi()
  69. del _derive_abi
  70. FILENAME_RE = re.compile(
  71. r'''
  72. (?P<nm>[^-]+)
  73. -(?P<vn>\d+[^-]*)
  74. (-(?P<bn>\d+[^-]*))?
  75. -(?P<py>\w+\d+(\.\w+\d+)*)
  76. -(?P<bi>\w+)
  77. -(?P<ar>\w+(\.\w+)*)
  78. \.whl$
  79. ''', re.IGNORECASE | re.VERBOSE)
  80. NAME_VERSION_RE = re.compile(r'''
  81. (?P<nm>[^-]+)
  82. -(?P<vn>\d+[^-]*)
  83. (-(?P<bn>\d+[^-]*))?$
  84. ''', re.IGNORECASE | re.VERBOSE)
  85. SHEBANG_RE = re.compile(br'\s*#![^\r\n]*')
  86. SHEBANG_DETAIL_RE = re.compile(br'^(\s*#!("[^"]+"|\S+))\s+(.*)$')
  87. SHEBANG_PYTHON = b'#!python'
  88. SHEBANG_PYTHONW = b'#!pythonw'
  89. if os.sep == '/':
  90. to_posix = lambda o: o
  91. else:
  92. to_posix = lambda o: o.replace(os.sep, '/')
  93. if sys.version_info[0] < 3:
  94. import imp
  95. else:
  96. imp = None
  97. import importlib.machinery
  98. import importlib.util
  99. def _get_suffixes():
  100. if imp:
  101. return [s[0] for s in imp.get_suffixes()]
  102. else:
  103. return importlib.machinery.EXTENSION_SUFFIXES
  104. def _load_dynamic(name, path):
  105. # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
  106. if imp:
  107. return imp.load_dynamic(name, path)
  108. else:
  109. spec = importlib.util.spec_from_file_location(name, path)
  110. module = importlib.util.module_from_spec(spec)
  111. sys.modules[name] = module
  112. spec.loader.exec_module(module)
  113. return module
  114. class Mounter(object):
  115. def __init__(self):
  116. self.impure_wheels = {}
  117. self.libs = {}
  118. def add(self, pathname, extensions):
  119. self.impure_wheels[pathname] = extensions
  120. self.libs.update(extensions)
  121. def remove(self, pathname):
  122. extensions = self.impure_wheels.pop(pathname)
  123. for k, v in extensions:
  124. if k in self.libs:
  125. del self.libs[k]
  126. def find_module(self, fullname, path=None):
  127. if fullname in self.libs:
  128. result = self
  129. else:
  130. result = None
  131. return result
  132. def load_module(self, fullname):
  133. if fullname in sys.modules:
  134. result = sys.modules[fullname]
  135. else:
  136. if fullname not in self.libs:
  137. raise ImportError('unable to find extension for %s' % fullname)
  138. result = _load_dynamic(fullname, self.libs[fullname])
  139. result.__loader__ = self
  140. parts = fullname.rsplit('.', 1)
  141. if len(parts) > 1:
  142. result.__package__ = parts[0]
  143. return result
  144. _hook = Mounter()
  145. class Wheel(object):
  146. """
  147. Class to build and install from Wheel files (PEP 427).
  148. """
  149. wheel_version = (1, 1)
  150. hash_kind = 'sha256'
  151. def __init__(self, filename=None, sign=False, verify=False):
  152. """
  153. Initialise an instance using a (valid) filename.
  154. """
  155. self.sign = sign
  156. self.should_verify = verify
  157. self.buildver = ''
  158. self.pyver = [PYVER]
  159. self.abi = ['none']
  160. self.arch = ['any']
  161. self.dirname = os.getcwd()
  162. if filename is None:
  163. self.name = 'dummy'
  164. self.version = '0.1'
  165. self._filename = self.filename
  166. else:
  167. m = NAME_VERSION_RE.match(filename)
  168. if m:
  169. info = m.groupdict('')
  170. self.name = info['nm']
  171. # Reinstate the local version separator
  172. self.version = info['vn'].replace('_', '-')
  173. self.buildver = info['bn']
  174. self._filename = self.filename
  175. else:
  176. dirname, filename = os.path.split(filename)
  177. m = FILENAME_RE.match(filename)
  178. if not m:
  179. raise DistlibException('Invalid name or '
  180. 'filename: %r' % filename)
  181. if dirname:
  182. self.dirname = os.path.abspath(dirname)
  183. self._filename = filename
  184. info = m.groupdict('')
  185. self.name = info['nm']
  186. self.version = info['vn']
  187. self.buildver = info['bn']
  188. self.pyver = info['py'].split('.')
  189. self.abi = info['bi'].split('.')
  190. self.arch = info['ar'].split('.')
  191. @property
  192. def filename(self):
  193. """
  194. Build and return a filename from the various components.
  195. """
  196. if self.buildver:
  197. buildver = '-' + self.buildver
  198. else:
  199. buildver = ''
  200. pyver = '.'.join(self.pyver)
  201. abi = '.'.join(self.abi)
  202. arch = '.'.join(self.arch)
  203. # replace - with _ as a local version separator
  204. version = self.version.replace('-', '_')
  205. return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver, pyver, abi, arch)
  206. @property
  207. def exists(self):
  208. path = os.path.join(self.dirname, self.filename)
  209. return os.path.isfile(path)
  210. @property
  211. def tags(self):
  212. for pyver in self.pyver:
  213. for abi in self.abi:
  214. for arch in self.arch:
  215. yield pyver, abi, arch
  216. @cached_property
  217. def metadata(self):
  218. pathname = os.path.join(self.dirname, self.filename)
  219. name_ver = '%s-%s' % (self.name, self.version)
  220. info_dir = '%s.dist-info' % name_ver
  221. wrapper = codecs.getreader('utf-8')
  222. with ZipFile(pathname, 'r') as zf:
  223. self.get_wheel_metadata(zf)
  224. # wv = wheel_metadata['Wheel-Version'].split('.', 1)
  225. # file_version = tuple([int(i) for i in wv])
  226. # if file_version < (1, 1):
  227. # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME,
  228. # LEGACY_METADATA_FILENAME]
  229. # else:
  230. # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME]
  231. fns = [WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME]
  232. result = None
  233. for fn in fns:
  234. try:
  235. metadata_filename = posixpath.join(info_dir, fn)
  236. with zf.open(metadata_filename) as bf:
  237. wf = wrapper(bf)
  238. result = Metadata(fileobj=wf)
  239. if result:
  240. break
  241. except KeyError:
  242. pass
  243. if not result:
  244. raise ValueError('Invalid wheel, because metadata is '
  245. 'missing: looked in %s' % ', '.join(fns))
  246. return result
  247. def get_wheel_metadata(self, zf):
  248. name_ver = '%s-%s' % (self.name, self.version)
  249. info_dir = '%s.dist-info' % name_ver
  250. metadata_filename = posixpath.join(info_dir, 'WHEEL')
  251. with zf.open(metadata_filename) as bf:
  252. wf = codecs.getreader('utf-8')(bf)
  253. message = message_from_file(wf)
  254. return dict(message)
  255. @cached_property
  256. def info(self):
  257. pathname = os.path.join(self.dirname, self.filename)
  258. with ZipFile(pathname, 'r') as zf:
  259. result = self.get_wheel_metadata(zf)
  260. return result
  261. def process_shebang(self, data):
  262. m = SHEBANG_RE.match(data)
  263. if m:
  264. end = m.end()
  265. shebang, data_after_shebang = data[:end], data[end:]
  266. # Preserve any arguments after the interpreter
  267. if b'pythonw' in shebang.lower():
  268. shebang_python = SHEBANG_PYTHONW
  269. else:
  270. shebang_python = SHEBANG_PYTHON
  271. m = SHEBANG_DETAIL_RE.match(shebang)
  272. if m:
  273. args = b' ' + m.groups()[-1]
  274. else:
  275. args = b''
  276. shebang = shebang_python + args
  277. data = shebang + data_after_shebang
  278. else:
  279. cr = data.find(b'\r')
  280. lf = data.find(b'\n')
  281. if cr < 0 or cr > lf:
  282. term = b'\n'
  283. else:
  284. if data[cr:cr + 2] == b'\r\n':
  285. term = b'\r\n'
  286. else:
  287. term = b'\r'
  288. data = SHEBANG_PYTHON + term + data
  289. return data
  290. def get_hash(self, data, hash_kind=None):
  291. if hash_kind is None:
  292. hash_kind = self.hash_kind
  293. try:
  294. hasher = getattr(hashlib, hash_kind)
  295. except AttributeError:
  296. raise DistlibException('Unsupported hash algorithm: %r' % hash_kind)
  297. result = hasher(data).digest()
  298. result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii')
  299. return hash_kind, result
  300. def write_record(self, records, record_path, archive_record_path):
  301. records = list(records) # make a copy, as mutated
  302. records.append((archive_record_path, '', ''))
  303. with CSVWriter(record_path) as writer:
  304. for row in records:
  305. writer.writerow(row)
  306. def write_records(self, info, libdir, archive_paths):
  307. records = []
  308. distinfo, info_dir = info
  309. # hasher = getattr(hashlib, self.hash_kind)
  310. for ap, p in archive_paths:
  311. with open(p, 'rb') as f:
  312. data = f.read()
  313. digest = '%s=%s' % self.get_hash(data)
  314. size = os.path.getsize(p)
  315. records.append((ap, digest, size))
  316. p = os.path.join(distinfo, 'RECORD')
  317. ap = to_posix(os.path.join(info_dir, 'RECORD'))
  318. self.write_record(records, p, ap)
  319. archive_paths.append((ap, p))
  320. def build_zip(self, pathname, archive_paths):
  321. with ZipFile(pathname, 'w', zipfile.ZIP_DEFLATED) as zf:
  322. for ap, p in archive_paths:
  323. logger.debug('Wrote %s to %s in wheel', p, ap)
  324. zf.write(p, ap)
  325. def build(self, paths, tags=None, wheel_version=None):
  326. """
  327. Build a wheel from files in specified paths, and use any specified tags
  328. when determining the name of the wheel.
  329. """
  330. if tags is None:
  331. tags = {}
  332. libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0]
  333. if libkey == 'platlib':
  334. is_pure = 'false'
  335. default_pyver = [IMPVER]
  336. default_abi = [ABI]
  337. default_arch = [ARCH]
  338. else:
  339. is_pure = 'true'
  340. default_pyver = [PYVER]
  341. default_abi = ['none']
  342. default_arch = ['any']
  343. self.pyver = tags.get('pyver', default_pyver)
  344. self.abi = tags.get('abi', default_abi)
  345. self.arch = tags.get('arch', default_arch)
  346. libdir = paths[libkey]
  347. name_ver = '%s-%s' % (self.name, self.version)
  348. data_dir = '%s.data' % name_ver
  349. info_dir = '%s.dist-info' % name_ver
  350. archive_paths = []
  351. # First, stuff which is not in site-packages
  352. for key in ('data', 'headers', 'scripts'):
  353. if key not in paths:
  354. continue
  355. path = paths[key]
  356. if os.path.isdir(path):
  357. for root, dirs, files in os.walk(path):
  358. for fn in files:
  359. p = fsdecode(os.path.join(root, fn))
  360. rp = os.path.relpath(p, path)
  361. ap = to_posix(os.path.join(data_dir, key, rp))
  362. archive_paths.append((ap, p))
  363. if key == 'scripts' and not p.endswith('.exe'):
  364. with open(p, 'rb') as f:
  365. data = f.read()
  366. data = self.process_shebang(data)
  367. with open(p, 'wb') as f:
  368. f.write(data)
  369. # Now, stuff which is in site-packages, other than the
  370. # distinfo stuff.
  371. path = libdir
  372. distinfo = None
  373. for root, dirs, files in os.walk(path):
  374. if root == path:
  375. # At the top level only, save distinfo for later
  376. # and skip it for now
  377. for i, dn in enumerate(dirs):
  378. dn = fsdecode(dn)
  379. if dn.endswith('.dist-info'):
  380. distinfo = os.path.join(root, dn)
  381. del dirs[i]
  382. break
  383. assert distinfo, '.dist-info directory expected, not found'
  384. for fn in files:
  385. # comment out next suite to leave .pyc files in
  386. if fsdecode(fn).endswith(('.pyc', '.pyo')):
  387. continue
  388. p = os.path.join(root, fn)
  389. rp = to_posix(os.path.relpath(p, path))
  390. archive_paths.append((rp, p))
  391. # Now distinfo. It may contain subdirectories (e.g. PEP 639)
  392. for root, _, files in os.walk(distinfo):
  393. for fn in files:
  394. if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'):
  395. p = fsdecode(os.path.join(root, fn))
  396. r = os.path.relpath(root, distinfo)
  397. ap = to_posix(os.path.normpath(os.path.join(info_dir, r, fn)))
  398. archive_paths.append((ap, p))
  399. wheel_metadata = [
  400. 'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version),
  401. 'Generator: distlib %s' % __version__,
  402. 'Root-Is-Purelib: %s' % is_pure,
  403. ]
  404. if self.buildver:
  405. wheel_metadata.append('Build: %s' % self.buildver)
  406. for pyver, abi, arch in self.tags:
  407. wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch))
  408. p = os.path.join(distinfo, 'WHEEL')
  409. with open(p, 'w') as f:
  410. f.write('\n'.join(wheel_metadata))
  411. ap = to_posix(os.path.join(info_dir, 'WHEEL'))
  412. archive_paths.append((ap, p))
  413. # sort the entries by archive path. Not needed by any spec, but it
  414. # keeps the archive listing and RECORD tidier than they would otherwise
  415. # be. Use the number of path segments to keep directory entries together,
  416. # and keep the dist-info stuff at the end.
  417. def sorter(t):
  418. ap = t[0]
  419. n = ap.count('/')
  420. if '.dist-info' in ap:
  421. n += 10000
  422. return (n, ap)
  423. archive_paths = sorted(archive_paths, key=sorter)
  424. # Now, at last, RECORD.
  425. # Paths in here are archive paths - nothing else makes sense.
  426. self.write_records((distinfo, info_dir), libdir, archive_paths)
  427. # Now, ready to build the zip file
  428. pathname = os.path.join(self.dirname, self.filename)
  429. self.build_zip(pathname, archive_paths)
  430. return pathname
  431. def skip_entry(self, arcname):
  432. """
  433. Determine whether an archive entry should be skipped when verifying
  434. or installing.
  435. """
  436. # The signature file won't be in RECORD,
  437. # and we don't currently don't do anything with it
  438. # We also skip directories, as they won't be in RECORD
  439. # either. See:
  440. #
  441. # https://github.com/pypa/wheel/issues/294
  442. # https://github.com/pypa/wheel/issues/287
  443. # https://github.com/pypa/wheel/pull/289
  444. #
  445. return arcname.endswith(('/', '/RECORD.jws'))
  446. def install(self, paths, maker, **kwargs):
  447. """
  448. Install a wheel to the specified paths. If kwarg ``warner`` is
  449. specified, it should be a callable, which will be called with two
  450. tuples indicating the wheel version of this software and the wheel
  451. version in the file, if there is a discrepancy in the versions.
  452. This can be used to issue any warnings to raise any exceptions.
  453. If kwarg ``lib_only`` is True, only the purelib/platlib files are
  454. installed, and the headers, scripts, data and dist-info metadata are
  455. not written. If kwarg ``bytecode_hashed_invalidation`` is True, written
  456. bytecode will try to use file-hash based invalidation (PEP-552) on
  457. supported interpreter versions (CPython 3.7+).
  458. The return value is a :class:`InstalledDistribution` instance unless
  459. ``options.lib_only`` is True, in which case the return value is ``None``.
  460. """
  461. dry_run = maker.dry_run
  462. warner = kwargs.get('warner')
  463. lib_only = kwargs.get('lib_only', False)
  464. bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False)
  465. pathname = os.path.join(self.dirname, self.filename)
  466. name_ver = '%s-%s' % (self.name, self.version)
  467. data_dir = '%s.data' % name_ver
  468. info_dir = '%s.dist-info' % name_ver
  469. metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)
  470. wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
  471. record_name = posixpath.join(info_dir, 'RECORD')
  472. wrapper = codecs.getreader('utf-8')
  473. with ZipFile(pathname, 'r') as zf:
  474. with zf.open(wheel_metadata_name) as bwf:
  475. wf = wrapper(bwf)
  476. message = message_from_file(wf)
  477. wv = message['Wheel-Version'].split('.', 1)
  478. file_version = tuple([int(i) for i in wv])
  479. if (file_version != self.wheel_version) and warner:
  480. warner(self.wheel_version, file_version)
  481. if message['Root-Is-Purelib'] == 'true':
  482. libdir = paths['purelib']
  483. else:
  484. libdir = paths['platlib']
  485. records = {}
  486. with zf.open(record_name) as bf:
  487. with CSVReader(stream=bf) as reader:
  488. for row in reader:
  489. p = row[0]
  490. records[p] = row
  491. data_pfx = posixpath.join(data_dir, '')
  492. info_pfx = posixpath.join(info_dir, '')
  493. script_pfx = posixpath.join(data_dir, 'scripts', '')
  494. # make a new instance rather than a copy of maker's,
  495. # as we mutate it
  496. fileop = FileOperator(dry_run=dry_run)
  497. fileop.record = True # so we can rollback if needed
  498. bc = not sys.dont_write_bytecode # Double negatives. Lovely!
  499. outfiles = [] # for RECORD writing
  500. # for script copying/shebang processing
  501. workdir = tempfile.mkdtemp()
  502. # set target dir later
  503. # we default add_launchers to False, as the
  504. # Python Launcher should be used instead
  505. maker.source_dir = workdir
  506. maker.target_dir = None
  507. try:
  508. for zinfo in zf.infolist():
  509. arcname = zinfo.filename
  510. if isinstance(arcname, text_type):
  511. u_arcname = arcname
  512. else:
  513. u_arcname = arcname.decode('utf-8')
  514. if self.skip_entry(u_arcname):
  515. continue
  516. row = records[u_arcname]
  517. if row[2] and str(zinfo.file_size) != row[2]:
  518. raise DistlibException('size mismatch for '
  519. '%s' % u_arcname)
  520. if row[1]:
  521. kind, value = row[1].split('=', 1)
  522. with zf.open(arcname) as bf:
  523. data = bf.read()
  524. _, digest = self.get_hash(data, kind)
  525. if digest != value:
  526. raise DistlibException('digest mismatch for '
  527. '%s' % arcname)
  528. if lib_only and u_arcname.startswith((info_pfx, data_pfx)):
  529. logger.debug('lib_only: skipping %s', u_arcname)
  530. continue
  531. is_script = (u_arcname.startswith(script_pfx) and not u_arcname.endswith('.exe'))
  532. if u_arcname.startswith(data_pfx):
  533. _, where, rp = u_arcname.split('/', 2)
  534. outfile = os.path.join(paths[where], convert_path(rp))
  535. else:
  536. # meant for site-packages.
  537. if u_arcname in (wheel_metadata_name, record_name):
  538. continue
  539. outfile = os.path.join(libdir, convert_path(u_arcname))
  540. if not is_script:
  541. with zf.open(arcname) as bf:
  542. fileop.copy_stream(bf, outfile)
  543. # Issue #147: permission bits aren't preserved. Using
  544. # zf.extract(zinfo, libdir) should have worked, but didn't,
  545. # see https://www.thetopsites.net/article/53834422.shtml
  546. # So ... manually preserve permission bits as given in zinfo
  547. if os.name == 'posix':
  548. # just set the normal permission bits
  549. os.chmod(outfile, (zinfo.external_attr >> 16) & 0x1FF)
  550. outfiles.append(outfile)
  551. # Double check the digest of the written file
  552. if not dry_run and row[1]:
  553. with open(outfile, 'rb') as bf:
  554. data = bf.read()
  555. _, newdigest = self.get_hash(data, kind)
  556. if newdigest != digest:
  557. raise DistlibException('digest mismatch '
  558. 'on write for '
  559. '%s' % outfile)
  560. if bc and outfile.endswith('.py'):
  561. try:
  562. pyc = fileop.byte_compile(outfile, hashed_invalidation=bc_hashed_invalidation)
  563. outfiles.append(pyc)
  564. except Exception:
  565. # Don't give up if byte-compilation fails,
  566. # but log it and perhaps warn the user
  567. logger.warning('Byte-compilation failed', exc_info=True)
  568. else:
  569. fn = os.path.basename(convert_path(arcname))
  570. workname = os.path.join(workdir, fn)
  571. with zf.open(arcname) as bf:
  572. fileop.copy_stream(bf, workname)
  573. dn, fn = os.path.split(outfile)
  574. maker.target_dir = dn
  575. filenames = maker.make(fn)
  576. fileop.set_executable_mode(filenames)
  577. outfiles.extend(filenames)
  578. if lib_only:
  579. logger.debug('lib_only: returning None')
  580. dist = None
  581. else:
  582. # Generate scripts
  583. # Try to get pydist.json so we can see if there are
  584. # any commands to generate. If this fails (e.g. because
  585. # of a legacy wheel), log a warning but don't give up.
  586. commands = None
  587. file_version = self.info['Wheel-Version']
  588. if file_version == '1.0':
  589. # Use legacy info
  590. ep = posixpath.join(info_dir, 'entry_points.txt')
  591. try:
  592. with zf.open(ep) as bwf:
  593. epdata = read_exports(bwf)
  594. commands = {}
  595. for key in ('console', 'gui'):
  596. k = '%s_scripts' % key
  597. if k in epdata:
  598. commands['wrap_%s' % key] = d = {}
  599. for v in epdata[k].values():
  600. s = '%s:%s' % (v.prefix, v.suffix)
  601. if v.flags:
  602. s += ' [%s]' % ','.join(v.flags)
  603. d[v.name] = s
  604. except Exception:
  605. logger.warning('Unable to read legacy script '
  606. 'metadata, so cannot generate '
  607. 'scripts')
  608. else:
  609. try:
  610. with zf.open(metadata_name) as bwf:
  611. wf = wrapper(bwf)
  612. commands = json.load(wf).get('extensions')
  613. if commands:
  614. commands = commands.get('python.commands')
  615. except Exception:
  616. logger.warning('Unable to read JSON metadata, so '
  617. 'cannot generate scripts')
  618. if commands:
  619. console_scripts = commands.get('wrap_console', {})
  620. gui_scripts = commands.get('wrap_gui', {})
  621. if console_scripts or gui_scripts:
  622. script_dir = paths.get('scripts', '')
  623. if not os.path.isdir(script_dir):
  624. raise ValueError('Valid script path not '
  625. 'specified')
  626. maker.target_dir = script_dir
  627. for k, v in console_scripts.items():
  628. script = '%s = %s' % (k, v)
  629. filenames = maker.make(script)
  630. fileop.set_executable_mode(filenames)
  631. if gui_scripts:
  632. options = {'gui': True}
  633. for k, v in gui_scripts.items():
  634. script = '%s = %s' % (k, v)
  635. filenames = maker.make(script, options)
  636. fileop.set_executable_mode(filenames)
  637. p = os.path.join(libdir, info_dir)
  638. dist = InstalledDistribution(p)
  639. # Write SHARED
  640. paths = dict(paths) # don't change passed in dict
  641. del paths['purelib']
  642. del paths['platlib']
  643. paths['lib'] = libdir
  644. p = dist.write_shared_locations(paths, dry_run)
  645. if p:
  646. outfiles.append(p)
  647. # Write RECORD
  648. dist.write_installed_files(outfiles, paths['prefix'], dry_run)
  649. return dist
  650. except Exception: # pragma: no cover
  651. logger.exception('installation failed.')
  652. fileop.rollback()
  653. raise
  654. finally:
  655. shutil.rmtree(workdir)
  656. def _get_dylib_cache(self):
  657. global cache
  658. if cache is None:
  659. # Use native string to avoid issues on 2.x: see Python #20140.
  660. base = os.path.join(get_cache_base(), str('dylib-cache'), '%s.%s' % sys.version_info[:2])
  661. cache = Cache(base)
  662. return cache
  663. def _get_extensions(self):
  664. pathname = os.path.join(self.dirname, self.filename)
  665. name_ver = '%s-%s' % (self.name, self.version)
  666. info_dir = '%s.dist-info' % name_ver
  667. arcname = posixpath.join(info_dir, 'EXTENSIONS')
  668. wrapper = codecs.getreader('utf-8')
  669. result = []
  670. with ZipFile(pathname, 'r') as zf:
  671. try:
  672. with zf.open(arcname) as bf:
  673. wf = wrapper(bf)
  674. extensions = json.load(wf)
  675. cache = self._get_dylib_cache()
  676. prefix = cache.prefix_to_dir(self.filename, use_abspath=False)
  677. cache_base = os.path.join(cache.base, prefix)
  678. if not os.path.isdir(cache_base):
  679. os.makedirs(cache_base)
  680. for name, relpath in extensions.items():
  681. dest = os.path.join(cache_base, convert_path(relpath))
  682. if not os.path.exists(dest):
  683. extract = True
  684. else:
  685. file_time = os.stat(dest).st_mtime
  686. file_time = datetime.datetime.fromtimestamp(file_time)
  687. info = zf.getinfo(relpath)
  688. wheel_time = datetime.datetime(*info.date_time)
  689. extract = wheel_time > file_time
  690. if extract:
  691. zf.extract(relpath, cache_base)
  692. result.append((name, dest))
  693. except KeyError:
  694. pass
  695. return result
  696. def is_compatible(self):
  697. """
  698. Determine if a wheel is compatible with the running system.
  699. """
  700. return is_compatible(self)
  701. def is_mountable(self):
  702. """
  703. Determine if a wheel is asserted as mountable by its metadata.
  704. """
  705. return True # for now - metadata details TBD
  706. def mount(self, append=False):
  707. pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
  708. if not self.is_compatible():
  709. msg = 'Wheel %s not compatible with this Python.' % pathname
  710. raise DistlibException(msg)
  711. if not self.is_mountable():
  712. msg = 'Wheel %s is marked as not mountable.' % pathname
  713. raise DistlibException(msg)
  714. if pathname in sys.path:
  715. logger.debug('%s already in path', pathname)
  716. else:
  717. if append:
  718. sys.path.append(pathname)
  719. else:
  720. sys.path.insert(0, pathname)
  721. extensions = self._get_extensions()
  722. if extensions:
  723. if _hook not in sys.meta_path:
  724. sys.meta_path.append(_hook)
  725. _hook.add(pathname, extensions)
  726. def unmount(self):
  727. pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
  728. if pathname not in sys.path:
  729. logger.debug('%s not in path', pathname)
  730. else:
  731. sys.path.remove(pathname)
  732. if pathname in _hook.impure_wheels:
  733. _hook.remove(pathname)
  734. if not _hook.impure_wheels:
  735. if _hook in sys.meta_path:
  736. sys.meta_path.remove(_hook)
  737. def verify(self):
  738. pathname = os.path.join(self.dirname, self.filename)
  739. name_ver = '%s-%s' % (self.name, self.version)
  740. # data_dir = '%s.data' % name_ver
  741. info_dir = '%s.dist-info' % name_ver
  742. # metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)
  743. wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
  744. record_name = posixpath.join(info_dir, 'RECORD')
  745. wrapper = codecs.getreader('utf-8')
  746. with ZipFile(pathname, 'r') as zf:
  747. with zf.open(wheel_metadata_name) as bwf:
  748. wf = wrapper(bwf)
  749. message_from_file(wf)
  750. # wv = message['Wheel-Version'].split('.', 1)
  751. # file_version = tuple([int(i) for i in wv])
  752. # TODO version verification
  753. records = {}
  754. with zf.open(record_name) as bf:
  755. with CSVReader(stream=bf) as reader:
  756. for row in reader:
  757. p = row[0]
  758. records[p] = row
  759. for zinfo in zf.infolist():
  760. arcname = zinfo.filename
  761. if isinstance(arcname, text_type):
  762. u_arcname = arcname
  763. else:
  764. u_arcname = arcname.decode('utf-8')
  765. # See issue #115: some wheels have .. in their entries, but
  766. # in the filename ... e.g. __main__..py ! So the check is
  767. # updated to look for .. in the directory portions
  768. p = u_arcname.split('/')
  769. if '..' in p:
  770. raise DistlibException('invalid entry in '
  771. 'wheel: %r' % u_arcname)
  772. if self.skip_entry(u_arcname):
  773. continue
  774. row = records[u_arcname]
  775. if row[2] and str(zinfo.file_size) != row[2]:
  776. raise DistlibException('size mismatch for '
  777. '%s' % u_arcname)
  778. if row[1]:
  779. kind, value = row[1].split('=', 1)
  780. with zf.open(arcname) as bf:
  781. data = bf.read()
  782. _, digest = self.get_hash(data, kind)
  783. if digest != value:
  784. raise DistlibException('digest mismatch for '
  785. '%s' % arcname)
  786. def update(self, modifier, dest_dir=None, **kwargs):
  787. """
  788. Update the contents of a wheel in a generic way. The modifier should
  789. be a callable which expects a dictionary argument: its keys are
  790. archive-entry paths, and its values are absolute filesystem paths
  791. where the contents the corresponding archive entries can be found. The
  792. modifier is free to change the contents of the files pointed to, add
  793. new entries and remove entries, before returning. This method will
  794. extract the entire contents of the wheel to a temporary location, call
  795. the modifier, and then use the passed (and possibly updated)
  796. dictionary to write a new wheel. If ``dest_dir`` is specified, the new
  797. wheel is written there -- otherwise, the original wheel is overwritten.
  798. The modifier should return True if it updated the wheel, else False.
  799. This method returns the same value the modifier returns.
  800. """
  801. def get_version(path_map, info_dir):
  802. version = path = None
  803. key = '%s/%s' % (info_dir, LEGACY_METADATA_FILENAME)
  804. if key not in path_map:
  805. key = '%s/PKG-INFO' % info_dir
  806. if key in path_map:
  807. path = path_map[key]
  808. version = Metadata(path=path).version
  809. return version, path
  810. def update_version(version, path):
  811. updated = None
  812. try:
  813. NormalizedVersion(version)
  814. i = version.find('-')
  815. if i < 0:
  816. updated = '%s+1' % version
  817. else:
  818. parts = [int(s) for s in version[i + 1:].split('.')]
  819. parts[-1] += 1
  820. updated = '%s+%s' % (version[:i], '.'.join(str(i) for i in parts))
  821. except UnsupportedVersionError:
  822. logger.debug('Cannot update non-compliant (PEP-440) '
  823. 'version %r', version)
  824. if updated:
  825. md = Metadata(path=path)
  826. md.version = updated
  827. legacy = path.endswith(LEGACY_METADATA_FILENAME)
  828. md.write(path=path, legacy=legacy)
  829. logger.debug('Version updated from %r to %r', version, updated)
  830. pathname = os.path.join(self.dirname, self.filename)
  831. name_ver = '%s-%s' % (self.name, self.version)
  832. info_dir = '%s.dist-info' % name_ver
  833. record_name = posixpath.join(info_dir, 'RECORD')
  834. with tempdir() as workdir:
  835. with ZipFile(pathname, 'r') as zf:
  836. path_map = {}
  837. for zinfo in zf.infolist():
  838. arcname = zinfo.filename
  839. if isinstance(arcname, text_type):
  840. u_arcname = arcname
  841. else:
  842. u_arcname = arcname.decode('utf-8')
  843. if u_arcname == record_name:
  844. continue
  845. if '..' in u_arcname:
  846. raise DistlibException('invalid entry in '
  847. 'wheel: %r' % u_arcname)
  848. zf.extract(zinfo, workdir)
  849. path = os.path.join(workdir, convert_path(u_arcname))
  850. path_map[u_arcname] = path
  851. # Remember the version.
  852. original_version, _ = get_version(path_map, info_dir)
  853. # Files extracted. Call the modifier.
  854. modified = modifier(path_map, **kwargs)
  855. if modified:
  856. # Something changed - need to build a new wheel.
  857. current_version, path = get_version(path_map, info_dir)
  858. if current_version and (current_version == original_version):
  859. # Add or update local version to signify changes.
  860. update_version(current_version, path)
  861. # Decide where the new wheel goes.
  862. if dest_dir is None:
  863. fd, newpath = tempfile.mkstemp(suffix='.whl', prefix='wheel-update-', dir=workdir)
  864. os.close(fd)
  865. else:
  866. if not os.path.isdir(dest_dir):
  867. raise DistlibException('Not a directory: %r' % dest_dir)
  868. newpath = os.path.join(dest_dir, self.filename)
  869. archive_paths = list(path_map.items())
  870. distinfo = os.path.join(workdir, info_dir)
  871. info = distinfo, info_dir
  872. self.write_records(info, workdir, archive_paths)
  873. self.build_zip(newpath, archive_paths)
  874. if dest_dir is None:
  875. shutil.copyfile(newpath, pathname)
  876. return modified
  877. def _get_glibc_version():
  878. import platform
  879. ver = platform.libc_ver()
  880. result = []
  881. if ver[0] == 'glibc':
  882. for s in ver[1].split('.'):
  883. result.append(int(s) if s.isdigit() else 0)
  884. result = tuple(result)
  885. return result
  886. def compatible_tags():
  887. """
  888. Return (pyver, abi, arch) tuples compatible with this Python.
  889. """
  890. class _Version:
  891. def __init__(self, major, minor):
  892. self.major = major
  893. self.major_minor = (major, minor)
  894. self.string = ''.join((str(major), str(minor)))
  895. def __str__(self):
  896. return self.string
  897. versions = [
  898. _Version(sys.version_info.major, minor_version)
  899. for minor_version in range(sys.version_info.minor, -1, -1)
  900. ]
  901. abis = []
  902. for suffix in _get_suffixes():
  903. if suffix.startswith('.abi'):
  904. abis.append(suffix.split('.', 2)[1])
  905. abis.sort()
  906. if ABI != 'none':
  907. abis.insert(0, ABI)
  908. abis.append('none')
  909. result = []
  910. arches = [ARCH]
  911. if sys.platform == 'darwin':
  912. m = re.match(r'(\w+)_(\d+)_(\d+)_(\w+)$', ARCH)
  913. if m:
  914. name, major, minor, arch = m.groups()
  915. minor = int(minor)
  916. matches = [arch]
  917. if arch in ('i386', 'ppc'):
  918. matches.append('fat')
  919. if arch in ('i386', 'ppc', 'x86_64'):
  920. matches.append('fat3')
  921. if arch in ('ppc64', 'x86_64'):
  922. matches.append('fat64')
  923. if arch in ('i386', 'x86_64'):
  924. matches.append('intel')
  925. if arch in ('i386', 'x86_64', 'intel', 'ppc', 'ppc64'):
  926. matches.append('universal')
  927. while minor >= 0:
  928. for match in matches:
  929. s = '%s_%s_%s_%s' % (name, major, minor, match)
  930. if s != ARCH: # already there
  931. arches.append(s)
  932. minor -= 1
  933. # Most specific - our Python version, ABI and arch
  934. for i, version_object in enumerate(versions):
  935. version = str(version_object)
  936. add_abis = []
  937. if i == 0:
  938. add_abis = abis
  939. if IMP_PREFIX == 'cp' and version_object.major_minor >= (3, 2):
  940. limited_api_abi = 'abi' + str(version_object.major)
  941. if limited_api_abi not in add_abis:
  942. add_abis.append(limited_api_abi)
  943. for abi in add_abis:
  944. for arch in arches:
  945. result.append((''.join((IMP_PREFIX, version)), abi, arch))
  946. # manylinux
  947. if abi != 'none' and sys.platform.startswith('linux'):
  948. arch = arch.replace('linux_', '')
  949. parts = _get_glibc_version()
  950. if len(parts) == 2:
  951. if parts >= (2, 5):
  952. result.append((''.join((IMP_PREFIX, version)), abi, 'manylinux1_%s' % arch))
  953. if parts >= (2, 12):
  954. result.append((''.join((IMP_PREFIX, version)), abi, 'manylinux2010_%s' % arch))
  955. if parts >= (2, 17):
  956. result.append((''.join((IMP_PREFIX, version)), abi, 'manylinux2014_%s' % arch))
  957. result.append((''.join(
  958. (IMP_PREFIX, version)), abi, 'manylinux_%s_%s_%s' % (parts[0], parts[1], arch)))
  959. # where no ABI / arch dependency, but IMP_PREFIX dependency
  960. for i, version_object in enumerate(versions):
  961. version = str(version_object)
  962. result.append((''.join((IMP_PREFIX, version)), 'none', 'any'))
  963. if i == 0:
  964. result.append((''.join((IMP_PREFIX, version[0])), 'none', 'any'))
  965. # no IMP_PREFIX, ABI or arch dependency
  966. for i, version_object in enumerate(versions):
  967. version = str(version_object)
  968. result.append((''.join(('py', version)), 'none', 'any'))
  969. if i == 0:
  970. result.append((''.join(('py', version[0])), 'none', 'any'))
  971. return set(result)
  972. COMPATIBLE_TAGS = compatible_tags()
  973. del compatible_tags
  974. def is_compatible(wheel, tags=None):
  975. if not isinstance(wheel, Wheel):
  976. wheel = Wheel(wheel) # assume it's a filename
  977. result = False
  978. if tags is None:
  979. tags = COMPATIBLE_TAGS
  980. for ver, abi, arch in tags:
  981. if ver in wheel.pyver and abi in wheel.abi and arch in wheel.arch:
  982. result = True
  983. break
  984. return result