macosx_libfile.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486
  1. """
  2. IMPORTANT: DO NOT IMPORT THIS MODULE DIRECTLY.
  3. THIS IS ONLY KEPT IN PLACE FOR BACKWARDS COMPATIBILITY WITH
  4. setuptools.command.bdist_wheel.
  5. This module contains function to analyse dynamic library
  6. headers to extract system information
  7. Currently only for MacOSX
  8. Library file on macosx system starts with Mach-O or Fat field.
  9. This can be distinguish by first 32 bites and it is called magic number.
  10. Proper value of magic number is with suffix _MAGIC. Suffix _CIGAM means
  11. reversed bytes order.
  12. Both fields can occur in two types: 32 and 64 bytes.
  13. FAT field inform that this library contains few version of library
  14. (typically for different types version). It contains
  15. information where Mach-O headers starts.
  16. Each section started with Mach-O header contains one library
  17. (So if file starts with this field it contains only one version).
  18. After filed Mach-O there are section fields.
  19. Each of them starts with two fields:
  20. cmd - magic number for this command
  21. cmdsize - total size occupied by this section information.
  22. In this case only sections LC_VERSION_MIN_MACOSX (for macosx 10.13 and earlier)
  23. and LC_BUILD_VERSION (for macosx 10.14 and newer) are interesting,
  24. because them contains information about minimal system version.
  25. Important remarks:
  26. - For fat files this implementation looks for maximum number version.
  27. It not check if it is 32 or 64 and do not compare it with currently built package.
  28. So it is possible to false report higher version that needed.
  29. - All structures signatures are taken form macosx header files.
  30. - I think that binary format will be more stable than `otool` output.
  31. and if apple introduce some changes both implementation will need to be updated.
  32. - The system compile will set the deployment target no lower than
  33. 11.0 for arm64 builds. For "Universal 2" builds use the x86_64 deployment
  34. target when the arm64 target is 11.0.
  35. """
  36. from __future__ import annotations
  37. import ctypes
  38. import os
  39. import sys
  40. from io import BufferedIOBase
  41. from typing import TYPE_CHECKING
  42. if TYPE_CHECKING:
  43. from typing import Union
  44. StrPath = Union[str, os.PathLike[str]]
  45. """here the needed const and struct from mach-o header files"""
  46. FAT_MAGIC = 0xCAFEBABE
  47. FAT_CIGAM = 0xBEBAFECA
  48. FAT_MAGIC_64 = 0xCAFEBABF
  49. FAT_CIGAM_64 = 0xBFBAFECA
  50. MH_MAGIC = 0xFEEDFACE
  51. MH_CIGAM = 0xCEFAEDFE
  52. MH_MAGIC_64 = 0xFEEDFACF
  53. MH_CIGAM_64 = 0xCFFAEDFE
  54. LC_VERSION_MIN_MACOSX = 0x24
  55. LC_BUILD_VERSION = 0x32
  56. CPU_TYPE_ARM64 = 0x0100000C
  57. mach_header_fields = [
  58. ("magic", ctypes.c_uint32),
  59. ("cputype", ctypes.c_int),
  60. ("cpusubtype", ctypes.c_int),
  61. ("filetype", ctypes.c_uint32),
  62. ("ncmds", ctypes.c_uint32),
  63. ("sizeofcmds", ctypes.c_uint32),
  64. ("flags", ctypes.c_uint32),
  65. ]
  66. """
  67. struct mach_header {
  68. uint32_t magic; /* mach magic number identifier */
  69. cpu_type_t cputype; /* cpu specifier */
  70. cpu_subtype_t cpusubtype; /* machine specifier */
  71. uint32_t filetype; /* type of file */
  72. uint32_t ncmds; /* number of load commands */
  73. uint32_t sizeofcmds; /* the size of all the load commands */
  74. uint32_t flags; /* flags */
  75. };
  76. typedef integer_t cpu_type_t;
  77. typedef integer_t cpu_subtype_t;
  78. """
  79. mach_header_fields_64 = mach_header_fields + [("reserved", ctypes.c_uint32)]
  80. """
  81. struct mach_header_64 {
  82. uint32_t magic; /* mach magic number identifier */
  83. cpu_type_t cputype; /* cpu specifier */
  84. cpu_subtype_t cpusubtype; /* machine specifier */
  85. uint32_t filetype; /* type of file */
  86. uint32_t ncmds; /* number of load commands */
  87. uint32_t sizeofcmds; /* the size of all the load commands */
  88. uint32_t flags; /* flags */
  89. uint32_t reserved; /* reserved */
  90. };
  91. """
  92. fat_header_fields = [("magic", ctypes.c_uint32), ("nfat_arch", ctypes.c_uint32)]
  93. """
  94. struct fat_header {
  95. uint32_t magic; /* FAT_MAGIC or FAT_MAGIC_64 */
  96. uint32_t nfat_arch; /* number of structs that follow */
  97. };
  98. """
  99. fat_arch_fields = [
  100. ("cputype", ctypes.c_int),
  101. ("cpusubtype", ctypes.c_int),
  102. ("offset", ctypes.c_uint32),
  103. ("size", ctypes.c_uint32),
  104. ("align", ctypes.c_uint32),
  105. ]
  106. """
  107. struct fat_arch {
  108. cpu_type_t cputype; /* cpu specifier (int) */
  109. cpu_subtype_t cpusubtype; /* machine specifier (int) */
  110. uint32_t offset; /* file offset to this object file */
  111. uint32_t size; /* size of this object file */
  112. uint32_t align; /* alignment as a power of 2 */
  113. };
  114. """
  115. fat_arch_64_fields = [
  116. ("cputype", ctypes.c_int),
  117. ("cpusubtype", ctypes.c_int),
  118. ("offset", ctypes.c_uint64),
  119. ("size", ctypes.c_uint64),
  120. ("align", ctypes.c_uint32),
  121. ("reserved", ctypes.c_uint32),
  122. ]
  123. """
  124. struct fat_arch_64 {
  125. cpu_type_t cputype; /* cpu specifier (int) */
  126. cpu_subtype_t cpusubtype; /* machine specifier (int) */
  127. uint64_t offset; /* file offset to this object file */
  128. uint64_t size; /* size of this object file */
  129. uint32_t align; /* alignment as a power of 2 */
  130. uint32_t reserved; /* reserved */
  131. };
  132. """
  133. segment_base_fields = [("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32)]
  134. """base for reading segment info"""
  135. segment_command_fields = [
  136. ("cmd", ctypes.c_uint32),
  137. ("cmdsize", ctypes.c_uint32),
  138. ("segname", ctypes.c_char * 16),
  139. ("vmaddr", ctypes.c_uint32),
  140. ("vmsize", ctypes.c_uint32),
  141. ("fileoff", ctypes.c_uint32),
  142. ("filesize", ctypes.c_uint32),
  143. ("maxprot", ctypes.c_int),
  144. ("initprot", ctypes.c_int),
  145. ("nsects", ctypes.c_uint32),
  146. ("flags", ctypes.c_uint32),
  147. ]
  148. """
  149. struct segment_command { /* for 32-bit architectures */
  150. uint32_t cmd; /* LC_SEGMENT */
  151. uint32_t cmdsize; /* includes sizeof section structs */
  152. char segname[16]; /* segment name */
  153. uint32_t vmaddr; /* memory address of this segment */
  154. uint32_t vmsize; /* memory size of this segment */
  155. uint32_t fileoff; /* file offset of this segment */
  156. uint32_t filesize; /* amount to map from the file */
  157. vm_prot_t maxprot; /* maximum VM protection */
  158. vm_prot_t initprot; /* initial VM protection */
  159. uint32_t nsects; /* number of sections in segment */
  160. uint32_t flags; /* flags */
  161. };
  162. typedef int vm_prot_t;
  163. """
  164. segment_command_fields_64 = [
  165. ("cmd", ctypes.c_uint32),
  166. ("cmdsize", ctypes.c_uint32),
  167. ("segname", ctypes.c_char * 16),
  168. ("vmaddr", ctypes.c_uint64),
  169. ("vmsize", ctypes.c_uint64),
  170. ("fileoff", ctypes.c_uint64),
  171. ("filesize", ctypes.c_uint64),
  172. ("maxprot", ctypes.c_int),
  173. ("initprot", ctypes.c_int),
  174. ("nsects", ctypes.c_uint32),
  175. ("flags", ctypes.c_uint32),
  176. ]
  177. """
  178. struct segment_command_64 { /* for 64-bit architectures */
  179. uint32_t cmd; /* LC_SEGMENT_64 */
  180. uint32_t cmdsize; /* includes sizeof section_64 structs */
  181. char segname[16]; /* segment name */
  182. uint64_t vmaddr; /* memory address of this segment */
  183. uint64_t vmsize; /* memory size of this segment */
  184. uint64_t fileoff; /* file offset of this segment */
  185. uint64_t filesize; /* amount to map from the file */
  186. vm_prot_t maxprot; /* maximum VM protection */
  187. vm_prot_t initprot; /* initial VM protection */
  188. uint32_t nsects; /* number of sections in segment */
  189. uint32_t flags; /* flags */
  190. };
  191. """
  192. version_min_command_fields = segment_base_fields + [
  193. ("version", ctypes.c_uint32),
  194. ("sdk", ctypes.c_uint32),
  195. ]
  196. """
  197. struct version_min_command {
  198. uint32_t cmd; /* LC_VERSION_MIN_MACOSX or
  199. LC_VERSION_MIN_IPHONEOS or
  200. LC_VERSION_MIN_WATCHOS or
  201. LC_VERSION_MIN_TVOS */
  202. uint32_t cmdsize; /* sizeof(struct min_version_command) */
  203. uint32_t version; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */
  204. uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */
  205. };
  206. """
  207. build_version_command_fields = segment_base_fields + [
  208. ("platform", ctypes.c_uint32),
  209. ("minos", ctypes.c_uint32),
  210. ("sdk", ctypes.c_uint32),
  211. ("ntools", ctypes.c_uint32),
  212. ]
  213. """
  214. struct build_version_command {
  215. uint32_t cmd; /* LC_BUILD_VERSION */
  216. uint32_t cmdsize; /* sizeof(struct build_version_command) plus */
  217. /* ntools * sizeof(struct build_tool_version) */
  218. uint32_t platform; /* platform */
  219. uint32_t minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */
  220. uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */
  221. uint32_t ntools; /* number of tool entries following this */
  222. };
  223. """
  224. def swap32(x: int) -> int:
  225. return (
  226. ((x << 24) & 0xFF000000)
  227. | ((x << 8) & 0x00FF0000)
  228. | ((x >> 8) & 0x0000FF00)
  229. | ((x >> 24) & 0x000000FF)
  230. )
  231. def get_base_class_and_magic_number(
  232. lib_file: BufferedIOBase,
  233. seek: int | None = None,
  234. ) -> tuple[type[ctypes.Structure], int]:
  235. if seek is None:
  236. seek = lib_file.tell()
  237. else:
  238. lib_file.seek(seek)
  239. magic_number = ctypes.c_uint32.from_buffer_copy(
  240. lib_file.read(ctypes.sizeof(ctypes.c_uint32))
  241. ).value
  242. # Handle wrong byte order
  243. if magic_number in [FAT_CIGAM, FAT_CIGAM_64, MH_CIGAM, MH_CIGAM_64]:
  244. if sys.byteorder == "little":
  245. BaseClass = ctypes.BigEndianStructure
  246. else:
  247. BaseClass = ctypes.LittleEndianStructure
  248. magic_number = swap32(magic_number)
  249. else:
  250. BaseClass = ctypes.Structure
  251. lib_file.seek(seek)
  252. return BaseClass, magic_number
  253. def read_data(struct_class: type[ctypes.Structure], lib_file: BufferedIOBase):
  254. return struct_class.from_buffer_copy(lib_file.read(ctypes.sizeof(struct_class)))
  255. def extract_macosx_min_system_version(path_to_lib: str):
  256. with open(path_to_lib, "rb") as lib_file:
  257. BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0)
  258. if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]:
  259. return
  260. if magic_number in [FAT_MAGIC, FAT_CIGAM_64]:
  261. class FatHeader(BaseClass):
  262. _fields_ = fat_header_fields
  263. fat_header = read_data(FatHeader, lib_file)
  264. if magic_number == FAT_MAGIC:
  265. class FatArch(BaseClass):
  266. _fields_ = fat_arch_fields
  267. else:
  268. class FatArch(BaseClass):
  269. _fields_ = fat_arch_64_fields
  270. fat_arch_list = [
  271. read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch)
  272. ]
  273. versions_list: list[tuple[int, int, int]] = []
  274. for el in fat_arch_list:
  275. try:
  276. version = read_mach_header(lib_file, el.offset)
  277. if version is not None:
  278. if el.cputype == CPU_TYPE_ARM64 and len(fat_arch_list) != 1:
  279. # Xcode will not set the deployment target below 11.0.0
  280. # for the arm64 architecture. Ignore the arm64 deployment
  281. # in fat binaries when the target is 11.0.0, that way
  282. # the other architectures can select a lower deployment
  283. # target.
  284. # This is safe because there is no arm64 variant for
  285. # macOS 10.15 or earlier.
  286. if version == (11, 0, 0):
  287. continue
  288. versions_list.append(version)
  289. except ValueError:
  290. pass
  291. if len(versions_list) > 0:
  292. return max(versions_list)
  293. else:
  294. return None
  295. else:
  296. try:
  297. return read_mach_header(lib_file, 0)
  298. except ValueError:
  299. """when some error during read library files"""
  300. return None
  301. def read_mach_header(
  302. lib_file: BufferedIOBase,
  303. seek: int | None = None,
  304. ) -> tuple[int, int, int] | None:
  305. """
  306. This function parses a Mach-O header and extracts
  307. information about the minimal macOS version.
  308. :param lib_file: reference to opened library file with pointer
  309. """
  310. base_class, magic_number = get_base_class_and_magic_number(lib_file, seek)
  311. arch = "32" if magic_number == MH_MAGIC else "64"
  312. class SegmentBase(base_class):
  313. _fields_ = segment_base_fields
  314. if arch == "32":
  315. class MachHeader(base_class):
  316. _fields_ = mach_header_fields
  317. else:
  318. class MachHeader(base_class):
  319. _fields_ = mach_header_fields_64
  320. mach_header = read_data(MachHeader, lib_file)
  321. for _i in range(mach_header.ncmds):
  322. pos = lib_file.tell()
  323. segment_base = read_data(SegmentBase, lib_file)
  324. lib_file.seek(pos)
  325. if segment_base.cmd == LC_VERSION_MIN_MACOSX:
  326. class VersionMinCommand(base_class):
  327. _fields_ = version_min_command_fields
  328. version_info = read_data(VersionMinCommand, lib_file)
  329. return parse_version(version_info.version)
  330. elif segment_base.cmd == LC_BUILD_VERSION:
  331. class VersionBuild(base_class):
  332. _fields_ = build_version_command_fields
  333. version_info = read_data(VersionBuild, lib_file)
  334. return parse_version(version_info.minos)
  335. else:
  336. lib_file.seek(pos + segment_base.cmdsize)
  337. continue
  338. def parse_version(version: int) -> tuple[int, int, int]:
  339. x = (version & 0xFFFF0000) >> 16
  340. y = (version & 0x0000FF00) >> 8
  341. z = version & 0x000000FF
  342. return x, y, z
  343. def calculate_macosx_platform_tag(archive_root: StrPath, platform_tag: str) -> str:
  344. """
  345. Calculate proper macosx platform tag basing on files which are included to wheel
  346. Example platform tag `macosx-10.14-x86_64`
  347. """
  348. prefix, base_version, suffix = platform_tag.split("-")
  349. base_version = tuple(int(x) for x in base_version.split("."))
  350. base_version = base_version[:2]
  351. if base_version[0] > 10:
  352. base_version = (base_version[0], 0)
  353. assert len(base_version) == 2
  354. if "MACOSX_DEPLOYMENT_TARGET" in os.environ:
  355. deploy_target = tuple(
  356. int(x) for x in os.environ["MACOSX_DEPLOYMENT_TARGET"].split(".")
  357. )
  358. deploy_target = deploy_target[:2]
  359. if deploy_target[0] > 10:
  360. deploy_target = (deploy_target[0], 0)
  361. if deploy_target < base_version:
  362. sys.stderr.write(
  363. "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value ({}) than "
  364. "the version on which the Python interpreter was compiled ({}), and "
  365. "will be ignored.\n".format(
  366. ".".join(str(x) for x in deploy_target),
  367. ".".join(str(x) for x in base_version),
  368. )
  369. )
  370. else:
  371. base_version = deploy_target
  372. assert len(base_version) == 2
  373. start_version = base_version
  374. versions_dict: dict[str, tuple[int, int]] = {}
  375. for dirpath, _dirnames, filenames in os.walk(archive_root):
  376. for filename in filenames:
  377. if filename.endswith(".dylib") or filename.endswith(".so"):
  378. lib_path = os.path.join(dirpath, filename)
  379. min_ver = extract_macosx_min_system_version(lib_path)
  380. if min_ver is not None:
  381. min_ver = min_ver[0:2]
  382. if min_ver[0] > 10:
  383. min_ver = (min_ver[0], 0)
  384. versions_dict[lib_path] = min_ver
  385. if len(versions_dict) > 0:
  386. base_version = max(base_version, max(versions_dict.values()))
  387. # macosx platform tag do not support minor bugfix release
  388. fin_base_version = "_".join([str(x) for x in base_version])
  389. if start_version < base_version:
  390. problematic_files = [k for k, v in versions_dict.items() if v > start_version]
  391. problematic_files = "\n".join(problematic_files)
  392. if len(problematic_files) == 1:
  393. files_form = "this file"
  394. else:
  395. files_form = "these files"
  396. error_message = (
  397. "[WARNING] This wheel needs a higher macOS version than {} "
  398. "To silence this warning, set MACOSX_DEPLOYMENT_TARGET to at least "
  399. + fin_base_version
  400. + " or recreate "
  401. + files_form
  402. + " with lower "
  403. "MACOSX_DEPLOYMENT_TARGET: \n" + problematic_files
  404. )
  405. if "MACOSX_DEPLOYMENT_TARGET" in os.environ:
  406. error_message = error_message.format(
  407. "is set in MACOSX_DEPLOYMENT_TARGET variable."
  408. )
  409. else:
  410. error_message = error_message.format(
  411. "the version your Python interpreter is compiled against."
  412. )
  413. sys.stderr.write(error_message)
  414. platform_tag = prefix + "_" + fin_base_version + "_" + suffix
  415. return platform_tag