build_py.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404
  1. """distutils.command.build_py
  2. Implements the Distutils 'build_py' command."""
  3. import glob
  4. import importlib.util
  5. import os
  6. import sys
  7. from distutils._log import log
  8. from typing import ClassVar
  9. from ..core import Command
  10. from ..errors import DistutilsFileError, DistutilsOptionError
  11. from ..util import convert_path
  12. class build_py(Command):
  13. description = "\"build\" pure Python modules (copy to build directory)"
  14. user_options = [
  15. ('build-lib=', 'd', "directory to \"build\" (copy) to"),
  16. ('compile', 'c', "compile .py to .pyc"),
  17. ('no-compile', None, "don't compile .py files [default]"),
  18. (
  19. 'optimize=',
  20. 'O',
  21. "also compile with optimization: -O1 for \"python -O\", "
  22. "-O2 for \"python -OO\", and -O0 to disable [default: -O0]",
  23. ),
  24. ('force', 'f', "forcibly build everything (ignore file timestamps)"),
  25. ]
  26. boolean_options: ClassVar[list[str]] = ['compile', 'force']
  27. negative_opt: ClassVar[dict[str, str]] = {'no-compile': 'compile'}
  28. def initialize_options(self):
  29. self.build_lib = None
  30. self.py_modules = None
  31. self.package = None
  32. self.package_data = None
  33. self.package_dir = None
  34. self.compile = False
  35. self.optimize = 0
  36. self.force = None
  37. def finalize_options(self) -> None:
  38. self.set_undefined_options(
  39. 'build', ('build_lib', 'build_lib'), ('force', 'force')
  40. )
  41. # Get the distribution options that are aliases for build_py
  42. # options -- list of packages and list of modules.
  43. self.packages = self.distribution.packages
  44. self.py_modules = self.distribution.py_modules
  45. self.package_data = self.distribution.package_data
  46. self.package_dir = {}
  47. if self.distribution.package_dir:
  48. for name, path in self.distribution.package_dir.items():
  49. self.package_dir[name] = convert_path(path)
  50. self.data_files = self.get_data_files()
  51. # Ick, copied straight from install_lib.py (fancy_getopt needs a
  52. # type system! Hell, *everything* needs a type system!!!)
  53. if not isinstance(self.optimize, int):
  54. try:
  55. self.optimize = int(self.optimize)
  56. assert 0 <= self.optimize <= 2
  57. except (ValueError, AssertionError):
  58. raise DistutilsOptionError("optimize must be 0, 1, or 2")
  59. def run(self) -> None:
  60. # XXX copy_file by default preserves atime and mtime. IMHO this is
  61. # the right thing to do, but perhaps it should be an option -- in
  62. # particular, a site administrator might want installed files to
  63. # reflect the time of installation rather than the last
  64. # modification time before the installed release.
  65. # XXX copy_file by default preserves mode, which appears to be the
  66. # wrong thing to do: if a file is read-only in the working
  67. # directory, we want it to be installed read/write so that the next
  68. # installation of the same module distribution can overwrite it
  69. # without problems. (This might be a Unix-specific issue.) Thus
  70. # we turn off 'preserve_mode' when copying to the build directory,
  71. # since the build directory is supposed to be exactly what the
  72. # installation will look like (ie. we preserve mode when
  73. # installing).
  74. # Two options control which modules will be installed: 'packages'
  75. # and 'py_modules'. The former lets us work with whole packages, not
  76. # specifying individual modules at all; the latter is for
  77. # specifying modules one-at-a-time.
  78. if self.py_modules:
  79. self.build_modules()
  80. if self.packages:
  81. self.build_packages()
  82. self.build_package_data()
  83. self.byte_compile(self.get_outputs(include_bytecode=False))
  84. def get_data_files(self):
  85. """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
  86. data = []
  87. if not self.packages:
  88. return data
  89. for package in self.packages:
  90. # Locate package source directory
  91. src_dir = self.get_package_dir(package)
  92. # Compute package build directory
  93. build_dir = os.path.join(*([self.build_lib] + package.split('.')))
  94. # Length of path to strip from found files
  95. plen = 0
  96. if src_dir:
  97. plen = len(src_dir) + 1
  98. # Strip directory from globbed filenames
  99. filenames = [file[plen:] for file in self.find_data_files(package, src_dir)]
  100. data.append((package, src_dir, build_dir, filenames))
  101. return data
  102. def find_data_files(self, package, src_dir):
  103. """Return filenames for package's data files in 'src_dir'"""
  104. globs = self.package_data.get('', []) + self.package_data.get(package, [])
  105. files = []
  106. for pattern in globs:
  107. # Each pattern has to be converted to a platform-specific path
  108. filelist = glob.glob(
  109. os.path.join(glob.escape(src_dir), convert_path(pattern))
  110. )
  111. # Files that match more than one pattern are only added once
  112. files.extend([
  113. fn for fn in filelist if fn not in files and os.path.isfile(fn)
  114. ])
  115. return files
  116. def build_package_data(self) -> None:
  117. """Copy data files into build directory"""
  118. for _package, src_dir, build_dir, filenames in self.data_files:
  119. for filename in filenames:
  120. target = os.path.join(build_dir, filename)
  121. self.mkpath(os.path.dirname(target))
  122. self.copy_file(
  123. os.path.join(src_dir, filename), target, preserve_mode=False
  124. )
  125. def get_package_dir(self, package):
  126. """Return the directory, relative to the top of the source
  127. distribution, where package 'package' should be found
  128. (at least according to the 'package_dir' option, if any)."""
  129. path = package.split('.')
  130. if not self.package_dir:
  131. if path:
  132. return os.path.join(*path)
  133. else:
  134. return ''
  135. else:
  136. tail = []
  137. while path:
  138. try:
  139. pdir = self.package_dir['.'.join(path)]
  140. except KeyError:
  141. tail.insert(0, path[-1])
  142. del path[-1]
  143. else:
  144. tail.insert(0, pdir)
  145. return os.path.join(*tail)
  146. else:
  147. # Oops, got all the way through 'path' without finding a
  148. # match in package_dir. If package_dir defines a directory
  149. # for the root (nameless) package, then fallback on it;
  150. # otherwise, we might as well have not consulted
  151. # package_dir at all, as we just use the directory implied
  152. # by 'tail' (which should be the same as the original value
  153. # of 'path' at this point).
  154. pdir = self.package_dir.get('')
  155. if pdir is not None:
  156. tail.insert(0, pdir)
  157. if tail:
  158. return os.path.join(*tail)
  159. else:
  160. return ''
  161. def check_package(self, package, package_dir):
  162. # Empty dir name means current directory, which we can probably
  163. # assume exists. Also, os.path.exists and isdir don't know about
  164. # my "empty string means current dir" convention, so we have to
  165. # circumvent them.
  166. if package_dir != "":
  167. if not os.path.exists(package_dir):
  168. raise DistutilsFileError(
  169. f"package directory '{package_dir}' does not exist"
  170. )
  171. if not os.path.isdir(package_dir):
  172. raise DistutilsFileError(
  173. f"supposed package directory '{package_dir}' exists, "
  174. "but is not a directory"
  175. )
  176. # Directories without __init__.py are namespace packages (PEP 420).
  177. if package:
  178. init_py = os.path.join(package_dir, "__init__.py")
  179. if os.path.isfile(init_py):
  180. return init_py
  181. # Either not in a package at all (__init__.py not expected), or
  182. # __init__.py doesn't exist -- so don't return the filename.
  183. return None
  184. def check_module(self, module, module_file):
  185. if not os.path.isfile(module_file):
  186. log.warning("file %s (for module %s) not found", module_file, module)
  187. return False
  188. else:
  189. return True
  190. def find_package_modules(self, package, package_dir):
  191. self.check_package(package, package_dir)
  192. module_files = glob.glob(os.path.join(glob.escape(package_dir), "*.py"))
  193. modules = []
  194. setup_script = os.path.abspath(self.distribution.script_name)
  195. for f in module_files:
  196. abs_f = os.path.abspath(f)
  197. if abs_f != setup_script:
  198. module = os.path.splitext(os.path.basename(f))[0]
  199. modules.append((package, module, f))
  200. else:
  201. self.debug_print(f"excluding {setup_script}")
  202. return modules
  203. def find_modules(self):
  204. """Finds individually-specified Python modules, ie. those listed by
  205. module name in 'self.py_modules'. Returns a list of tuples (package,
  206. module_base, filename): 'package' is a tuple of the path through
  207. package-space to the module; 'module_base' is the bare (no
  208. packages, no dots) module name, and 'filename' is the path to the
  209. ".py" file (relative to the distribution root) that implements the
  210. module.
  211. """
  212. # Map package names to tuples of useful info about the package:
  213. # (package_dir, checked)
  214. # package_dir - the directory where we'll find source files for
  215. # this package
  216. # checked - true if we have checked that the package directory
  217. # is valid (exists, contains __init__.py, ... ?)
  218. packages = {}
  219. # List of (package, module, filename) tuples to return
  220. modules = []
  221. # We treat modules-in-packages almost the same as toplevel modules,
  222. # just the "package" for a toplevel is empty (either an empty
  223. # string or empty list, depending on context). Differences:
  224. # - don't check for __init__.py in directory for empty package
  225. for module in self.py_modules:
  226. path = module.split('.')
  227. package = '.'.join(path[0:-1])
  228. module_base = path[-1]
  229. try:
  230. (package_dir, checked) = packages[package]
  231. except KeyError:
  232. package_dir = self.get_package_dir(package)
  233. checked = False
  234. if not checked:
  235. init_py = self.check_package(package, package_dir)
  236. packages[package] = (package_dir, 1)
  237. if init_py:
  238. modules.append((package, "__init__", init_py))
  239. # XXX perhaps we should also check for just .pyc files
  240. # (so greedy closed-source bastards can distribute Python
  241. # modules too)
  242. module_file = os.path.join(package_dir, module_base + ".py")
  243. if not self.check_module(module, module_file):
  244. continue
  245. modules.append((package, module_base, module_file))
  246. return modules
  247. def find_all_modules(self):
  248. """Compute the list of all modules that will be built, whether
  249. they are specified one-module-at-a-time ('self.py_modules') or
  250. by whole packages ('self.packages'). Return a list of tuples
  251. (package, module, module_file), just like 'find_modules()' and
  252. 'find_package_modules()' do."""
  253. modules = []
  254. if self.py_modules:
  255. modules.extend(self.find_modules())
  256. if self.packages:
  257. for package in self.packages:
  258. package_dir = self.get_package_dir(package)
  259. m = self.find_package_modules(package, package_dir)
  260. modules.extend(m)
  261. return modules
  262. def get_source_files(self):
  263. return [module[-1] for module in self.find_all_modules()]
  264. def get_module_outfile(self, build_dir, package, module):
  265. outfile_path = [build_dir] + list(package) + [module + ".py"]
  266. return os.path.join(*outfile_path)
  267. def get_outputs(self, include_bytecode: bool = True) -> list[str]:
  268. modules = self.find_all_modules()
  269. outputs = []
  270. for package, module, _module_file in modules:
  271. package = package.split('.')
  272. filename = self.get_module_outfile(self.build_lib, package, module)
  273. outputs.append(filename)
  274. if include_bytecode:
  275. if self.compile:
  276. outputs.append(
  277. importlib.util.cache_from_source(filename, optimization='')
  278. )
  279. if self.optimize > 0:
  280. outputs.append(
  281. importlib.util.cache_from_source(
  282. filename, optimization=self.optimize
  283. )
  284. )
  285. outputs += [
  286. os.path.join(build_dir, filename)
  287. for package, src_dir, build_dir, filenames in self.data_files
  288. for filename in filenames
  289. ]
  290. return outputs
  291. def build_module(self, module, module_file, package):
  292. if isinstance(package, str):
  293. package = package.split('.')
  294. elif not isinstance(package, (list, tuple)):
  295. raise TypeError(
  296. "'package' must be a string (dot-separated), list, or tuple"
  297. )
  298. # Now put the module source file into the "build" area -- this is
  299. # easy, we just copy it somewhere under self.build_lib (the build
  300. # directory for Python source).
  301. outfile = self.get_module_outfile(self.build_lib, package, module)
  302. dir = os.path.dirname(outfile)
  303. self.mkpath(dir)
  304. return self.copy_file(module_file, outfile, preserve_mode=False)
  305. def build_modules(self) -> None:
  306. modules = self.find_modules()
  307. for package, module, module_file in modules:
  308. # Now "build" the module -- ie. copy the source file to
  309. # self.build_lib (the build directory for Python source).
  310. # (Actually, it gets copied to the directory for this package
  311. # under self.build_lib.)
  312. self.build_module(module, module_file, package)
  313. def build_packages(self) -> None:
  314. for package in self.packages:
  315. # Get list of (package, module, module_file) tuples based on
  316. # scanning the package directory. 'package' is only included
  317. # in the tuple so that 'find_modules()' and
  318. # 'find_package_tuples()' have a consistent interface; it's
  319. # ignored here (apart from a sanity check). Also, 'module' is
  320. # the *unqualified* module name (ie. no dots, no package -- we
  321. # already know its package!), and 'module_file' is the path to
  322. # the .py file, relative to the current directory
  323. # (ie. including 'package_dir').
  324. package_dir = self.get_package_dir(package)
  325. modules = self.find_package_modules(package, package_dir)
  326. # Now loop over the modules we found, "building" each one (just
  327. # copy it to self.build_lib).
  328. for package_, module, module_file in modules:
  329. assert package == package_
  330. self.build_module(module, module_file, package)
  331. def byte_compile(self, files) -> None:
  332. if sys.dont_write_bytecode:
  333. self.warn('byte-compiling is disabled, skipping.')
  334. return
  335. from ..util import byte_compile
  336. prefix = self.build_lib
  337. if prefix[-1] != os.sep:
  338. prefix = prefix + os.sep
  339. # XXX this code is essentially the same as the 'byte_compile()
  340. # method of the "install_lib" command, except for the determination
  341. # of the 'prefix' string. Hmmm.
  342. if self.compile:
  343. byte_compile(files, optimize=0, force=self.force, prefix=prefix)
  344. if self.optimize > 0:
  345. byte_compile(
  346. files,
  347. optimize=self.optimize,
  348. force=self.force,
  349. prefix=prefix,
  350. )