test_sdist.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980
  1. """sdist tests"""
  2. import contextlib
  3. import io
  4. import logging
  5. import os
  6. import pathlib
  7. import sys
  8. import tarfile
  9. import tempfile
  10. import unicodedata
  11. from inspect import cleandoc
  12. from pathlib import Path
  13. from unittest import mock
  14. import jaraco.path
  15. import pytest
  16. from setuptools import Command, SetuptoolsDeprecationWarning
  17. from setuptools._importlib import metadata
  18. from setuptools.command.egg_info import manifest_maker
  19. from setuptools.command.sdist import sdist
  20. from setuptools.dist import Distribution
  21. from setuptools.extension import Extension
  22. from setuptools.tests import fail_on_ascii
  23. from .text import Filenames
  24. import distutils
  25. from distutils.core import run_setup
  26. SETUP_ATTRS = {
  27. 'name': 'sdist_test',
  28. 'version': '0.0',
  29. 'packages': ['sdist_test'],
  30. 'package_data': {'sdist_test': ['*.txt']},
  31. 'data_files': [("data", [os.path.join("d", "e.dat")])],
  32. }
  33. SETUP_PY = f"""\
  34. from setuptools import setup
  35. setup(**{SETUP_ATTRS!r})
  36. """
  37. EXTENSION = Extension(
  38. name="sdist_test.f",
  39. sources=[os.path.join("sdist_test", "f.c")],
  40. depends=[os.path.join("sdist_test", "f.h")],
  41. )
  42. EXTENSION_SOURCES = EXTENSION.sources + EXTENSION.depends
  43. @contextlib.contextmanager
  44. def quiet():
  45. old_stdout, old_stderr = sys.stdout, sys.stderr
  46. sys.stdout, sys.stderr = io.StringIO(), io.StringIO()
  47. try:
  48. yield
  49. finally:
  50. sys.stdout, sys.stderr = old_stdout, old_stderr
  51. # Convert to POSIX path
  52. def posix(path):
  53. if not isinstance(path, str):
  54. return path.replace(os.sep.encode('ascii'), b'/')
  55. else:
  56. return path.replace(os.sep, '/')
  57. # HFS Plus uses decomposed UTF-8
  58. def decompose(path):
  59. if isinstance(path, str):
  60. return unicodedata.normalize('NFD', path)
  61. try:
  62. path = path.decode('utf-8')
  63. path = unicodedata.normalize('NFD', path)
  64. path = path.encode('utf-8')
  65. except UnicodeError:
  66. pass # Not UTF-8
  67. return path
  68. def read_all_bytes(filename):
  69. with open(filename, 'rb') as fp:
  70. return fp.read()
  71. def latin1_fail():
  72. try:
  73. desc, filename = tempfile.mkstemp(suffix=Filenames.latin_1)
  74. os.close(desc)
  75. os.remove(filename)
  76. except Exception:
  77. return True
  78. fail_on_latin1_encoded_filenames = pytest.mark.xfail(
  79. latin1_fail(),
  80. reason="System does not support latin-1 filenames",
  81. )
  82. skip_under_xdist = pytest.mark.skipif(
  83. "os.environ.get('PYTEST_XDIST_WORKER')",
  84. reason="pytest-dev/pytest-xdist#843",
  85. )
  86. skip_under_stdlib_distutils = pytest.mark.skipif(
  87. not distutils.__package__.startswith('setuptools'),
  88. reason="the test is not supported with stdlib distutils",
  89. )
  90. def touch(path):
  91. open(path, 'wb').close()
  92. return path
  93. def symlink_or_skip_test(src, dst):
  94. try:
  95. os.symlink(src, dst)
  96. except (OSError, NotImplementedError):
  97. pytest.skip("symlink not supported in OS")
  98. return None
  99. return dst
  100. class TestSdistTest:
  101. @pytest.fixture(autouse=True)
  102. def source_dir(self, tmpdir):
  103. tmpdir = tmpdir / "project_root"
  104. tmpdir.mkdir()
  105. (tmpdir / 'setup.py').write_text(SETUP_PY, encoding='utf-8')
  106. # Set up the rest of the test package
  107. test_pkg = tmpdir / 'sdist_test'
  108. test_pkg.mkdir()
  109. data_folder = tmpdir / 'd'
  110. data_folder.mkdir()
  111. # *.rst was not included in package_data, so c.rst should not be
  112. # automatically added to the manifest when not under version control
  113. for fname in ['__init__.py', 'a.txt', 'b.txt', 'c.rst']:
  114. touch(test_pkg / fname)
  115. touch(data_folder / 'e.dat')
  116. # C sources are not included by default, but they will be,
  117. # if an extension module uses them as sources or depends
  118. for fname in EXTENSION_SOURCES:
  119. touch(tmpdir / fname)
  120. with tmpdir.as_cwd():
  121. yield tmpdir
  122. def assert_package_data_in_manifest(self, cmd):
  123. manifest = cmd.filelist.files
  124. assert os.path.join('sdist_test', 'a.txt') in manifest
  125. assert os.path.join('sdist_test', 'b.txt') in manifest
  126. assert os.path.join('sdist_test', 'c.rst') not in manifest
  127. assert os.path.join('d', 'e.dat') in manifest
  128. def setup_with_extension(self):
  129. setup_attrs = {**SETUP_ATTRS, 'ext_modules': [EXTENSION]}
  130. dist = Distribution(setup_attrs)
  131. dist.script_name = 'setup.py'
  132. cmd = sdist(dist)
  133. cmd.ensure_finalized()
  134. with quiet():
  135. cmd.run()
  136. return cmd
  137. def test_package_data_in_sdist(self):
  138. """Regression test for pull request #4: ensures that files listed in
  139. package_data are included in the manifest even if they're not added to
  140. version control.
  141. """
  142. dist = Distribution(SETUP_ATTRS)
  143. dist.script_name = 'setup.py'
  144. cmd = sdist(dist)
  145. cmd.ensure_finalized()
  146. with quiet():
  147. cmd.run()
  148. self.assert_package_data_in_manifest(cmd)
  149. def test_package_data_and_include_package_data_in_sdist(self):
  150. """
  151. Ensure package_data and include_package_data work
  152. together.
  153. """
  154. setup_attrs = {**SETUP_ATTRS, 'include_package_data': True}
  155. assert setup_attrs['package_data']
  156. dist = Distribution(setup_attrs)
  157. dist.script_name = 'setup.py'
  158. cmd = sdist(dist)
  159. cmd.ensure_finalized()
  160. with quiet():
  161. cmd.run()
  162. self.assert_package_data_in_manifest(cmd)
  163. def test_extension_sources_in_sdist(self):
  164. """
  165. Ensure that the files listed in Extension.sources and Extension.depends
  166. are automatically included in the manifest.
  167. """
  168. cmd = self.setup_with_extension()
  169. self.assert_package_data_in_manifest(cmd)
  170. manifest = cmd.filelist.files
  171. for path in EXTENSION_SOURCES:
  172. assert path in manifest
  173. def test_missing_extension_sources(self):
  174. """
  175. Similar to test_extension_sources_in_sdist but the referenced files don't exist.
  176. Missing files should not be included in distribution (with no error raised).
  177. """
  178. for path in EXTENSION_SOURCES:
  179. os.remove(path)
  180. cmd = self.setup_with_extension()
  181. self.assert_package_data_in_manifest(cmd)
  182. manifest = cmd.filelist.files
  183. for path in EXTENSION_SOURCES:
  184. assert path not in manifest
  185. def test_symlinked_extension_sources(self):
  186. """
  187. Similar to test_extension_sources_in_sdist but the referenced files are
  188. instead symbolic links to project-local files. Referenced file paths
  189. should be included. Symlink targets themselves should NOT be included.
  190. """
  191. symlinked = []
  192. for path in EXTENSION_SOURCES:
  193. base, ext = os.path.splitext(path)
  194. target = base + "_target." + ext
  195. os.rename(path, target)
  196. symlink_or_skip_test(os.path.basename(target), path)
  197. symlinked.append(target)
  198. cmd = self.setup_with_extension()
  199. self.assert_package_data_in_manifest(cmd)
  200. manifest = cmd.filelist.files
  201. for path in EXTENSION_SOURCES:
  202. assert path in manifest
  203. for path in symlinked:
  204. assert path not in manifest
  205. _INVALID_PATHS = {
  206. "must be relative": lambda: os.path.abspath(os.path.join("sdist_test", "f.h")),
  207. "can't have `..` segments": lambda: os.path.join(
  208. "sdist_test", "..", "sdist_test", "f.h"
  209. ),
  210. "doesn't exist": lambda: os.path.join(
  211. "sdist_test", "this_file_does_not_exist.h"
  212. ),
  213. "must be inside the project root": lambda: symlink_or_skip_test(
  214. touch(os.path.join("..", "outside_of_project_root.h")),
  215. "symlink.h",
  216. ),
  217. }
  218. @skip_under_stdlib_distutils
  219. @pytest.mark.parametrize("reason", _INVALID_PATHS.keys())
  220. def test_invalid_extension_depends(self, reason, caplog):
  221. """
  222. Due to backwards compatibility reasons, `Extension.depends` should accept
  223. invalid/weird paths, but then ignore them when building a sdist.
  224. This test verifies that the source distribution is still built
  225. successfully with such paths, but that instead of adding these paths to
  226. the manifest, we emit an informational message, notifying the user that
  227. the invalid path won't be automatically included.
  228. """
  229. invalid_path = self._INVALID_PATHS[reason]()
  230. extension = Extension(
  231. name="sdist_test.f",
  232. sources=[],
  233. depends=[invalid_path],
  234. )
  235. setup_attrs = {**SETUP_ATTRS, 'ext_modules': [extension]}
  236. dist = Distribution(setup_attrs)
  237. dist.script_name = 'setup.py'
  238. cmd = sdist(dist)
  239. cmd.ensure_finalized()
  240. with quiet(), caplog.at_level(logging.INFO):
  241. cmd.run()
  242. self.assert_package_data_in_manifest(cmd)
  243. manifest = cmd.filelist.files
  244. assert invalid_path not in manifest
  245. expected_message = [
  246. message
  247. for (logger, level, message) in caplog.record_tuples
  248. if (
  249. logger == "root" #
  250. and level == logging.INFO #
  251. and invalid_path in message #
  252. )
  253. ]
  254. assert len(expected_message) == 1
  255. (expected_message,) = expected_message
  256. assert reason in expected_message
  257. def test_custom_build_py(self):
  258. """
  259. Ensure projects defining custom build_py don't break
  260. when creating sdists (issue #2849)
  261. """
  262. from distutils.command.build_py import build_py as OrigBuildPy
  263. using_custom_command_guard = mock.Mock()
  264. class CustomBuildPy(OrigBuildPy):
  265. """
  266. Some projects have custom commands inheriting from `distutils`
  267. """
  268. def get_data_files(self):
  269. using_custom_command_guard()
  270. return super().get_data_files()
  271. setup_attrs = {**SETUP_ATTRS, 'include_package_data': True}
  272. assert setup_attrs['package_data']
  273. dist = Distribution(setup_attrs)
  274. dist.script_name = 'setup.py'
  275. cmd = sdist(dist)
  276. cmd.ensure_finalized()
  277. # Make sure we use the custom command
  278. cmd.cmdclass = {'build_py': CustomBuildPy}
  279. cmd.distribution.cmdclass = {'build_py': CustomBuildPy}
  280. assert cmd.distribution.get_command_class('build_py') == CustomBuildPy
  281. msg = "setuptools instead of distutils"
  282. with quiet(), pytest.warns(SetuptoolsDeprecationWarning, match=msg):
  283. cmd.run()
  284. using_custom_command_guard.assert_called()
  285. self.assert_package_data_in_manifest(cmd)
  286. def test_setup_py_exists(self):
  287. dist = Distribution(SETUP_ATTRS)
  288. dist.script_name = 'foo.py'
  289. cmd = sdist(dist)
  290. cmd.ensure_finalized()
  291. with quiet():
  292. cmd.run()
  293. manifest = cmd.filelist.files
  294. assert 'setup.py' in manifest
  295. def test_setup_py_missing(self):
  296. dist = Distribution(SETUP_ATTRS)
  297. dist.script_name = 'foo.py'
  298. cmd = sdist(dist)
  299. cmd.ensure_finalized()
  300. if os.path.exists("setup.py"):
  301. os.remove("setup.py")
  302. with quiet():
  303. cmd.run()
  304. manifest = cmd.filelist.files
  305. assert 'setup.py' not in manifest
  306. def test_setup_py_excluded(self):
  307. with open("MANIFEST.in", "w", encoding="utf-8") as manifest_file:
  308. manifest_file.write("exclude setup.py")
  309. dist = Distribution(SETUP_ATTRS)
  310. dist.script_name = 'foo.py'
  311. cmd = sdist(dist)
  312. cmd.ensure_finalized()
  313. with quiet():
  314. cmd.run()
  315. manifest = cmd.filelist.files
  316. assert 'setup.py' not in manifest
  317. def test_defaults_case_sensitivity(self, source_dir):
  318. """
  319. Make sure default files (README.*, etc.) are added in a case-sensitive
  320. way to avoid problems with packages built on Windows.
  321. """
  322. touch(source_dir / 'readme.rst')
  323. touch(source_dir / 'SETUP.cfg')
  324. dist = Distribution(SETUP_ATTRS)
  325. # the extension deliberately capitalized for this test
  326. # to make sure the actual filename (not capitalized) gets added
  327. # to the manifest
  328. dist.script_name = 'setup.PY'
  329. cmd = sdist(dist)
  330. cmd.ensure_finalized()
  331. with quiet():
  332. cmd.run()
  333. # lowercase all names so we can test in a
  334. # case-insensitive way to make sure the files
  335. # are not included.
  336. manifest = map(lambda x: x.lower(), cmd.filelist.files)
  337. assert 'readme.rst' not in manifest, manifest
  338. assert 'setup.py' not in manifest, manifest
  339. assert 'setup.cfg' not in manifest, manifest
  340. def test_exclude_dev_only_cache_folders(self, source_dir):
  341. included = {
  342. # Emulate problem in https://github.com/pypa/setuptools/issues/4601
  343. "MANIFEST.in": (
  344. "global-include LICEN[CS]E* COPYING* NOTICE* AUTHORS*\n"
  345. "global-include *.txt\n"
  346. ),
  347. # For the sake of being conservative and limiting unforeseen side-effects
  348. # we just exclude dev-only cache folders at the root of the repository:
  349. "test/.venv/lib/python3.9/site-packages/bar-2.dist-info/AUTHORS.rst": "",
  350. "src/.nox/py/lib/python3.12/site-packages/bar-2.dist-info/COPYING.txt": "",
  351. "doc/.tox/default/lib/python3.11/site-packages/foo-4.dist-info/LICENSE": "",
  352. # Let's test against false positives with similarly named files:
  353. ".venv-requirements.txt": "",
  354. ".tox-coveragerc.txt": "",
  355. ".noxy/coveragerc.txt": "",
  356. }
  357. excluded = {
  358. # .tox/.nox/.venv are well-know folders present at the root of Python repos
  359. # and therefore should be excluded
  360. ".tox/release/lib/python3.11/site-packages/foo-4.dist-info/LICENSE": "",
  361. ".nox/py/lib/python3.12/site-packages/bar-2.dist-info/COPYING.txt": "",
  362. ".venv/lib/python3.9/site-packages/bar-2.dist-info/AUTHORS.rst": "",
  363. }
  364. for file, content in {**excluded, **included}.items():
  365. Path(source_dir, file).parent.mkdir(parents=True, exist_ok=True)
  366. Path(source_dir, file).write_text(content, encoding="utf-8")
  367. cmd = self.setup_with_extension()
  368. self.assert_package_data_in_manifest(cmd)
  369. manifest = {f.replace(os.sep, '/') for f in cmd.filelist.files}
  370. for path in excluded:
  371. assert os.path.exists(path)
  372. assert path not in manifest, (path, manifest)
  373. for path in included:
  374. assert os.path.exists(path)
  375. assert path in manifest, (path, manifest)
  376. @fail_on_ascii
  377. def test_manifest_is_written_with_utf8_encoding(self):
  378. # Test for #303.
  379. dist = Distribution(SETUP_ATTRS)
  380. dist.script_name = 'setup.py'
  381. mm = manifest_maker(dist)
  382. mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  383. os.mkdir('sdist_test.egg-info')
  384. # UTF-8 filename
  385. filename = os.path.join('sdist_test', 'smörbröd.py')
  386. # Must create the file or it will get stripped.
  387. touch(filename)
  388. # Add UTF-8 filename and write manifest
  389. with quiet():
  390. mm.run()
  391. mm.filelist.append(filename)
  392. mm.write_manifest()
  393. contents = read_all_bytes(mm.manifest)
  394. # The manifest should be UTF-8 encoded
  395. u_contents = contents.decode('UTF-8')
  396. # The manifest should contain the UTF-8 filename
  397. assert posix(filename) in u_contents
  398. @fail_on_ascii
  399. def test_write_manifest_allows_utf8_filenames(self):
  400. # Test for #303.
  401. dist = Distribution(SETUP_ATTRS)
  402. dist.script_name = 'setup.py'
  403. mm = manifest_maker(dist)
  404. mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  405. os.mkdir('sdist_test.egg-info')
  406. filename = os.path.join(b'sdist_test', Filenames.utf_8)
  407. # Must touch the file or risk removal
  408. touch(filename)
  409. # Add filename and write manifest
  410. with quiet():
  411. mm.run()
  412. u_filename = filename.decode('utf-8')
  413. mm.filelist.files.append(u_filename)
  414. # Re-write manifest
  415. mm.write_manifest()
  416. contents = read_all_bytes(mm.manifest)
  417. # The manifest should be UTF-8 encoded
  418. contents.decode('UTF-8')
  419. # The manifest should contain the UTF-8 filename
  420. assert posix(filename) in contents
  421. # The filelist should have been updated as well
  422. assert u_filename in mm.filelist.files
  423. @skip_under_xdist
  424. def test_write_manifest_skips_non_utf8_filenames(self):
  425. """
  426. Files that cannot be encoded to UTF-8 (specifically, those that
  427. weren't originally successfully decoded and have surrogate
  428. escapes) should be omitted from the manifest.
  429. See https://bitbucket.org/tarek/distribute/issue/303 for history.
  430. """
  431. dist = Distribution(SETUP_ATTRS)
  432. dist.script_name = 'setup.py'
  433. mm = manifest_maker(dist)
  434. mm.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  435. os.mkdir('sdist_test.egg-info')
  436. # Latin-1 filename
  437. filename = os.path.join(b'sdist_test', Filenames.latin_1)
  438. # Add filename with surrogates and write manifest
  439. with quiet():
  440. mm.run()
  441. u_filename = filename.decode('utf-8', 'surrogateescape')
  442. mm.filelist.append(u_filename)
  443. # Re-write manifest
  444. mm.write_manifest()
  445. contents = read_all_bytes(mm.manifest)
  446. # The manifest should be UTF-8 encoded
  447. contents.decode('UTF-8')
  448. # The Latin-1 filename should have been skipped
  449. assert posix(filename) not in contents
  450. # The filelist should have been updated as well
  451. assert u_filename not in mm.filelist.files
  452. @fail_on_ascii
  453. def test_manifest_is_read_with_utf8_encoding(self):
  454. # Test for #303.
  455. dist = Distribution(SETUP_ATTRS)
  456. dist.script_name = 'setup.py'
  457. cmd = sdist(dist)
  458. cmd.ensure_finalized()
  459. # Create manifest
  460. with quiet():
  461. cmd.run()
  462. # Add UTF-8 filename to manifest
  463. filename = os.path.join(b'sdist_test', Filenames.utf_8)
  464. cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  465. manifest = open(cmd.manifest, 'ab')
  466. manifest.write(b'\n' + filename)
  467. manifest.close()
  468. # The file must exist to be included in the filelist
  469. touch(filename)
  470. # Re-read manifest
  471. cmd.filelist.files = []
  472. with quiet():
  473. cmd.read_manifest()
  474. # The filelist should contain the UTF-8 filename
  475. filename = filename.decode('utf-8')
  476. assert filename in cmd.filelist.files
  477. @fail_on_latin1_encoded_filenames
  478. def test_read_manifest_skips_non_utf8_filenames(self):
  479. # Test for #303.
  480. dist = Distribution(SETUP_ATTRS)
  481. dist.script_name = 'setup.py'
  482. cmd = sdist(dist)
  483. cmd.ensure_finalized()
  484. # Create manifest
  485. with quiet():
  486. cmd.run()
  487. # Add Latin-1 filename to manifest
  488. filename = os.path.join(b'sdist_test', Filenames.latin_1)
  489. cmd.manifest = os.path.join('sdist_test.egg-info', 'SOURCES.txt')
  490. manifest = open(cmd.manifest, 'ab')
  491. manifest.write(b'\n' + filename)
  492. manifest.close()
  493. # The file must exist to be included in the filelist
  494. touch(filename)
  495. # Re-read manifest
  496. cmd.filelist.files = []
  497. with quiet():
  498. cmd.read_manifest()
  499. # The Latin-1 filename should have been skipped
  500. filename = filename.decode('latin-1')
  501. assert filename not in cmd.filelist.files
  502. @fail_on_ascii
  503. @fail_on_latin1_encoded_filenames
  504. def test_sdist_with_utf8_encoded_filename(self):
  505. # Test for #303.
  506. dist = Distribution(self.make_strings(SETUP_ATTRS))
  507. dist.script_name = 'setup.py'
  508. cmd = sdist(dist)
  509. cmd.ensure_finalized()
  510. filename = os.path.join(b'sdist_test', Filenames.utf_8)
  511. touch(filename)
  512. with quiet():
  513. cmd.run()
  514. if sys.platform == 'darwin':
  515. filename = decompose(filename)
  516. fs_enc = sys.getfilesystemencoding()
  517. if sys.platform == 'win32':
  518. if fs_enc == 'cp1252':
  519. # Python mangles the UTF-8 filename
  520. filename = filename.decode('cp1252')
  521. assert filename in cmd.filelist.files
  522. else:
  523. filename = filename.decode('mbcs')
  524. assert filename in cmd.filelist.files
  525. else:
  526. filename = filename.decode('utf-8')
  527. assert filename in cmd.filelist.files
  528. @classmethod
  529. def make_strings(cls, item):
  530. if isinstance(item, dict):
  531. return {key: cls.make_strings(value) for key, value in item.items()}
  532. if isinstance(item, list):
  533. return list(map(cls.make_strings, item))
  534. return str(item)
  535. @fail_on_latin1_encoded_filenames
  536. @skip_under_xdist
  537. def test_sdist_with_latin1_encoded_filename(self):
  538. # Test for #303.
  539. dist = Distribution(self.make_strings(SETUP_ATTRS))
  540. dist.script_name = 'setup.py'
  541. cmd = sdist(dist)
  542. cmd.ensure_finalized()
  543. # Latin-1 filename
  544. filename = os.path.join(b'sdist_test', Filenames.latin_1)
  545. touch(filename)
  546. assert os.path.isfile(filename)
  547. with quiet():
  548. cmd.run()
  549. # not all windows systems have a default FS encoding of cp1252
  550. if sys.platform == 'win32':
  551. # Latin-1 is similar to Windows-1252 however
  552. # on mbcs filesys it is not in latin-1 encoding
  553. fs_enc = sys.getfilesystemencoding()
  554. if fs_enc != 'mbcs':
  555. fs_enc = 'latin-1'
  556. filename = filename.decode(fs_enc)
  557. assert filename in cmd.filelist.files
  558. else:
  559. # The Latin-1 filename should have been skipped
  560. filename = filename.decode('latin-1')
  561. assert filename not in cmd.filelist.files
  562. _EXAMPLE_DIRECTIVES = {
  563. "setup.cfg - long_description and version": """
  564. [metadata]
  565. name = testing
  566. version = file: src/VERSION.txt
  567. license_files = DOWHATYOUWANT
  568. long_description = file: README.rst, USAGE.rst
  569. """,
  570. "pyproject.toml - static readme/license files and dynamic version": """
  571. [project]
  572. name = "testing"
  573. readme = "USAGE.rst"
  574. license-files = ["DOWHATYOUWANT"]
  575. dynamic = ["version"]
  576. [tool.setuptools.dynamic]
  577. version = {file = ["src/VERSION.txt"]}
  578. """,
  579. "pyproject.toml - directive with str instead of list": """
  580. [project]
  581. name = "testing"
  582. readme = "USAGE.rst"
  583. license-files = ["DOWHATYOUWANT"]
  584. dynamic = ["version"]
  585. [tool.setuptools.dynamic]
  586. version = {file = "src/VERSION.txt"}
  587. """,
  588. "pyproject.toml - deprecated license table with file entry": """
  589. [project]
  590. name = "testing"
  591. readme = "USAGE.rst"
  592. license = {file = "DOWHATYOUWANT"}
  593. dynamic = ["version"]
  594. [tool.setuptools.dynamic]
  595. version = {file = "src/VERSION.txt"}
  596. """,
  597. }
  598. @pytest.mark.parametrize("config", _EXAMPLE_DIRECTIVES.keys())
  599. @pytest.mark.filterwarnings(
  600. "ignore:.project.license. as a TOML table is deprecated"
  601. )
  602. def test_add_files_referenced_by_config_directives(self, source_dir, config):
  603. config_file, _, _ = config.partition(" - ")
  604. config_text = self._EXAMPLE_DIRECTIVES[config]
  605. (source_dir / 'src').mkdir()
  606. (source_dir / 'src/VERSION.txt').write_text("0.42", encoding="utf-8")
  607. (source_dir / 'README.rst').write_text("hello world!", encoding="utf-8")
  608. (source_dir / 'USAGE.rst').write_text("hello world!", encoding="utf-8")
  609. (source_dir / 'DOWHATYOUWANT').write_text("hello world!", encoding="utf-8")
  610. (source_dir / config_file).write_text(config_text, encoding="utf-8")
  611. dist = Distribution({"packages": []})
  612. dist.script_name = 'setup.py'
  613. dist.parse_config_files()
  614. cmd = sdist(dist)
  615. cmd.ensure_finalized()
  616. with quiet():
  617. cmd.run()
  618. assert (
  619. 'src/VERSION.txt' in cmd.filelist.files
  620. or 'src\\VERSION.txt' in cmd.filelist.files
  621. )
  622. assert 'USAGE.rst' in cmd.filelist.files
  623. assert 'DOWHATYOUWANT' in cmd.filelist.files
  624. assert '/' not in cmd.filelist.files
  625. assert '\\' not in cmd.filelist.files
  626. def test_pyproject_toml_in_sdist(self, source_dir):
  627. """
  628. Check if pyproject.toml is included in source distribution if present
  629. """
  630. touch(source_dir / 'pyproject.toml')
  631. dist = Distribution(SETUP_ATTRS)
  632. dist.script_name = 'setup.py'
  633. cmd = sdist(dist)
  634. cmd.ensure_finalized()
  635. with quiet():
  636. cmd.run()
  637. manifest = cmd.filelist.files
  638. assert 'pyproject.toml' in manifest
  639. def test_pyproject_toml_excluded(self, source_dir):
  640. """
  641. Check that pyproject.toml can excluded even if present
  642. """
  643. touch(source_dir / 'pyproject.toml')
  644. with open('MANIFEST.in', 'w', encoding="utf-8") as mts:
  645. print('exclude pyproject.toml', file=mts)
  646. dist = Distribution(SETUP_ATTRS)
  647. dist.script_name = 'setup.py'
  648. cmd = sdist(dist)
  649. cmd.ensure_finalized()
  650. with quiet():
  651. cmd.run()
  652. manifest = cmd.filelist.files
  653. assert 'pyproject.toml' not in manifest
  654. def test_build_subcommand_source_files(self, source_dir):
  655. touch(source_dir / '.myfile~')
  656. # Sanity check: without custom commands file list should not be affected
  657. dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"})
  658. cmd = sdist(dist)
  659. cmd.ensure_finalized()
  660. with quiet():
  661. cmd.run()
  662. manifest = cmd.filelist.files
  663. assert '.myfile~' not in manifest
  664. # Test: custom command should be able to augment file list
  665. dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"})
  666. build = dist.get_command_obj("build")
  667. build.sub_commands = [*build.sub_commands, ("build_custom", None)]
  668. class build_custom(Command):
  669. def initialize_options(self): ...
  670. def finalize_options(self): ...
  671. def run(self): ...
  672. def get_source_files(self):
  673. return ['.myfile~']
  674. dist.cmdclass.update(build_custom=build_custom)
  675. cmd = sdist(dist)
  676. cmd.use_defaults = True
  677. cmd.ensure_finalized()
  678. with quiet():
  679. cmd.run()
  680. manifest = cmd.filelist.files
  681. assert '.myfile~' in manifest
  682. @pytest.mark.skipif("os.environ.get('SETUPTOOLS_USE_DISTUTILS') == 'stdlib'")
  683. def test_build_base_pathlib(self, source_dir):
  684. """
  685. Ensure if build_base is a pathlib.Path, the build still succeeds.
  686. """
  687. dist = Distribution({
  688. **SETUP_ATTRS,
  689. "script_name": "setup.py",
  690. "options": {"build": {"build_base": pathlib.Path('build')}},
  691. })
  692. cmd = sdist(dist)
  693. cmd.ensure_finalized()
  694. with quiet():
  695. cmd.run()
  696. def test_default_revctrl():
  697. """
  698. When _default_revctrl was removed from the `setuptools.command.sdist`
  699. module in 10.0, it broke some systems which keep an old install of
  700. setuptools (Distribute) around. Those old versions require that the
  701. setuptools package continue to implement that interface, so this
  702. function provides that interface, stubbed. See #320 for details.
  703. This interface must be maintained until Ubuntu 12.04 is no longer
  704. supported (by Setuptools).
  705. """
  706. (ep,) = metadata.EntryPoints._from_text(
  707. """
  708. [setuptools.file_finders]
  709. svn_cvs = setuptools.command.sdist:_default_revctrl
  710. """
  711. )
  712. res = ep.load()
  713. assert hasattr(res, '__iter__')
  714. class TestRegressions:
  715. """
  716. Can be removed/changed if the project decides to change how it handles symlinks
  717. or external files.
  718. """
  719. @staticmethod
  720. def files_for_symlink_in_extension_depends(tmp_path, dep_path):
  721. return {
  722. "external": {
  723. "dir": {"file.h": ""},
  724. },
  725. "project": {
  726. "setup.py": cleandoc(
  727. f"""
  728. from setuptools import Extension, setup
  729. setup(
  730. name="myproj",
  731. version="42",
  732. ext_modules=[
  733. Extension(
  734. "hello", sources=["hello.pyx"],
  735. depends=[{dep_path!r}]
  736. )
  737. ],
  738. )
  739. """
  740. ),
  741. "hello.pyx": "",
  742. "MANIFEST.in": "global-include *.h",
  743. },
  744. }
  745. @pytest.mark.parametrize(
  746. "dep_path", ("myheaders/dir/file.h", "myheaders/dir/../dir/file.h")
  747. )
  748. def test_symlink_in_extension_depends(self, monkeypatch, tmp_path, dep_path):
  749. # Given a project with a symlinked dir and a "depends" targeting that dir
  750. files = self.files_for_symlink_in_extension_depends(tmp_path, dep_path)
  751. jaraco.path.build(files, prefix=str(tmp_path))
  752. symlink_or_skip_test(tmp_path / "external", tmp_path / "project/myheaders")
  753. # When `sdist` runs, there should be no error
  754. members = run_sdist(monkeypatch, tmp_path / "project")
  755. # and the sdist should contain the symlinked files
  756. for expected in (
  757. "myproj-42/hello.pyx",
  758. "myproj-42/myheaders/dir/file.h",
  759. ):
  760. assert expected in members
  761. @staticmethod
  762. def files_for_external_path_in_extension_depends(tmp_path, dep_path):
  763. head, _, tail = dep_path.partition("$tmp_path$/")
  764. dep_path = tmp_path / tail if tail else head
  765. return {
  766. "external": {
  767. "dir": {"file.h": ""},
  768. },
  769. "project": {
  770. "setup.py": cleandoc(
  771. f"""
  772. from setuptools import Extension, setup
  773. setup(
  774. name="myproj",
  775. version="42",
  776. ext_modules=[
  777. Extension(
  778. "hello", sources=["hello.pyx"],
  779. depends=[{str(dep_path)!r}]
  780. )
  781. ],
  782. )
  783. """
  784. ),
  785. "hello.pyx": "",
  786. "MANIFEST.in": "global-include *.h",
  787. },
  788. }
  789. @pytest.mark.parametrize(
  790. "dep_path", ("$tmp_path$/external/dir/file.h", "../external/dir/file.h")
  791. )
  792. def test_external_path_in_extension_depends(self, monkeypatch, tmp_path, dep_path):
  793. # Given a project with a "depends" targeting an external dir
  794. files = self.files_for_external_path_in_extension_depends(tmp_path, dep_path)
  795. jaraco.path.build(files, prefix=str(tmp_path))
  796. # When `sdist` runs, there should be no error
  797. members = run_sdist(monkeypatch, tmp_path / "project")
  798. # and the sdist should not contain the external file
  799. for name in members:
  800. assert "file.h" not in name
  801. def run_sdist(monkeypatch, project):
  802. """Given a project directory, run the sdist and return its contents"""
  803. monkeypatch.chdir(project)
  804. with quiet():
  805. run_setup("setup.py", ["sdist"])
  806. archive = next((project / "dist").glob("*.tar.gz"))
  807. with tarfile.open(str(archive)) as tar:
  808. return set(tar.getnames())
  809. def test_sanity_check_setuptools_own_sdist(setuptools_sdist):
  810. with tarfile.open(setuptools_sdist) as tar:
  811. files = tar.getnames()
  812. # setuptools sdist should not include the .tox folder
  813. tox_files = [name for name in files if ".tox" in name]
  814. assert len(tox_files) == 0, f"not empty {tox_files}"