filesystem.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203
  1. from __future__ import annotations
  2. import fnmatch
  3. import os
  4. import os.path
  5. import random
  6. import sys
  7. from collections.abc import Generator
  8. from contextlib import contextmanager
  9. from pathlib import Path
  10. from tempfile import NamedTemporaryFile
  11. from typing import Any, BinaryIO, Callable, cast
  12. from pip._internal.utils.compat import get_path_uid
  13. from pip._internal.utils.misc import format_size
  14. from pip._internal.utils.retry import retry
  15. def check_path_owner(path: str) -> bool:
  16. # If we don't have a way to check the effective uid of this process, then
  17. # we'll just assume that we own the directory.
  18. if sys.platform == "win32" or not hasattr(os, "geteuid"):
  19. return True
  20. assert os.path.isabs(path)
  21. previous = None
  22. while path != previous:
  23. if os.path.lexists(path):
  24. # Check if path is writable by current user.
  25. if os.geteuid() == 0:
  26. # Special handling for root user in order to handle properly
  27. # cases where users use sudo without -H flag.
  28. try:
  29. path_uid = get_path_uid(path)
  30. except OSError:
  31. return False
  32. return path_uid == 0
  33. else:
  34. return os.access(path, os.W_OK)
  35. else:
  36. previous, path = path, os.path.dirname(path)
  37. return False # assume we don't own the path
  38. @contextmanager
  39. def adjacent_tmp_file(path: str, **kwargs: Any) -> Generator[BinaryIO, None, None]:
  40. """Return a file-like object pointing to a tmp file next to path.
  41. The file is created securely and is ensured to be written to disk
  42. after the context reaches its end.
  43. kwargs will be passed to tempfile.NamedTemporaryFile to control
  44. the way the temporary file will be opened.
  45. """
  46. with NamedTemporaryFile(
  47. delete=False,
  48. dir=os.path.dirname(path),
  49. prefix=os.path.basename(path),
  50. suffix=".tmp",
  51. **kwargs,
  52. ) as f:
  53. result = cast(BinaryIO, f)
  54. try:
  55. yield result
  56. finally:
  57. result.flush()
  58. os.fsync(result.fileno())
  59. replace = retry(stop_after_delay=1, wait=0.25)(os.replace)
  60. # test_writable_dir and _test_writable_dir_win are copied from Flit,
  61. # with the author's agreement to also place them under pip's license.
  62. def test_writable_dir(path: str) -> bool:
  63. """Check if a directory is writable.
  64. Uses os.access() on POSIX, tries creating files on Windows.
  65. """
  66. # If the directory doesn't exist, find the closest parent that does.
  67. while not os.path.isdir(path):
  68. parent = os.path.dirname(path)
  69. if parent == path:
  70. break # Should never get here, but infinite loops are bad
  71. path = parent
  72. if os.name == "posix":
  73. return os.access(path, os.W_OK)
  74. return _test_writable_dir_win(path)
  75. def _test_writable_dir_win(path: str) -> bool:
  76. # os.access doesn't work on Windows: http://bugs.python.org/issue2528
  77. # and we can't use tempfile: http://bugs.python.org/issue22107
  78. basename = "accesstest_deleteme_fishfingers_custard_"
  79. alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"
  80. for _ in range(10):
  81. name = basename + "".join(random.choice(alphabet) for _ in range(6))
  82. file = os.path.join(path, name)
  83. try:
  84. fd = os.open(file, os.O_RDWR | os.O_CREAT | os.O_EXCL)
  85. except FileExistsError:
  86. pass
  87. except PermissionError:
  88. # This could be because there's a directory with the same name.
  89. # But it's highly unlikely there's a directory called that,
  90. # so we'll assume it's because the parent dir is not writable.
  91. # This could as well be because the parent dir is not readable,
  92. # due to non-privileged user access.
  93. return False
  94. else:
  95. os.close(fd)
  96. os.unlink(file)
  97. return True
  98. # This should never be reached
  99. raise OSError("Unexpected condition testing for writable directory")
  100. def find_files(path: str, pattern: str) -> list[str]:
  101. """Returns a list of absolute paths of files beneath path, recursively,
  102. with filenames which match the UNIX-style shell glob pattern."""
  103. result: list[str] = []
  104. for root, _, files in os.walk(path):
  105. matches = fnmatch.filter(files, pattern)
  106. result.extend(os.path.join(root, f) for f in matches)
  107. return result
  108. def file_size(path: str) -> int | float:
  109. # If it's a symlink, return 0.
  110. if os.path.islink(path):
  111. return 0
  112. return os.path.getsize(path)
  113. def format_file_size(path: str) -> str:
  114. return format_size(file_size(path))
  115. def directory_size(path: str) -> int | float:
  116. size = 0.0
  117. for root, _dirs, files in os.walk(path):
  118. for filename in files:
  119. file_path = os.path.join(root, filename)
  120. size += file_size(file_path)
  121. return size
  122. def format_directory_size(path: str) -> str:
  123. return format_size(directory_size(path))
  124. def copy_directory_permissions(directory: str, target_file: BinaryIO) -> None:
  125. mode = (
  126. os.stat(directory).st_mode & 0o666 # select read/write permissions of directory
  127. | 0o600 # set owner read/write permissions
  128. )
  129. # Change permissions only if there is no risk of following a symlink.
  130. if os.chmod in os.supports_fd:
  131. os.chmod(target_file.fileno(), mode)
  132. elif os.chmod in os.supports_follow_symlinks:
  133. os.chmod(target_file.name, mode, follow_symlinks=False)
  134. def _subdirs_without_generic(
  135. path: str, predicate: Callable[[str, list[str]], bool]
  136. ) -> Generator[Path]:
  137. """Yields every subdirectory of +path+ that has no files matching the
  138. predicate under it."""
  139. directories = []
  140. excluded = set()
  141. for root_str, _, filenames in os.walk(Path(path).resolve()):
  142. root = Path(root_str)
  143. if predicate(root_str, filenames):
  144. # This directory should be excluded, so exclude it and all of its
  145. # parent directories.
  146. # The last item in root.parents is ".", so we ignore it.
  147. #
  148. # Wrapping this in `list()` is only needed for Python 3.9.
  149. excluded.update(list(root.parents)[:-1])
  150. excluded.add(root)
  151. directories.append(root)
  152. for d in sorted(directories, reverse=True):
  153. if d not in excluded:
  154. yield d
  155. def subdirs_without_files(path: str) -> Generator[Path]:
  156. """Yields every subdirectory of +path+ that has no files under it."""
  157. return _subdirs_without_generic(path, lambda root, filenames: len(filenames) > 0)
  158. def subdirs_without_wheels(path: str) -> Generator[Path]:
  159. """Yields every subdirectory of +path+ that has no .whl files under it."""
  160. return _subdirs_without_generic(
  161. path, lambda root, filenames: any(x.endswith(".whl") for x in filenames)
  162. )