via_disk_folder.py 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181
  1. r"""A rough layout of the current storage goes as:
  2. ::
  3. virtualenv-app-data
  4. ├── py - <version> <cache information about python interpreters>
  5. │ └── *.json/lock
  6. ├── wheel <cache wheels used for seeding>
  7. │ ├── house
  8. │ │ └── *.whl <wheels downloaded go here>
  9. │ └── <python major.minor> -> 3.9
  10. │ ├── img-<version>
  11. │ │ └── image
  12. │ │ └── <install class> -> CopyPipInstall / SymlinkPipInstall
  13. │ │ └── <wheel name> -> pip-20.1.1-py2.py3-none-any
  14. │ └── embed
  15. │ └── 3 -> json format versioning
  16. │ └── *.json -> for every distribution contains data about newer embed versions and releases
  17. └─── unzip <in zip app we cannot refer to some internal files, so first extract them>
  18. └── <virtualenv version>
  19. ├── py_info.py
  20. ├── debug.py
  21. └── _virtualenv.py
  22. """ # noqa: D415
  23. from __future__ import annotations
  24. import json
  25. import logging
  26. from abc import ABC
  27. from contextlib import contextmanager, suppress
  28. from hashlib import sha256
  29. from typing import TYPE_CHECKING, Any
  30. from virtualenv.util.lock import ReentrantFileLock
  31. from virtualenv.util.path import safe_delete
  32. from virtualenv.util.zipapp import extract
  33. from virtualenv.version import __version__
  34. from .base import AppData, ContentStore
  35. if TYPE_CHECKING:
  36. from collections.abc import Generator
  37. from pathlib import Path
  38. LOGGER = logging.getLogger(__name__)
  39. class AppDataDiskFolder(AppData):
  40. """Store the application data on the disk within a folder layout."""
  41. transient = False
  42. can_update = True
  43. def __init__(self, folder: str) -> None:
  44. self.lock = ReentrantFileLock(folder)
  45. def __repr__(self) -> str:
  46. return f"{type(self).__name__}({self.lock.path})"
  47. def __str__(self) -> str:
  48. return str(self.lock.path)
  49. def reset(self) -> None:
  50. LOGGER.debug("reset app data folder %s", self.lock.path)
  51. safe_delete(self.lock.path)
  52. def close(self) -> None:
  53. """Do nothing."""
  54. @contextmanager
  55. def locked(self, path: Path) -> Generator[None]:
  56. path_lock = self.lock / path # ty: ignore[unsupported-operator]
  57. with path_lock:
  58. yield path_lock.path
  59. @contextmanager
  60. def extract(self, path: Path, to_folder: Path | None) -> Generator[Path]:
  61. root = ReentrantFileLock(to_folder()) if to_folder is not None else self.lock / "unzip" / __version__ # ty: ignore[call-non-callable]
  62. with root.lock_for_key(path.name):
  63. dest = root.path / path.name
  64. if not dest.exists():
  65. extract(path, dest)
  66. yield dest
  67. @property
  68. def py_info_at(self) -> ReentrantFileLock:
  69. return self.lock / "py_info" / "4" # ty: ignore[invalid-return-type]
  70. def py_info(self, path: Path) -> PyInfoStoreDisk:
  71. return PyInfoStoreDisk(self.py_info_at, path)
  72. def py_info_clear(self) -> None:
  73. """clear py info."""
  74. py_info_folder = self.py_info_at
  75. with py_info_folder:
  76. for filename in py_info_folder.path.iterdir():
  77. if filename.suffix == ".json":
  78. with py_info_folder.lock_for_key(filename.stem):
  79. if filename.exists():
  80. filename.unlink()
  81. def embed_update_log(self, distribution: str, for_py_version: str) -> EmbedDistributionUpdateStoreDisk:
  82. return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "3", distribution) # ty: ignore[invalid-argument-type]
  83. @property
  84. def house(self) -> Path:
  85. path = self.lock.path / "wheel" / "house"
  86. path.mkdir(parents=True, exist_ok=True)
  87. return path
  88. def wheel_image(self, for_py_version: str, name: str) -> Path:
  89. return self.lock.path / "wheel" / for_py_version / "image" / "1" / name
  90. class JSONStoreDisk(ContentStore, ABC):
  91. def __init__(self, in_folder: ReentrantFileLock, key: str, msg_args: tuple[str, ...]) -> None:
  92. self.in_folder = in_folder
  93. self.key = key
  94. self.msg_args = (*msg_args, self.file)
  95. @property
  96. def file(self) -> Path:
  97. return self.in_folder.path / f"{self.key}.json"
  98. def exists(self) -> bool:
  99. return self.file.exists()
  100. def read(self) -> Any: # noqa: ANN401
  101. data, bad_format = None, False
  102. try:
  103. data = json.loads(self.file.read_text(encoding="utf-8"))
  104. except ValueError:
  105. bad_format = True
  106. except Exception: # noqa: BLE001, S110
  107. pass
  108. else:
  109. LOGGER.debug("got %s %s from %s", *self.msg_args)
  110. return data
  111. if bad_format:
  112. with suppress(OSError): # reading and writing on the same file may cause race on multiple processes
  113. self.remove()
  114. return None
  115. def remove(self) -> None:
  116. self.file.unlink()
  117. LOGGER.debug("removed %s %s at %s", *self.msg_args)
  118. @contextmanager
  119. def locked(self) -> Generator[None]:
  120. with self.in_folder.lock_for_key(self.key):
  121. yield
  122. def write(self, content: Any) -> None: # noqa: ANN401
  123. folder = self.file.parent
  124. folder.mkdir(parents=True, exist_ok=True)
  125. self.file.write_text(json.dumps(content, sort_keys=True, indent=2), encoding="utf-8")
  126. LOGGER.debug("wrote %s %s at %s", *self.msg_args)
  127. class PyInfoStoreDisk(JSONStoreDisk):
  128. def __init__(self, in_folder: ReentrantFileLock, path: Path) -> None:
  129. key = sha256(str(path).encode("utf-8")).hexdigest()
  130. super().__init__(in_folder, key, ("python info of", path)) # ty: ignore[invalid-argument-type]
  131. class EmbedDistributionUpdateStoreDisk(JSONStoreDisk):
  132. def __init__(self, in_folder: ReentrantFileLock, distribution: str) -> None:
  133. super().__init__(
  134. in_folder,
  135. distribution,
  136. ("embed update of distribution", distribution),
  137. )
  138. __all__ = [
  139. "AppDataDiskFolder",
  140. "JSONStoreDisk",
  141. "PyInfoStoreDisk",
  142. ]