_metadata.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184
  1. """
  2. Tools for converting old- to new-style metadata.
  3. """
  4. from __future__ import annotations
  5. import functools
  6. import itertools
  7. import os.path
  8. import re
  9. import textwrap
  10. from collections.abc import Generator, Iterable, Iterator
  11. from email.message import Message
  12. from email.parser import Parser
  13. from typing import Literal
  14. from packaging.requirements import Requirement
  15. def _nonblank(str: str) -> bool | Literal[""]:
  16. return str and not str.startswith("#")
  17. @functools.singledispatch
  18. def yield_lines(iterable: Iterable[str]) -> Iterator[str]:
  19. r"""
  20. Yield valid lines of a string or iterable.
  21. >>> list(yield_lines(''))
  22. []
  23. >>> list(yield_lines(['foo', 'bar']))
  24. ['foo', 'bar']
  25. >>> list(yield_lines('foo\nbar'))
  26. ['foo', 'bar']
  27. >>> list(yield_lines('\nfoo\n#bar\nbaz #comment'))
  28. ['foo', 'baz #comment']
  29. >>> list(yield_lines(['foo\nbar', 'baz', 'bing\n\n\n']))
  30. ['foo', 'bar', 'baz', 'bing']
  31. """
  32. return itertools.chain.from_iterable(map(yield_lines, iterable))
  33. @yield_lines.register(str)
  34. def _(text: str) -> Iterator[str]:
  35. return filter(_nonblank, map(str.strip, text.splitlines()))
  36. def split_sections(
  37. s: str | Iterator[str],
  38. ) -> Generator[tuple[str | None, list[str]], None, None]:
  39. """Split a string or iterable thereof into (section, content) pairs
  40. Each ``section`` is a stripped version of the section header ("[section]")
  41. and each ``content`` is a list of stripped lines excluding blank lines and
  42. comment-only lines. If there are any such lines before the first section
  43. header, they're returned in a first ``section`` of ``None``.
  44. """
  45. section = None
  46. content: list[str] = []
  47. for line in yield_lines(s):
  48. if line.startswith("["):
  49. if line.endswith("]"):
  50. if section or content:
  51. yield section, content
  52. section = line[1:-1].strip()
  53. content = []
  54. else:
  55. raise ValueError("Invalid section heading", line)
  56. else:
  57. content.append(line)
  58. # wrap up last segment
  59. yield section, content
  60. def safe_extra(extra: str) -> str:
  61. """Convert an arbitrary string to a standard 'extra' name
  62. Any runs of non-alphanumeric characters are replaced with a single '_',
  63. and the result is always lowercased.
  64. """
  65. return re.sub("[^A-Za-z0-9.-]+", "_", extra).lower()
  66. def safe_name(name: str) -> str:
  67. """Convert an arbitrary string to a standard distribution name
  68. Any runs of non-alphanumeric/. characters are replaced with a single '-'.
  69. """
  70. return re.sub("[^A-Za-z0-9.]+", "-", name)
  71. def requires_to_requires_dist(requirement: Requirement) -> str:
  72. """Return the version specifier for a requirement in PEP 345/566 fashion."""
  73. if requirement.url:
  74. return " @ " + requirement.url
  75. requires_dist: list[str] = []
  76. for spec in requirement.specifier:
  77. requires_dist.append(spec.operator + spec.version)
  78. if requires_dist:
  79. return " " + ",".join(sorted(requires_dist))
  80. else:
  81. return ""
  82. def convert_requirements(requirements: list[str]) -> Iterator[str]:
  83. """Yield Requires-Dist: strings for parsed requirements strings."""
  84. for req in requirements:
  85. parsed_requirement = Requirement(req)
  86. spec = requires_to_requires_dist(parsed_requirement)
  87. extras = ",".join(sorted(safe_extra(e) for e in parsed_requirement.extras))
  88. if extras:
  89. extras = f"[{extras}]"
  90. yield safe_name(parsed_requirement.name) + extras + spec
  91. def generate_requirements(
  92. extras_require: dict[str | None, list[str]],
  93. ) -> Iterator[tuple[str, str]]:
  94. """
  95. Convert requirements from a setup()-style dictionary to
  96. ('Requires-Dist', 'requirement') and ('Provides-Extra', 'extra') tuples.
  97. extras_require is a dictionary of {extra: [requirements]} as passed to setup(),
  98. using the empty extra {'': [requirements]} to hold install_requires.
  99. """
  100. for extra, depends in extras_require.items():
  101. condition = ""
  102. extra = extra or ""
  103. if ":" in extra: # setuptools extra:condition syntax
  104. extra, condition = extra.split(":", 1)
  105. extra = safe_extra(extra)
  106. if extra:
  107. yield "Provides-Extra", extra
  108. if condition:
  109. condition = "(" + condition + ") and "
  110. condition += f"extra == '{extra}'"
  111. if condition:
  112. condition = " ; " + condition
  113. for new_req in convert_requirements(depends):
  114. canonical_req = str(Requirement(new_req + condition))
  115. yield "Requires-Dist", canonical_req
  116. def pkginfo_to_metadata(egg_info_path: str, pkginfo_path: str) -> Message:
  117. """
  118. Convert .egg-info directory with PKG-INFO to the Metadata 2.1 format
  119. """
  120. with open(pkginfo_path, encoding="utf-8") as headers:
  121. pkg_info = Parser().parse(headers)
  122. pkg_info.replace_header("Metadata-Version", "2.1")
  123. # Those will be regenerated from `requires.txt`.
  124. del pkg_info["Provides-Extra"]
  125. del pkg_info["Requires-Dist"]
  126. requires_path = os.path.join(egg_info_path, "requires.txt")
  127. if os.path.exists(requires_path):
  128. with open(requires_path, encoding="utf-8") as requires_file:
  129. requires = requires_file.read()
  130. parsed_requirements = sorted(split_sections(requires), key=lambda x: x[0] or "")
  131. for extra, reqs in parsed_requirements:
  132. for key, value in generate_requirements({extra: reqs}):
  133. if (key, value) not in pkg_info.items():
  134. pkg_info[key] = value
  135. description = pkg_info["Description"]
  136. if description:
  137. description_lines = pkg_info["Description"].splitlines()
  138. dedented_description = "\n".join(
  139. # if the first line of long_description is blank,
  140. # the first line here will be indented.
  141. (
  142. description_lines[0].lstrip(),
  143. textwrap.dedent("\n".join(description_lines[1:])),
  144. "\n",
  145. )
  146. )
  147. pkg_info.set_payload(dedented_description)
  148. del pkg_info["Description"]
  149. return pkg_info