_manylinux.py 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260
  1. import collections
  2. import contextlib
  3. import functools
  4. import os
  5. import re
  6. import sys
  7. import warnings
  8. from typing import Dict, Generator, Iterator, NamedTuple, Optional, Sequence, Tuple
  9. from ._elffile import EIClass, EIData, ELFFile, EMachine
  10. EF_ARM_ABIMASK = 0xFF000000
  11. EF_ARM_ABI_VER5 = 0x05000000
  12. EF_ARM_ABI_FLOAT_HARD = 0x00000400
  13. # `os.PathLike` not a generic type until Python 3.9, so sticking with `str`
  14. # as the type for `path` until then.
  15. @contextlib.contextmanager
  16. def _parse_elf(path: str) -> Generator[Optional[ELFFile], None, None]:
  17. try:
  18. with open(path, "rb") as f:
  19. yield ELFFile(f)
  20. except (OSError, TypeError, ValueError):
  21. yield None
  22. def _is_linux_armhf(executable: str) -> bool:
  23. # hard-float ABI can be detected from the ELF header of the running
  24. # process
  25. # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf
  26. with _parse_elf(executable) as f:
  27. return (
  28. f is not None
  29. and f.capacity == EIClass.C32
  30. and f.encoding == EIData.Lsb
  31. and f.machine == EMachine.Arm
  32. and f.flags & EF_ARM_ABIMASK == EF_ARM_ABI_VER5
  33. and f.flags & EF_ARM_ABI_FLOAT_HARD == EF_ARM_ABI_FLOAT_HARD
  34. )
  35. def _is_linux_i686(executable: str) -> bool:
  36. with _parse_elf(executable) as f:
  37. return (
  38. f is not None
  39. and f.capacity == EIClass.C32
  40. and f.encoding == EIData.Lsb
  41. and f.machine == EMachine.I386
  42. )
  43. def _have_compatible_abi(executable: str, archs: Sequence[str]) -> bool:
  44. if "armv7l" in archs:
  45. return _is_linux_armhf(executable)
  46. if "i686" in archs:
  47. return _is_linux_i686(executable)
  48. allowed_archs = {
  49. "x86_64",
  50. "aarch64",
  51. "ppc64",
  52. "ppc64le",
  53. "s390x",
  54. "loongarch64",
  55. "riscv64",
  56. }
  57. return any(arch in allowed_archs for arch in archs)
  58. # If glibc ever changes its major version, we need to know what the last
  59. # minor version was, so we can build the complete list of all versions.
  60. # For now, guess what the highest minor version might be, assume it will
  61. # be 50 for testing. Once this actually happens, update the dictionary
  62. # with the actual value.
  63. _LAST_GLIBC_MINOR: Dict[int, int] = collections.defaultdict(lambda: 50)
  64. class _GLibCVersion(NamedTuple):
  65. major: int
  66. minor: int
  67. def _glibc_version_string_confstr() -> Optional[str]:
  68. """
  69. Primary implementation of glibc_version_string using os.confstr.
  70. """
  71. # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely
  72. # to be broken or missing. This strategy is used in the standard library
  73. # platform module.
  74. # https://github.com/python/cpython/blob/fcf1d003bf4f0100c/Lib/platform.py#L175-L183
  75. try:
  76. # Should be a string like "glibc 2.17".
  77. version_string: Optional[str] = os.confstr("CS_GNU_LIBC_VERSION")
  78. assert version_string is not None
  79. _, version = version_string.rsplit()
  80. except (AssertionError, AttributeError, OSError, ValueError):
  81. # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)...
  82. return None
  83. return version
  84. def _glibc_version_string_ctypes() -> Optional[str]:
  85. """
  86. Fallback implementation of glibc_version_string using ctypes.
  87. """
  88. try:
  89. import ctypes
  90. except ImportError:
  91. return None
  92. # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen
  93. # manpage says, "If filename is NULL, then the returned handle is for the
  94. # main program". This way we can let the linker do the work to figure out
  95. # which libc our process is actually using.
  96. #
  97. # We must also handle the special case where the executable is not a
  98. # dynamically linked executable. This can occur when using musl libc,
  99. # for example. In this situation, dlopen() will error, leading to an
  100. # OSError. Interestingly, at least in the case of musl, there is no
  101. # errno set on the OSError. The single string argument used to construct
  102. # OSError comes from libc itself and is therefore not portable to
  103. # hard code here. In any case, failure to call dlopen() means we
  104. # can proceed, so we bail on our attempt.
  105. try:
  106. process_namespace = ctypes.CDLL(None)
  107. except OSError:
  108. return None
  109. try:
  110. gnu_get_libc_version = process_namespace.gnu_get_libc_version
  111. except AttributeError:
  112. # Symbol doesn't exist -> therefore, we are not linked to
  113. # glibc.
  114. return None
  115. # Call gnu_get_libc_version, which returns a string like "2.5"
  116. gnu_get_libc_version.restype = ctypes.c_char_p
  117. version_str: str = gnu_get_libc_version()
  118. # py2 / py3 compatibility:
  119. if not isinstance(version_str, str):
  120. version_str = version_str.decode("ascii")
  121. return version_str
  122. def _glibc_version_string() -> Optional[str]:
  123. """Returns glibc version string, or None if not using glibc."""
  124. return _glibc_version_string_confstr() or _glibc_version_string_ctypes()
  125. def _parse_glibc_version(version_str: str) -> Tuple[int, int]:
  126. """Parse glibc version.
  127. We use a regexp instead of str.split because we want to discard any
  128. random junk that might come after the minor version -- this might happen
  129. in patched/forked versions of glibc (e.g. Linaro's version of glibc
  130. uses version strings like "2.20-2014.11"). See gh-3588.
  131. """
  132. m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str)
  133. if not m:
  134. warnings.warn(
  135. f"Expected glibc version with 2 components major.minor,"
  136. f" got: {version_str}",
  137. RuntimeWarning,
  138. )
  139. return -1, -1
  140. return int(m.group("major")), int(m.group("minor"))
  141. @functools.lru_cache()
  142. def _get_glibc_version() -> Tuple[int, int]:
  143. version_str = _glibc_version_string()
  144. if version_str is None:
  145. return (-1, -1)
  146. return _parse_glibc_version(version_str)
  147. # From PEP 513, PEP 600
  148. def _is_compatible(arch: str, version: _GLibCVersion) -> bool:
  149. sys_glibc = _get_glibc_version()
  150. if sys_glibc < version:
  151. return False
  152. # Check for presence of _manylinux module.
  153. try:
  154. import _manylinux
  155. except ImportError:
  156. return True
  157. if hasattr(_manylinux, "manylinux_compatible"):
  158. result = _manylinux.manylinux_compatible(version[0], version[1], arch)
  159. if result is not None:
  160. return bool(result)
  161. return True
  162. if version == _GLibCVersion(2, 5):
  163. if hasattr(_manylinux, "manylinux1_compatible"):
  164. return bool(_manylinux.manylinux1_compatible)
  165. if version == _GLibCVersion(2, 12):
  166. if hasattr(_manylinux, "manylinux2010_compatible"):
  167. return bool(_manylinux.manylinux2010_compatible)
  168. if version == _GLibCVersion(2, 17):
  169. if hasattr(_manylinux, "manylinux2014_compatible"):
  170. return bool(_manylinux.manylinux2014_compatible)
  171. return True
  172. _LEGACY_MANYLINUX_MAP = {
  173. # CentOS 7 w/ glibc 2.17 (PEP 599)
  174. (2, 17): "manylinux2014",
  175. # CentOS 6 w/ glibc 2.12 (PEP 571)
  176. (2, 12): "manylinux2010",
  177. # CentOS 5 w/ glibc 2.5 (PEP 513)
  178. (2, 5): "manylinux1",
  179. }
  180. def platform_tags(archs: Sequence[str]) -> Iterator[str]:
  181. """Generate manylinux tags compatible to the current platform.
  182. :param archs: Sequence of compatible architectures.
  183. The first one shall be the closest to the actual architecture and be the part of
  184. platform tag after the ``linux_`` prefix, e.g. ``x86_64``.
  185. The ``linux_`` prefix is assumed as a prerequisite for the current platform to
  186. be manylinux-compatible.
  187. :returns: An iterator of compatible manylinux tags.
  188. """
  189. if not _have_compatible_abi(sys.executable, archs):
  190. return
  191. # Oldest glibc to be supported regardless of architecture is (2, 17).
  192. too_old_glibc2 = _GLibCVersion(2, 16)
  193. if set(archs) & {"x86_64", "i686"}:
  194. # On x86/i686 also oldest glibc to be supported is (2, 5).
  195. too_old_glibc2 = _GLibCVersion(2, 4)
  196. current_glibc = _GLibCVersion(*_get_glibc_version())
  197. glibc_max_list = [current_glibc]
  198. # We can assume compatibility across glibc major versions.
  199. # https://sourceware.org/bugzilla/show_bug.cgi?id=24636
  200. #
  201. # Build a list of maximum glibc versions so that we can
  202. # output the canonical list of all glibc from current_glibc
  203. # down to too_old_glibc2, including all intermediary versions.
  204. for glibc_major in range(current_glibc.major - 1, 1, -1):
  205. glibc_minor = _LAST_GLIBC_MINOR[glibc_major]
  206. glibc_max_list.append(_GLibCVersion(glibc_major, glibc_minor))
  207. for arch in archs:
  208. for glibc_max in glibc_max_list:
  209. if glibc_max.major == too_old_glibc2.major:
  210. min_minor = too_old_glibc2.minor
  211. else:
  212. # For other glibc major versions oldest supported is (x, 0).
  213. min_minor = -1
  214. for glibc_minor in range(glibc_max.minor, min_minor, -1):
  215. glibc_version = _GLibCVersion(glibc_max.major, glibc_minor)
  216. tag = "manylinux_{}_{}".format(*glibc_version)
  217. if _is_compatible(arch, glibc_version):
  218. yield f"{tag}_{arch}"
  219. # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags.
  220. if glibc_version in _LEGACY_MANYLINUX_MAP:
  221. legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version]
  222. if _is_compatible(arch, glibc_version):
  223. yield f"{legacy_tag}_{arch}"