util.py 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312
  1. from collections import namedtuple
  2. from hashlib import sha256
  3. import os
  4. import shutil
  5. import sys
  6. import fnmatch
  7. from sympy.testing.pytest import XFAIL
  8. def may_xfail(func):
  9. if sys.platform.lower() == 'darwin' or os.name == 'nt':
  10. # sympy.utilities._compilation needs more testing on Windows and macOS
  11. # once those two platforms are reliably supported this xfail decorator
  12. # may be removed.
  13. return XFAIL(func)
  14. else:
  15. return func
  16. class CompilerNotFoundError(FileNotFoundError):
  17. pass
  18. class CompileError (Exception):
  19. """Failure to compile one or more C/C++ source files."""
  20. def get_abspath(path, cwd='.'):
  21. """ Returns the absolute path.
  22. Parameters
  23. ==========
  24. path : str
  25. (relative) path.
  26. cwd : str
  27. Path to root of relative path.
  28. """
  29. if os.path.isabs(path):
  30. return path
  31. else:
  32. if not os.path.isabs(cwd):
  33. cwd = os.path.abspath(cwd)
  34. return os.path.abspath(
  35. os.path.join(cwd, path)
  36. )
  37. def make_dirs(path):
  38. """ Create directories (equivalent of ``mkdir -p``). """
  39. if path[-1] == '/':
  40. parent = os.path.dirname(path[:-1])
  41. else:
  42. parent = os.path.dirname(path)
  43. if len(parent) > 0:
  44. if not os.path.exists(parent):
  45. make_dirs(parent)
  46. if not os.path.exists(path):
  47. os.mkdir(path, 0o777)
  48. else:
  49. assert os.path.isdir(path)
  50. def missing_or_other_newer(path, other_path, cwd=None):
  51. """
  52. Investigate if path is non-existent or older than provided reference
  53. path.
  54. Parameters
  55. ==========
  56. path: string
  57. path to path which might be missing or too old
  58. other_path: string
  59. reference path
  60. cwd: string
  61. working directory (root of relative paths)
  62. Returns
  63. =======
  64. True if path is older or missing.
  65. """
  66. cwd = cwd or '.'
  67. path = get_abspath(path, cwd=cwd)
  68. other_path = get_abspath(other_path, cwd=cwd)
  69. if not os.path.exists(path):
  70. return True
  71. if os.path.getmtime(other_path) - 1e-6 >= os.path.getmtime(path):
  72. # 1e-6 is needed because http://stackoverflow.com/questions/17086426/
  73. return True
  74. return False
  75. def copy(src, dst, only_update=False, copystat=True, cwd=None,
  76. dest_is_dir=False, create_dest_dirs=False):
  77. """ Variation of ``shutil.copy`` with extra options.
  78. Parameters
  79. ==========
  80. src : str
  81. Path to source file.
  82. dst : str
  83. Path to destination.
  84. only_update : bool
  85. Only copy if source is newer than destination
  86. (returns None if it was newer), default: ``False``.
  87. copystat : bool
  88. See ``shutil.copystat``. default: ``True``.
  89. cwd : str
  90. Path to working directory (root of relative paths).
  91. dest_is_dir : bool
  92. Ensures that dst is treated as a directory. default: ``False``
  93. create_dest_dirs : bool
  94. Creates directories if needed.
  95. Returns
  96. =======
  97. Path to the copied file.
  98. """
  99. if cwd: # Handle working directory
  100. if not os.path.isabs(src):
  101. src = os.path.join(cwd, src)
  102. if not os.path.isabs(dst):
  103. dst = os.path.join(cwd, dst)
  104. if not os.path.exists(src): # Make sure source file exists
  105. raise FileNotFoundError("Source: `{}` does not exist".format(src))
  106. # We accept both (re)naming destination file _or_
  107. # passing a (possible non-existent) destination directory
  108. if dest_is_dir:
  109. if not dst[-1] == '/':
  110. dst = dst+'/'
  111. else:
  112. if os.path.exists(dst) and os.path.isdir(dst):
  113. dest_is_dir = True
  114. if dest_is_dir:
  115. dest_dir = dst
  116. dest_fname = os.path.basename(src)
  117. dst = os.path.join(dest_dir, dest_fname)
  118. else:
  119. dest_dir = os.path.dirname(dst)
  120. if not os.path.exists(dest_dir):
  121. if create_dest_dirs:
  122. make_dirs(dest_dir)
  123. else:
  124. raise FileNotFoundError("You must create directory first.")
  125. if only_update:
  126. if not missing_or_other_newer(dst, src):
  127. return
  128. if os.path.islink(dst):
  129. dst = os.path.abspath(os.path.realpath(dst), cwd=cwd)
  130. shutil.copy(src, dst)
  131. if copystat:
  132. shutil.copystat(src, dst)
  133. return dst
  134. Glob = namedtuple('Glob', 'pathname')
  135. ArbitraryDepthGlob = namedtuple('ArbitraryDepthGlob', 'filename')
  136. def glob_at_depth(filename_glob, cwd=None):
  137. if cwd is not None:
  138. cwd = '.'
  139. globbed = []
  140. for root, dirs, filenames in os.walk(cwd):
  141. for fn in filenames:
  142. # This is not tested:
  143. if fnmatch.fnmatch(fn, filename_glob):
  144. globbed.append(os.path.join(root, fn))
  145. return globbed
  146. def sha256_of_file(path, nblocks=128):
  147. """ Computes the SHA256 hash of a file.
  148. Parameters
  149. ==========
  150. path : string
  151. Path to file to compute hash of.
  152. nblocks : int
  153. Number of blocks to read per iteration.
  154. Returns
  155. =======
  156. hashlib sha256 hash object. Use ``.digest()`` or ``.hexdigest()``
  157. on returned object to get binary or hex encoded string.
  158. """
  159. sh = sha256()
  160. with open(path, 'rb') as f:
  161. for chunk in iter(lambda: f.read(nblocks*sh.block_size), b''):
  162. sh.update(chunk)
  163. return sh
  164. def sha256_of_string(string):
  165. """ Computes the SHA256 hash of a string. """
  166. sh = sha256()
  167. sh.update(string)
  168. return sh
  169. def pyx_is_cplus(path):
  170. """
  171. Inspect a Cython source file (.pyx) and look for comment line like:
  172. # distutils: language = c++
  173. Returns True if such a file is present in the file, else False.
  174. """
  175. with open(path) as fh:
  176. for line in fh:
  177. if line.startswith('#') and '=' in line:
  178. splitted = line.split('=')
  179. if len(splitted) != 2:
  180. continue
  181. lhs, rhs = splitted
  182. if lhs.strip().split()[-1].lower() == 'language' and \
  183. rhs.strip().split()[0].lower() == 'c++':
  184. return True
  185. return False
  186. def import_module_from_file(filename, only_if_newer_than=None):
  187. """ Imports Python extension (from shared object file)
  188. Provide a list of paths in `only_if_newer_than` to check
  189. timestamps of dependencies. import_ raises an ImportError
  190. if any is newer.
  191. Word of warning: The OS may cache shared objects which makes
  192. reimporting same path of an shared object file very problematic.
  193. It will not detect the new time stamp, nor new checksum, but will
  194. instead silently use old module. Use unique names for this reason.
  195. Parameters
  196. ==========
  197. filename : str
  198. Path to shared object.
  199. only_if_newer_than : iterable of strings
  200. Paths to dependencies of the shared object.
  201. Raises
  202. ======
  203. ``ImportError`` if any of the files specified in ``only_if_newer_than`` are newer
  204. than the file given by filename.
  205. """
  206. path, name = os.path.split(filename)
  207. name, ext = os.path.splitext(name)
  208. name = name.split('.')[0]
  209. if sys.version_info[0] == 2:
  210. from imp import find_module, load_module
  211. fobj, filename, data = find_module(name, [path])
  212. if only_if_newer_than:
  213. for dep in only_if_newer_than:
  214. if os.path.getmtime(filename) < os.path.getmtime(dep):
  215. raise ImportError("{} is newer than {}".format(dep, filename))
  216. mod = load_module(name, fobj, filename, data)
  217. else:
  218. import importlib.util
  219. spec = importlib.util.spec_from_file_location(name, filename)
  220. if spec is None:
  221. raise ImportError("Failed to import: '%s'" % filename)
  222. mod = importlib.util.module_from_spec(spec)
  223. spec.loader.exec_module(mod)
  224. return mod
  225. def find_binary_of_command(candidates):
  226. """ Finds binary first matching name among candidates.
  227. Calls ``which`` from shutils for provided candidates and returns
  228. first hit.
  229. Parameters
  230. ==========
  231. candidates : iterable of str
  232. Names of candidate commands
  233. Raises
  234. ======
  235. CompilerNotFoundError if no candidates match.
  236. """
  237. from shutil import which
  238. for c in candidates:
  239. binary_path = which(c)
  240. if c and binary_path:
  241. return c, binary_path
  242. raise CompilerNotFoundError('No binary located for candidates: {}'.format(candidates))
  243. def unique_list(l):
  244. """ Uniquify a list (skip duplicate items). """
  245. result = []
  246. for x in l:
  247. if x not in result:
  248. result.append(x)
  249. return result