_internal.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963
  1. """
  2. A place for internal code
  3. Some things are more easily handled Python.
  4. """
  5. import ast
  6. import math
  7. import re
  8. import sys
  9. import warnings
  10. from ..exceptions import DTypePromotionError
  11. from .multiarray import dtype, array, ndarray, promote_types, StringDType
  12. from numpy import _NoValue
  13. try:
  14. import ctypes
  15. except ImportError:
  16. ctypes = None
  17. IS_PYPY = sys.implementation.name == 'pypy'
  18. if sys.byteorder == 'little':
  19. _nbo = '<'
  20. else:
  21. _nbo = '>'
  22. def _makenames_list(adict, align):
  23. allfields = []
  24. for fname, obj in adict.items():
  25. n = len(obj)
  26. if not isinstance(obj, tuple) or n not in (2, 3):
  27. raise ValueError("entry not a 2- or 3- tuple")
  28. if n > 2 and obj[2] == fname:
  29. continue
  30. num = int(obj[1])
  31. if num < 0:
  32. raise ValueError("invalid offset.")
  33. format = dtype(obj[0], align=align)
  34. if n > 2:
  35. title = obj[2]
  36. else:
  37. title = None
  38. allfields.append((fname, format, num, title))
  39. # sort by offsets
  40. allfields.sort(key=lambda x: x[2])
  41. names = [x[0] for x in allfields]
  42. formats = [x[1] for x in allfields]
  43. offsets = [x[2] for x in allfields]
  44. titles = [x[3] for x in allfields]
  45. return names, formats, offsets, titles
  46. # Called in PyArray_DescrConverter function when
  47. # a dictionary without "names" and "formats"
  48. # fields is used as a data-type descriptor.
  49. def _usefields(adict, align):
  50. try:
  51. names = adict[-1]
  52. except KeyError:
  53. names = None
  54. if names is None:
  55. names, formats, offsets, titles = _makenames_list(adict, align)
  56. else:
  57. formats = []
  58. offsets = []
  59. titles = []
  60. for name in names:
  61. res = adict[name]
  62. formats.append(res[0])
  63. offsets.append(res[1])
  64. if len(res) > 2:
  65. titles.append(res[2])
  66. else:
  67. titles.append(None)
  68. return dtype({"names": names,
  69. "formats": formats,
  70. "offsets": offsets,
  71. "titles": titles}, align)
  72. # construct an array_protocol descriptor list
  73. # from the fields attribute of a descriptor
  74. # This calls itself recursively but should eventually hit
  75. # a descriptor that has no fields and then return
  76. # a simple typestring
  77. def _array_descr(descriptor):
  78. fields = descriptor.fields
  79. if fields is None:
  80. subdtype = descriptor.subdtype
  81. if subdtype is None:
  82. if descriptor.metadata is None:
  83. return descriptor.str
  84. else:
  85. new = descriptor.metadata.copy()
  86. if new:
  87. return (descriptor.str, new)
  88. else:
  89. return descriptor.str
  90. else:
  91. return (_array_descr(subdtype[0]), subdtype[1])
  92. names = descriptor.names
  93. ordered_fields = [fields[x] + (x,) for x in names]
  94. result = []
  95. offset = 0
  96. for field in ordered_fields:
  97. if field[1] > offset:
  98. num = field[1] - offset
  99. result.append(('', f'|V{num}'))
  100. offset += num
  101. elif field[1] < offset:
  102. raise ValueError(
  103. "dtype.descr is not defined for types with overlapping or "
  104. "out-of-order fields")
  105. if len(field) > 3:
  106. name = (field[2], field[3])
  107. else:
  108. name = field[2]
  109. if field[0].subdtype:
  110. tup = (name, _array_descr(field[0].subdtype[0]),
  111. field[0].subdtype[1])
  112. else:
  113. tup = (name, _array_descr(field[0]))
  114. offset += field[0].itemsize
  115. result.append(tup)
  116. if descriptor.itemsize > offset:
  117. num = descriptor.itemsize - offset
  118. result.append(('', f'|V{num}'))
  119. return result
  120. # format_re was originally from numarray by J. Todd Miller
  121. format_re = re.compile(r'(?P<order1>[<>|=]?)'
  122. r'(?P<repeats> *[(]?[ ,0-9]*[)]? *)'
  123. r'(?P<order2>[<>|=]?)'
  124. r'(?P<dtype>[A-Za-z0-9.?]*(?:\[[a-zA-Z0-9,.]+\])?)')
  125. sep_re = re.compile(r'\s*,\s*')
  126. space_re = re.compile(r'\s+$')
  127. # astr is a string (perhaps comma separated)
  128. _convorder = {'=': _nbo}
  129. def _commastring(astr):
  130. startindex = 0
  131. result = []
  132. islist = False
  133. while startindex < len(astr):
  134. mo = format_re.match(astr, pos=startindex)
  135. try:
  136. (order1, repeats, order2, dtype) = mo.groups()
  137. except (TypeError, AttributeError):
  138. raise ValueError(
  139. f'format number {len(result)+1} of "{astr}" is not recognized'
  140. ) from None
  141. startindex = mo.end()
  142. # Separator or ending padding
  143. if startindex < len(astr):
  144. if space_re.match(astr, pos=startindex):
  145. startindex = len(astr)
  146. else:
  147. mo = sep_re.match(astr, pos=startindex)
  148. if not mo:
  149. raise ValueError(
  150. 'format number %d of "%s" is not recognized' %
  151. (len(result)+1, astr))
  152. startindex = mo.end()
  153. islist = True
  154. if order2 == '':
  155. order = order1
  156. elif order1 == '':
  157. order = order2
  158. else:
  159. order1 = _convorder.get(order1, order1)
  160. order2 = _convorder.get(order2, order2)
  161. if (order1 != order2):
  162. raise ValueError(
  163. 'inconsistent byte-order specification %s and %s' %
  164. (order1, order2))
  165. order = order1
  166. if order in ('|', '=', _nbo):
  167. order = ''
  168. dtype = order + dtype
  169. if repeats == '':
  170. newitem = dtype
  171. else:
  172. if (repeats[0] == "(" and repeats[-1] == ")"
  173. and repeats[1:-1].strip() != ""
  174. and "," not in repeats):
  175. warnings.warn(
  176. 'Passing in a parenthesized single number for repeats '
  177. 'is deprecated; pass either a single number or indicate '
  178. 'a tuple with a comma, like "(2,)".', DeprecationWarning,
  179. stacklevel=2)
  180. newitem = (dtype, ast.literal_eval(repeats))
  181. result.append(newitem)
  182. return result if islist else result[0]
  183. class dummy_ctype:
  184. def __init__(self, cls):
  185. self._cls = cls
  186. def __mul__(self, other):
  187. return self
  188. def __call__(self, *other):
  189. return self._cls(other)
  190. def __eq__(self, other):
  191. return self._cls == other._cls
  192. def __ne__(self, other):
  193. return self._cls != other._cls
  194. def _getintp_ctype():
  195. val = _getintp_ctype.cache
  196. if val is not None:
  197. return val
  198. if ctypes is None:
  199. import numpy as np
  200. val = dummy_ctype(np.intp)
  201. else:
  202. char = dtype('n').char
  203. if char == 'i':
  204. val = ctypes.c_int
  205. elif char == 'l':
  206. val = ctypes.c_long
  207. elif char == 'q':
  208. val = ctypes.c_longlong
  209. else:
  210. val = ctypes.c_long
  211. _getintp_ctype.cache = val
  212. return val
  213. _getintp_ctype.cache = None
  214. # Used for .ctypes attribute of ndarray
  215. class _missing_ctypes:
  216. def cast(self, num, obj):
  217. return num.value
  218. class c_void_p:
  219. def __init__(self, ptr):
  220. self.value = ptr
  221. class _ctypes:
  222. def __init__(self, array, ptr=None):
  223. self._arr = array
  224. if ctypes:
  225. self._ctypes = ctypes
  226. self._data = self._ctypes.c_void_p(ptr)
  227. else:
  228. # fake a pointer-like object that holds onto the reference
  229. self._ctypes = _missing_ctypes()
  230. self._data = self._ctypes.c_void_p(ptr)
  231. self._data._objects = array
  232. if self._arr.ndim == 0:
  233. self._zerod = True
  234. else:
  235. self._zerod = False
  236. def data_as(self, obj):
  237. """
  238. Return the data pointer cast to a particular c-types object.
  239. For example, calling ``self._as_parameter_`` is equivalent to
  240. ``self.data_as(ctypes.c_void_p)``. Perhaps you want to use
  241. the data as a pointer to a ctypes array of floating-point data:
  242. ``self.data_as(ctypes.POINTER(ctypes.c_double))``.
  243. The returned pointer will keep a reference to the array.
  244. """
  245. # _ctypes.cast function causes a circular reference of self._data in
  246. # self._data._objects. Attributes of self._data cannot be released
  247. # until gc.collect is called. Make a copy of the pointer first then
  248. # let it hold the array reference. This is a workaround to circumvent
  249. # the CPython bug https://bugs.python.org/issue12836.
  250. ptr = self._ctypes.cast(self._data, obj)
  251. ptr._arr = self._arr
  252. return ptr
  253. def shape_as(self, obj):
  254. """
  255. Return the shape tuple as an array of some other c-types
  256. type. For example: ``self.shape_as(ctypes.c_short)``.
  257. """
  258. if self._zerod:
  259. return None
  260. return (obj*self._arr.ndim)(*self._arr.shape)
  261. def strides_as(self, obj):
  262. """
  263. Return the strides tuple as an array of some other
  264. c-types type. For example: ``self.strides_as(ctypes.c_longlong)``.
  265. """
  266. if self._zerod:
  267. return None
  268. return (obj*self._arr.ndim)(*self._arr.strides)
  269. @property
  270. def data(self):
  271. """
  272. A pointer to the memory area of the array as a Python integer.
  273. This memory area may contain data that is not aligned, or not in
  274. correct byte-order. The memory area may not even be writeable.
  275. The array flags and data-type of this array should be respected
  276. when passing this attribute to arbitrary C-code to avoid trouble
  277. that can include Python crashing. User Beware! The value of this
  278. attribute is exactly the same as:
  279. ``self._array_interface_['data'][0]``.
  280. Note that unlike ``data_as``, a reference won't be kept to the array:
  281. code like ``ctypes.c_void_p((a + b).ctypes.data)`` will result in a
  282. pointer to a deallocated array, and should be spelt
  283. ``(a + b).ctypes.data_as(ctypes.c_void_p)``
  284. """
  285. return self._data.value
  286. @property
  287. def shape(self):
  288. """
  289. (c_intp*self.ndim): A ctypes array of length self.ndim where
  290. the basetype is the C-integer corresponding to ``dtype('p')`` on this
  291. platform (see `~numpy.ctypeslib.c_intp`). This base-type could be
  292. `ctypes.c_int`, `ctypes.c_long`, or `ctypes.c_longlong` depending on
  293. the platform. The ctypes array contains the shape of
  294. the underlying array.
  295. """
  296. return self.shape_as(_getintp_ctype())
  297. @property
  298. def strides(self):
  299. """
  300. (c_intp*self.ndim): A ctypes array of length self.ndim where
  301. the basetype is the same as for the shape attribute. This ctypes
  302. array contains the strides information from the underlying array.
  303. This strides information is important for showing how many bytes
  304. must be jumped to get to the next element in the array.
  305. """
  306. return self.strides_as(_getintp_ctype())
  307. @property
  308. def _as_parameter_(self):
  309. """
  310. Overrides the ctypes semi-magic method
  311. Enables `c_func(some_array.ctypes)`
  312. """
  313. return self.data_as(ctypes.c_void_p)
  314. # Numpy 1.21.0, 2021-05-18
  315. def get_data(self):
  316. """Deprecated getter for the `_ctypes.data` property.
  317. .. deprecated:: 1.21
  318. """
  319. warnings.warn('"get_data" is deprecated. Use "data" instead',
  320. DeprecationWarning, stacklevel=2)
  321. return self.data
  322. def get_shape(self):
  323. """Deprecated getter for the `_ctypes.shape` property.
  324. .. deprecated:: 1.21
  325. """
  326. warnings.warn('"get_shape" is deprecated. Use "shape" instead',
  327. DeprecationWarning, stacklevel=2)
  328. return self.shape
  329. def get_strides(self):
  330. """Deprecated getter for the `_ctypes.strides` property.
  331. .. deprecated:: 1.21
  332. """
  333. warnings.warn('"get_strides" is deprecated. Use "strides" instead',
  334. DeprecationWarning, stacklevel=2)
  335. return self.strides
  336. def get_as_parameter(self):
  337. """Deprecated getter for the `_ctypes._as_parameter_` property.
  338. .. deprecated:: 1.21
  339. """
  340. warnings.warn(
  341. '"get_as_parameter" is deprecated. Use "_as_parameter_" instead',
  342. DeprecationWarning, stacklevel=2,
  343. )
  344. return self._as_parameter_
  345. def _newnames(datatype, order):
  346. """
  347. Given a datatype and an order object, return a new names tuple, with the
  348. order indicated
  349. """
  350. oldnames = datatype.names
  351. nameslist = list(oldnames)
  352. if isinstance(order, str):
  353. order = [order]
  354. seen = set()
  355. if isinstance(order, (list, tuple)):
  356. for name in order:
  357. try:
  358. nameslist.remove(name)
  359. except ValueError:
  360. if name in seen:
  361. raise ValueError(f"duplicate field name: {name}") from None
  362. else:
  363. raise ValueError(f"unknown field name: {name}") from None
  364. seen.add(name)
  365. return tuple(list(order) + nameslist)
  366. raise ValueError(f"unsupported order value: {order}")
  367. def _copy_fields(ary):
  368. """Return copy of structured array with padding between fields removed.
  369. Parameters
  370. ----------
  371. ary : ndarray
  372. Structured array from which to remove padding bytes
  373. Returns
  374. -------
  375. ary_copy : ndarray
  376. Copy of ary with padding bytes removed
  377. """
  378. dt = ary.dtype
  379. copy_dtype = {'names': dt.names,
  380. 'formats': [dt.fields[name][0] for name in dt.names]}
  381. return array(ary, dtype=copy_dtype, copy=True)
  382. def _promote_fields(dt1, dt2):
  383. """ Perform type promotion for two structured dtypes.
  384. Parameters
  385. ----------
  386. dt1 : structured dtype
  387. First dtype.
  388. dt2 : structured dtype
  389. Second dtype.
  390. Returns
  391. -------
  392. out : dtype
  393. The promoted dtype
  394. Notes
  395. -----
  396. If one of the inputs is aligned, the result will be. The titles of
  397. both descriptors must match (point to the same field).
  398. """
  399. # Both must be structured and have the same names in the same order
  400. if (dt1.names is None or dt2.names is None) or dt1.names != dt2.names:
  401. raise DTypePromotionError(
  402. f"field names `{dt1.names}` and `{dt2.names}` mismatch.")
  403. # if both are identical, we can (maybe!) just return the same dtype.
  404. identical = dt1 is dt2
  405. new_fields = []
  406. for name in dt1.names:
  407. field1 = dt1.fields[name]
  408. field2 = dt2.fields[name]
  409. new_descr = promote_types(field1[0], field2[0])
  410. identical = identical and new_descr is field1[0]
  411. # Check that the titles match (if given):
  412. if field1[2:] != field2[2:]:
  413. raise DTypePromotionError(
  414. f"field titles of field '{name}' mismatch")
  415. if len(field1) == 2:
  416. new_fields.append((name, new_descr))
  417. else:
  418. new_fields.append(((field1[2], name), new_descr))
  419. res = dtype(new_fields, align=dt1.isalignedstruct or dt2.isalignedstruct)
  420. # Might as well preserve identity (and metadata) if the dtype is identical
  421. # and the itemsize, offsets are also unmodified. This could probably be
  422. # sped up, but also probably just be removed entirely.
  423. if identical and res.itemsize == dt1.itemsize:
  424. for name in dt1.names:
  425. if dt1.fields[name][1] != res.fields[name][1]:
  426. return res # the dtype changed.
  427. return dt1
  428. return res
  429. def _getfield_is_safe(oldtype, newtype, offset):
  430. """ Checks safety of getfield for object arrays.
  431. As in _view_is_safe, we need to check that memory containing objects is not
  432. reinterpreted as a non-object datatype and vice versa.
  433. Parameters
  434. ----------
  435. oldtype : data-type
  436. Data type of the original ndarray.
  437. newtype : data-type
  438. Data type of the field being accessed by ndarray.getfield
  439. offset : int
  440. Offset of the field being accessed by ndarray.getfield
  441. Raises
  442. ------
  443. TypeError
  444. If the field access is invalid
  445. """
  446. if newtype.hasobject or oldtype.hasobject:
  447. if offset == 0 and newtype == oldtype:
  448. return
  449. if oldtype.names is not None:
  450. for name in oldtype.names:
  451. if (oldtype.fields[name][1] == offset and
  452. oldtype.fields[name][0] == newtype):
  453. return
  454. raise TypeError("Cannot get/set field of an object array")
  455. return
  456. def _view_is_safe(oldtype, newtype):
  457. """ Checks safety of a view involving object arrays, for example when
  458. doing::
  459. np.zeros(10, dtype=oldtype).view(newtype)
  460. Parameters
  461. ----------
  462. oldtype : data-type
  463. Data type of original ndarray
  464. newtype : data-type
  465. Data type of the view
  466. Raises
  467. ------
  468. TypeError
  469. If the new type is incompatible with the old type.
  470. """
  471. # if the types are equivalent, there is no problem.
  472. # for example: dtype((np.record, 'i4,i4')) == dtype((np.void, 'i4,i4'))
  473. if oldtype == newtype:
  474. return
  475. if newtype.hasobject or oldtype.hasobject:
  476. raise TypeError("Cannot change data-type for array of references.")
  477. return
  478. # Given a string containing a PEP 3118 format specifier,
  479. # construct a NumPy dtype
  480. _pep3118_native_map = {
  481. '?': '?',
  482. 'c': 'S1',
  483. 'b': 'b',
  484. 'B': 'B',
  485. 'h': 'h',
  486. 'H': 'H',
  487. 'i': 'i',
  488. 'I': 'I',
  489. 'l': 'l',
  490. 'L': 'L',
  491. 'q': 'q',
  492. 'Q': 'Q',
  493. 'e': 'e',
  494. 'f': 'f',
  495. 'd': 'd',
  496. 'g': 'g',
  497. 'Zf': 'F',
  498. 'Zd': 'D',
  499. 'Zg': 'G',
  500. 's': 'S',
  501. 'w': 'U',
  502. 'O': 'O',
  503. 'x': 'V', # padding
  504. }
  505. _pep3118_native_typechars = ''.join(_pep3118_native_map.keys())
  506. _pep3118_standard_map = {
  507. '?': '?',
  508. 'c': 'S1',
  509. 'b': 'b',
  510. 'B': 'B',
  511. 'h': 'i2',
  512. 'H': 'u2',
  513. 'i': 'i4',
  514. 'I': 'u4',
  515. 'l': 'i4',
  516. 'L': 'u4',
  517. 'q': 'i8',
  518. 'Q': 'u8',
  519. 'e': 'f2',
  520. 'f': 'f',
  521. 'd': 'd',
  522. 'Zf': 'F',
  523. 'Zd': 'D',
  524. 's': 'S',
  525. 'w': 'U',
  526. 'O': 'O',
  527. 'x': 'V', # padding
  528. }
  529. _pep3118_standard_typechars = ''.join(_pep3118_standard_map.keys())
  530. _pep3118_unsupported_map = {
  531. 'u': 'UCS-2 strings',
  532. '&': 'pointers',
  533. 't': 'bitfields',
  534. 'X': 'function pointers',
  535. }
  536. class _Stream:
  537. def __init__(self, s):
  538. self.s = s
  539. self.byteorder = '@'
  540. def advance(self, n):
  541. res = self.s[:n]
  542. self.s = self.s[n:]
  543. return res
  544. def consume(self, c):
  545. if self.s[:len(c)] == c:
  546. self.advance(len(c))
  547. return True
  548. return False
  549. def consume_until(self, c):
  550. if callable(c):
  551. i = 0
  552. while i < len(self.s) and not c(self.s[i]):
  553. i = i + 1
  554. return self.advance(i)
  555. else:
  556. i = self.s.index(c)
  557. res = self.advance(i)
  558. self.advance(len(c))
  559. return res
  560. @property
  561. def next(self):
  562. return self.s[0]
  563. def __bool__(self):
  564. return bool(self.s)
  565. def _dtype_from_pep3118(spec):
  566. stream = _Stream(spec)
  567. dtype, align = __dtype_from_pep3118(stream, is_subdtype=False)
  568. return dtype
  569. def __dtype_from_pep3118(stream, is_subdtype):
  570. field_spec = dict(
  571. names=[],
  572. formats=[],
  573. offsets=[],
  574. itemsize=0
  575. )
  576. offset = 0
  577. common_alignment = 1
  578. is_padding = False
  579. # Parse spec
  580. while stream:
  581. value = None
  582. # End of structure, bail out to upper level
  583. if stream.consume('}'):
  584. break
  585. # Sub-arrays (1)
  586. shape = None
  587. if stream.consume('('):
  588. shape = stream.consume_until(')')
  589. shape = tuple(map(int, shape.split(',')))
  590. # Byte order
  591. if stream.next in ('@', '=', '<', '>', '^', '!'):
  592. byteorder = stream.advance(1)
  593. if byteorder == '!':
  594. byteorder = '>'
  595. stream.byteorder = byteorder
  596. # Byte order characters also control native vs. standard type sizes
  597. if stream.byteorder in ('@', '^'):
  598. type_map = _pep3118_native_map
  599. type_map_chars = _pep3118_native_typechars
  600. else:
  601. type_map = _pep3118_standard_map
  602. type_map_chars = _pep3118_standard_typechars
  603. # Item sizes
  604. itemsize_str = stream.consume_until(lambda c: not c.isdigit())
  605. if itemsize_str:
  606. itemsize = int(itemsize_str)
  607. else:
  608. itemsize = 1
  609. # Data types
  610. is_padding = False
  611. if stream.consume('T{'):
  612. value, align = __dtype_from_pep3118(
  613. stream, is_subdtype=True)
  614. elif stream.next in type_map_chars:
  615. if stream.next == 'Z':
  616. typechar = stream.advance(2)
  617. else:
  618. typechar = stream.advance(1)
  619. is_padding = (typechar == 'x')
  620. dtypechar = type_map[typechar]
  621. if dtypechar in 'USV':
  622. dtypechar += '%d' % itemsize
  623. itemsize = 1
  624. numpy_byteorder = {'@': '=', '^': '='}.get(
  625. stream.byteorder, stream.byteorder)
  626. value = dtype(numpy_byteorder + dtypechar)
  627. align = value.alignment
  628. elif stream.next in _pep3118_unsupported_map:
  629. desc = _pep3118_unsupported_map[stream.next]
  630. raise NotImplementedError(
  631. "Unrepresentable PEP 3118 data type {!r} ({})"
  632. .format(stream.next, desc))
  633. else:
  634. raise ValueError(
  635. "Unknown PEP 3118 data type specifier %r" % stream.s
  636. )
  637. #
  638. # Native alignment may require padding
  639. #
  640. # Here we assume that the presence of a '@' character implicitly
  641. # implies that the start of the array is *already* aligned.
  642. #
  643. extra_offset = 0
  644. if stream.byteorder == '@':
  645. start_padding = (-offset) % align
  646. intra_padding = (-value.itemsize) % align
  647. offset += start_padding
  648. if intra_padding != 0:
  649. if itemsize > 1 or (shape is not None and _prod(shape) > 1):
  650. # Inject internal padding to the end of the sub-item
  651. value = _add_trailing_padding(value, intra_padding)
  652. else:
  653. # We can postpone the injection of internal padding,
  654. # as the item appears at most once
  655. extra_offset += intra_padding
  656. # Update common alignment
  657. common_alignment = _lcm(align, common_alignment)
  658. # Convert itemsize to sub-array
  659. if itemsize != 1:
  660. value = dtype((value, (itemsize,)))
  661. # Sub-arrays (2)
  662. if shape is not None:
  663. value = dtype((value, shape))
  664. # Field name
  665. if stream.consume(':'):
  666. name = stream.consume_until(':')
  667. else:
  668. name = None
  669. if not (is_padding and name is None):
  670. if name is not None and name in field_spec['names']:
  671. raise RuntimeError(
  672. f"Duplicate field name '{name}' in PEP3118 format"
  673. )
  674. field_spec['names'].append(name)
  675. field_spec['formats'].append(value)
  676. field_spec['offsets'].append(offset)
  677. offset += value.itemsize
  678. offset += extra_offset
  679. field_spec['itemsize'] = offset
  680. # extra final padding for aligned types
  681. if stream.byteorder == '@':
  682. field_spec['itemsize'] += (-offset) % common_alignment
  683. # Check if this was a simple 1-item type, and unwrap it
  684. if (field_spec['names'] == [None]
  685. and field_spec['offsets'][0] == 0
  686. and field_spec['itemsize'] == field_spec['formats'][0].itemsize
  687. and not is_subdtype):
  688. ret = field_spec['formats'][0]
  689. else:
  690. _fix_names(field_spec)
  691. ret = dtype(field_spec)
  692. # Finished
  693. return ret, common_alignment
  694. def _fix_names(field_spec):
  695. """ Replace names which are None with the next unused f%d name """
  696. names = field_spec['names']
  697. for i, name in enumerate(names):
  698. if name is not None:
  699. continue
  700. j = 0
  701. while True:
  702. name = f'f{j}'
  703. if name not in names:
  704. break
  705. j = j + 1
  706. names[i] = name
  707. def _add_trailing_padding(value, padding):
  708. """Inject the specified number of padding bytes at the end of a dtype"""
  709. if value.fields is None:
  710. field_spec = dict(
  711. names=['f0'],
  712. formats=[value],
  713. offsets=[0],
  714. itemsize=value.itemsize
  715. )
  716. else:
  717. fields = value.fields
  718. names = value.names
  719. field_spec = dict(
  720. names=names,
  721. formats=[fields[name][0] for name in names],
  722. offsets=[fields[name][1] for name in names],
  723. itemsize=value.itemsize
  724. )
  725. field_spec['itemsize'] += padding
  726. return dtype(field_spec)
  727. def _prod(a):
  728. p = 1
  729. for x in a:
  730. p *= x
  731. return p
  732. def _gcd(a, b):
  733. """Calculate the greatest common divisor of a and b"""
  734. if not (math.isfinite(a) and math.isfinite(b)):
  735. raise ValueError('Can only find greatest common divisor of '
  736. f'finite arguments, found "{a}" and "{b}"')
  737. while b:
  738. a, b = b, a % b
  739. return a
  740. def _lcm(a, b):
  741. return a // _gcd(a, b) * b
  742. def array_ufunc_errmsg_formatter(dummy, ufunc, method, *inputs, **kwargs):
  743. """ Format the error message for when __array_ufunc__ gives up. """
  744. args_string = ', '.join(['{!r}'.format(arg) for arg in inputs] +
  745. ['{}={!r}'.format(k, v)
  746. for k, v in kwargs.items()])
  747. args = inputs + kwargs.get('out', ())
  748. types_string = ', '.join(repr(type(arg).__name__) for arg in args)
  749. return ('operand type(s) all returned NotImplemented from '
  750. '__array_ufunc__({!r}, {!r}, {}): {}'
  751. .format(ufunc, method, args_string, types_string))
  752. def array_function_errmsg_formatter(public_api, types):
  753. """ Format the error message for when __array_ufunc__ gives up. """
  754. func_name = '{}.{}'.format(public_api.__module__, public_api.__name__)
  755. return ("no implementation found for '{}' on types that implement "
  756. '__array_function__: {}'.format(func_name, list(types)))
  757. def _ufunc_doc_signature_formatter(ufunc):
  758. """
  759. Builds a signature string which resembles PEP 457
  760. This is used to construct the first line of the docstring
  761. """
  762. # input arguments are simple
  763. if ufunc.nin == 1:
  764. in_args = 'x'
  765. else:
  766. in_args = ', '.join(f'x{i+1}' for i in range(ufunc.nin))
  767. # output arguments are both keyword or positional
  768. if ufunc.nout == 0:
  769. out_args = ', /, out=()'
  770. elif ufunc.nout == 1:
  771. out_args = ', /, out=None'
  772. else:
  773. out_args = '[, {positional}], / [, out={default}]'.format(
  774. positional=', '.join(
  775. 'out{}'.format(i+1) for i in range(ufunc.nout)),
  776. default=repr((None,)*ufunc.nout)
  777. )
  778. # keyword only args depend on whether this is a gufunc
  779. kwargs = (
  780. ", casting='same_kind'"
  781. ", order='K'"
  782. ", dtype=None"
  783. ", subok=True"
  784. )
  785. # NOTE: gufuncs may or may not support the `axis` parameter
  786. if ufunc.signature is None:
  787. kwargs = f", where=True{kwargs}[, signature]"
  788. else:
  789. kwargs += "[, signature, axes, axis]"
  790. # join all the parts together
  791. return '{name}({in_args}{out_args}, *{kwargs})'.format(
  792. name=ufunc.__name__,
  793. in_args=in_args,
  794. out_args=out_args,
  795. kwargs=kwargs
  796. )
  797. def npy_ctypes_check(cls):
  798. # determine if a class comes from ctypes, in order to work around
  799. # a bug in the buffer protocol for those objects, bpo-10746
  800. try:
  801. # ctypes class are new-style, so have an __mro__. This probably fails
  802. # for ctypes classes with multiple inheritance.
  803. if IS_PYPY:
  804. # (..., _ctypes.basics._CData, Bufferable, object)
  805. ctype_base = cls.__mro__[-3]
  806. else:
  807. # # (..., _ctypes._CData, object)
  808. ctype_base = cls.__mro__[-2]
  809. # right now, they're part of the _ctypes module
  810. return '_ctypes' in ctype_base.__module__
  811. except Exception:
  812. return False
  813. # used to handle the _NoValue default argument for na_object
  814. # in the C implementation of the __reduce__ method for stringdtype
  815. def _convert_to_stringdtype_kwargs(coerce, na_object=_NoValue):
  816. if na_object is _NoValue:
  817. return StringDType(coerce=coerce)
  818. return StringDType(coerce=coerce, na_object=na_object)