test_mio.py 46 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399
  1. import os
  2. from collections import OrderedDict
  3. from os.path import join as pjoin, dirname
  4. from glob import glob
  5. from io import BytesIO
  6. import re
  7. from tempfile import mkdtemp
  8. import warnings
  9. import shutil
  10. import gzip
  11. from numpy.testing import (assert_array_equal, assert_array_almost_equal,
  12. assert_equal, assert_, assert_allclose)
  13. import pytest
  14. from pytest import raises as assert_raises, warns as assert_warns
  15. import numpy as np
  16. from numpy import array
  17. from scipy.sparse import issparse, eye_array, coo_array, csc_array
  18. import scipy.io
  19. from scipy.io.matlab import MatlabOpaque, MatlabFunction, MatlabObject
  20. import scipy.io.matlab._byteordercodes as boc
  21. from scipy.io.matlab._miobase import (matdims, MatWriteError, MatReadError,
  22. matfile_version, MatWriteWarning)
  23. from scipy.io.matlab._mio import mat_reader_factory, loadmat, savemat, whosmat
  24. from scipy.io.matlab._mio5 import (
  25. MatFile5Writer, MatFile5Reader, varmats_from_mat, to_writeable,
  26. EmptyStructMarker)
  27. import scipy.io.matlab._mio5_params as mio5p
  28. test_data_path = pjoin(dirname(__file__), 'data')
  29. pytestmark = pytest.mark.thread_unsafe
  30. def mlarr(*args, **kwargs):
  31. """Convenience function to return matlab-compatible 2-D array."""
  32. arr = np.array(*args, **kwargs)
  33. arr = arr.reshape(matdims(arr))
  34. return arr
  35. # Define cases to test
  36. theta = np.pi/4*np.arange(9,dtype=float).reshape(1,9)
  37. case_table4 = [
  38. {'name': 'double',
  39. 'classes': {'testdouble': 'double'},
  40. 'expected': {'testdouble': theta}
  41. }]
  42. case_table4.append(
  43. {'name': 'string',
  44. 'classes': {'teststring': 'char'},
  45. 'expected': {'teststring':
  46. array(['"Do nine men interpret?" "Nine men," I nod.'])}
  47. })
  48. case_table4.append(
  49. {'name': 'complex',
  50. 'classes': {'testcomplex': 'double'},
  51. 'expected': {'testcomplex': np.cos(theta) + 1j*np.sin(theta)}
  52. })
  53. A = np.zeros((3,5))
  54. A[0] = list(range(1,6))
  55. A[:,0] = list(range(1,4))
  56. case_table4.append(
  57. {'name': 'matrix',
  58. 'classes': {'testmatrix': 'double'},
  59. 'expected': {'testmatrix': A},
  60. })
  61. case_table4.append(
  62. {'name': 'sparse',
  63. 'classes': {'testsparse': 'sparse'},
  64. 'expected': {'testsparse': coo_array(A)},
  65. })
  66. B = A.astype(complex)
  67. B[0,0] += 1j
  68. case_table4.append(
  69. {'name': 'sparsecomplex',
  70. 'classes': {'testsparsecomplex': 'sparse'},
  71. 'expected': {'testsparsecomplex': coo_array(B)},
  72. })
  73. case_table4.append(
  74. {'name': 'multi',
  75. 'classes': {'theta': 'double', 'a': 'double'},
  76. 'expected': {'theta': theta, 'a': A},
  77. })
  78. case_table4.append(
  79. {'name': 'minus',
  80. 'classes': {'testminus': 'double'},
  81. 'expected': {'testminus': mlarr(-1)},
  82. })
  83. case_table4.append(
  84. {'name': 'onechar',
  85. 'classes': {'testonechar': 'char'},
  86. 'expected': {'testonechar': array(['r'])},
  87. })
  88. # Cell arrays stored as object arrays
  89. CA = mlarr(( # tuple for object array creation
  90. [],
  91. mlarr([1]),
  92. mlarr([[1,2]]),
  93. mlarr([[1,2,3]])), dtype=object).reshape(1,-1)
  94. CA[0,0] = array(
  95. ['This cell contains this string and 3 arrays of increasing length'])
  96. case_table5 = [
  97. {'name': 'cell',
  98. 'classes': {'testcell': 'cell'},
  99. 'expected': {'testcell': CA}}]
  100. CAE = mlarr(( # tuple for object array creation
  101. mlarr(1),
  102. mlarr(2),
  103. mlarr([]),
  104. mlarr([]),
  105. mlarr(3)), dtype=object).reshape(1,-1)
  106. objarr = np.empty((1,1),dtype=object)
  107. objarr[0,0] = mlarr(1)
  108. case_table5.append(
  109. {'name': 'scalarcell',
  110. 'classes': {'testscalarcell': 'cell'},
  111. 'expected': {'testscalarcell': objarr}
  112. })
  113. case_table5.append(
  114. {'name': 'emptycell',
  115. 'classes': {'testemptycell': 'cell'},
  116. 'expected': {'testemptycell': CAE}})
  117. case_table5.append(
  118. {'name': 'stringarray',
  119. 'classes': {'teststringarray': 'char'},
  120. 'expected': {'teststringarray': array(
  121. ['one ', 'two ', 'three'])},
  122. })
  123. case_table5.append(
  124. {'name': '3dmatrix',
  125. 'classes': {'test3dmatrix': 'double'},
  126. 'expected': {
  127. 'test3dmatrix': np.transpose(np.reshape(list(range(1,25)), (4,3,2)))}
  128. })
  129. st_sub_arr = array([np.sqrt(2),np.exp(1),np.pi]).reshape(1,3)
  130. dtype = [(n, object) for n in ['stringfield', 'doublefield', 'complexfield']]
  131. st1 = np.zeros((1,1), dtype)
  132. st1['stringfield'][0,0] = array(['Rats live on no evil star.'])
  133. st1['doublefield'][0,0] = st_sub_arr
  134. st1['complexfield'][0,0] = st_sub_arr * (1 + 1j)
  135. case_table5.append(
  136. {'name': 'struct',
  137. 'classes': {'teststruct': 'struct'},
  138. 'expected': {'teststruct': st1}
  139. })
  140. CN = np.zeros((1,2), dtype=object)
  141. CN[0,0] = mlarr(1)
  142. CN[0,1] = np.zeros((1,3), dtype=object)
  143. CN[0,1][0,0] = mlarr(2, dtype=np.uint8)
  144. CN[0,1][0,1] = mlarr([[3]], dtype=np.uint8)
  145. CN[0,1][0,2] = np.zeros((1,2), dtype=object)
  146. CN[0,1][0,2][0,0] = mlarr(4, dtype=np.uint8)
  147. CN[0,1][0,2][0,1] = mlarr(5, dtype=np.uint8)
  148. case_table5.append(
  149. {'name': 'cellnest',
  150. 'classes': {'testcellnest': 'cell'},
  151. 'expected': {'testcellnest': CN},
  152. })
  153. st2 = np.empty((1,1), dtype=[(n, object) for n in ['one', 'two']])
  154. st2[0,0]['one'] = mlarr(1)
  155. st2[0,0]['two'] = np.empty((1,1), dtype=[('three', object)])
  156. st2[0,0]['two'][0,0]['three'] = array(['number 3'])
  157. case_table5.append(
  158. {'name': 'structnest',
  159. 'classes': {'teststructnest': 'struct'},
  160. 'expected': {'teststructnest': st2}
  161. })
  162. a = np.empty((1,2), dtype=[(n, object) for n in ['one', 'two']])
  163. a[0,0]['one'] = mlarr(1)
  164. a[0,0]['two'] = mlarr(2)
  165. a[0,1]['one'] = array(['number 1'])
  166. a[0,1]['two'] = array(['number 2'])
  167. case_table5.append(
  168. {'name': 'structarr',
  169. 'classes': {'teststructarr': 'struct'},
  170. 'expected': {'teststructarr': a}
  171. })
  172. ODT = np.dtype([(n, object) for n in
  173. ['expr', 'inputExpr', 'args',
  174. 'isEmpty', 'numArgs', 'version']])
  175. MO = MatlabObject(np.zeros((1,1), dtype=ODT), 'inline')
  176. m0 = MO[0,0]
  177. m0['expr'] = array(['x'])
  178. m0['inputExpr'] = array([' x = INLINE_INPUTS_{1};'])
  179. m0['args'] = array(['x'])
  180. m0['isEmpty'] = mlarr(0)
  181. m0['numArgs'] = mlarr(1)
  182. m0['version'] = mlarr(1)
  183. case_table5.append(
  184. {'name': 'object',
  185. 'classes': {'testobject': 'object'},
  186. 'expected': {'testobject': MO}
  187. })
  188. fp_u_str = open(pjoin(test_data_path, 'japanese_utf8.txt'), 'rb')
  189. u_str = fp_u_str.read().decode('utf-8')
  190. fp_u_str.close()
  191. case_table5.append(
  192. {'name': 'unicode',
  193. 'classes': {'testunicode': 'char'},
  194. 'expected': {'testunicode': array([u_str])}
  195. })
  196. case_table5.append(
  197. {'name': 'sparse',
  198. 'classes': {'testsparse': 'sparse'},
  199. 'expected': {'testsparse': coo_array(A)},
  200. })
  201. case_table5.append(
  202. {'name': 'sparsecomplex',
  203. 'classes': {'testsparsecomplex': 'sparse'},
  204. 'expected': {'testsparsecomplex': coo_array(B)},
  205. })
  206. case_table5.append(
  207. {'name': 'bool',
  208. 'classes': {'testbools': 'logical'},
  209. 'expected': {'testbools':
  210. array([[True], [False]])},
  211. })
  212. case_table5_rt = case_table5[:]
  213. # Inline functions can't be concatenated in matlab, so RT only
  214. case_table5_rt.append(
  215. {'name': 'objectarray',
  216. 'classes': {'testobjectarray': 'object'},
  217. 'expected': {'testobjectarray': np.repeat(MO, 2).reshape(1,2)}})
  218. def types_compatible(var1, var2):
  219. """Check if types are same or compatible.
  220. 0-D numpy scalars are compatible with bare python scalars.
  221. """
  222. type1 = type(var1)
  223. type2 = type(var2)
  224. if type1 is type2:
  225. return True
  226. if type1 is np.ndarray and var1.shape == ():
  227. return type(var1.item()) is type2
  228. if type2 is np.ndarray and var2.shape == ():
  229. return type(var2.item()) is type1
  230. return False
  231. def _check_level(label, expected, actual):
  232. """ Check one level of a potentially nested array """
  233. if issparse(expected): # allow different types of sparse matrices
  234. assert_(issparse(actual))
  235. assert_array_almost_equal(actual.toarray(),
  236. expected.toarray(),
  237. err_msg=label,
  238. decimal=5)
  239. return
  240. # Check types are as expected
  241. assert_(types_compatible(expected, actual),
  242. f"Expected type {type(expected)}, got {type(actual)} at {label}")
  243. # A field in a record array may not be an ndarray
  244. # A scalar from a record array will be type np.void
  245. if not isinstance(expected, np.void | np.ndarray | MatlabObject):
  246. assert_equal(expected, actual)
  247. return
  248. # This is an ndarray-like thing
  249. assert_(expected.shape == actual.shape,
  250. msg=f'Expected shape {expected.shape}, got {actual.shape} at {label}')
  251. ex_dtype = expected.dtype
  252. if ex_dtype.hasobject: # array of objects
  253. if isinstance(expected, MatlabObject):
  254. assert_equal(expected.classname, actual.classname)
  255. for i, ev in enumerate(expected):
  256. level_label = f"{label}, [{i}], "
  257. _check_level(level_label, ev, actual[i])
  258. return
  259. if ex_dtype.fields: # probably recarray
  260. for fn in ex_dtype.fields:
  261. level_label = f"{label}, field {fn}, "
  262. _check_level(level_label,
  263. expected[fn], actual[fn])
  264. return
  265. if ex_dtype.type in (str, # string or bool
  266. np.str_,
  267. np.bool_):
  268. assert_equal(actual, expected, err_msg=label)
  269. return
  270. # Something numeric
  271. assert_array_almost_equal(actual, expected, err_msg=label, decimal=5)
  272. def _load_check_case(name, files, case):
  273. for file_name in files:
  274. matdict = loadmat(file_name, struct_as_record=True, spmatrix=False)
  275. label = f"test {name}; file {file_name}"
  276. for k, expected in case.items():
  277. k_label = f"{label}, variable {k}"
  278. assert_(k in matdict, f"Missing key at {k_label}")
  279. _check_level(k_label, expected, matdict[k])
  280. def _whos_check_case(name, files, case, classes):
  281. for file_name in files:
  282. label = f"test {name}; file {file_name}"
  283. whos = whosmat(file_name)
  284. expected_whos = [
  285. (k, expected.shape, classes[k]) for k, expected in case.items()]
  286. whos.sort()
  287. expected_whos.sort()
  288. assert_equal(whos, expected_whos,
  289. f"{label}: {whos!r} != {expected_whos!r}"
  290. )
  291. # Round trip tests
  292. def _rt_check_case(name, expected, format):
  293. mat_stream = BytesIO()
  294. savemat(mat_stream, expected, format=format)
  295. mat_stream.seek(0)
  296. _load_check_case(name, [mat_stream], expected)
  297. # generator for tests
  298. def _cases(version, filt='test%(name)s_*.mat'):
  299. if version == '4':
  300. cases = case_table4
  301. elif version == '5':
  302. cases = case_table5
  303. else:
  304. assert version == '5_rt'
  305. cases = case_table5_rt
  306. for case in cases:
  307. name = case['name']
  308. expected = case['expected']
  309. if filt is None:
  310. files = None
  311. else:
  312. use_filt = pjoin(test_data_path, filt % dict(name=name))
  313. files = glob(use_filt)
  314. assert len(files) > 0, \
  315. f"No files for test {name} using filter {filt}"
  316. classes = case['classes']
  317. yield name, files, expected, classes
  318. @pytest.mark.parametrize('version', ('4', '5'))
  319. def test_load(version):
  320. for case in _cases(version):
  321. _load_check_case(*case[:3])
  322. @pytest.mark.parametrize('version', ('4', '5'))
  323. def test_whos(version):
  324. for case in _cases(version):
  325. _whos_check_case(*case)
  326. # generator for round trip tests
  327. @pytest.mark.parametrize('version, fmts', [
  328. ('4', ['4', '5']),
  329. ('5_rt', ['5']),
  330. ])
  331. def test_round_trip(version, fmts):
  332. for case in _cases(version, filt=None):
  333. for fmt in fmts:
  334. _rt_check_case(case[0], case[2], fmt)
  335. def test_gzip_simple():
  336. xdense = np.zeros((20,20))
  337. xdense[2,3] = 2.3
  338. xdense[4,5] = 4.5
  339. x = csc_array(xdense)
  340. name = 'gzip_test'
  341. expected = {'x':x}
  342. format = '4'
  343. tmpdir = mkdtemp()
  344. try:
  345. fname = pjoin(tmpdir,name)
  346. mat_stream = gzip.open(fname, mode='wb')
  347. savemat(mat_stream, expected, format=format)
  348. mat_stream.close()
  349. mat_stream = gzip.open(fname, mode='rb')
  350. actual = loadmat(mat_stream, struct_as_record=True, spmatrix=False)
  351. mat_stream.close()
  352. finally:
  353. shutil.rmtree(tmpdir)
  354. assert_array_almost_equal(actual['x'].toarray(),
  355. expected['x'].toarray(),
  356. err_msg=repr(actual))
  357. def test_multiple_open():
  358. # Ticket #1039, on Windows: check that files are not left open
  359. tmpdir = mkdtemp()
  360. try:
  361. x = dict(x=np.zeros((2, 2)))
  362. fname = pjoin(tmpdir, "a.mat")
  363. # Check that file is not left open
  364. savemat(fname, x)
  365. os.unlink(fname)
  366. savemat(fname, x)
  367. loadmat(fname)
  368. os.unlink(fname)
  369. # Check that stream is left open
  370. f = open(fname, 'wb')
  371. savemat(f, x)
  372. f.seek(0)
  373. f.close()
  374. f = open(fname, 'rb')
  375. loadmat(f)
  376. f.seek(0)
  377. f.close()
  378. finally:
  379. shutil.rmtree(tmpdir)
  380. def test_mat73():
  381. # Check any hdf5 files raise an error
  382. filenames = glob(
  383. pjoin(test_data_path, 'testhdf5*.mat'))
  384. assert_(len(filenames) > 0)
  385. for filename in filenames:
  386. fp = open(filename, 'rb')
  387. assert_raises(NotImplementedError,
  388. loadmat,
  389. fp,
  390. struct_as_record=True)
  391. fp.close()
  392. def test_warnings():
  393. # This test is an echo of the previous behavior, which was to raise a
  394. # warning if the user triggered a search for mat files on the Python system
  395. # path. We can remove the test in the next version after upcoming (0.13).
  396. fname = pjoin(test_data_path, 'testdouble_7.1_GLNX86.mat')
  397. with warnings.catch_warnings():
  398. warnings.simplefilter('error')
  399. # This should not generate a warning
  400. loadmat(fname, struct_as_record=True)
  401. # This neither
  402. loadmat(fname, struct_as_record=False)
  403. def test_regression_653():
  404. # Saving a dictionary with only invalid keys used to raise an error. Now we
  405. # save this as an empty struct in matlab space.
  406. sio = BytesIO()
  407. savemat(sio, {'d':{1:2}}, format='5')
  408. back = loadmat(sio)['d']
  409. # Check we got an empty struct equivalent
  410. assert_equal(back.shape, (1,1))
  411. assert_equal(back.dtype, np.dtype(object))
  412. assert_(back[0,0] is None)
  413. def test_structname_len():
  414. # Test limit for length of field names in structs
  415. lim = 31
  416. fldname = 'a' * lim
  417. st1 = np.zeros((1,1), dtype=[(fldname, object)])
  418. savemat(BytesIO(), {'longstruct': st1}, format='5')
  419. fldname = 'a' * (lim+1)
  420. st1 = np.zeros((1,1), dtype=[(fldname, object)])
  421. assert_raises(ValueError, savemat, BytesIO(),
  422. {'longstruct': st1}, format='5')
  423. def test_4_and_long_field_names_incompatible():
  424. # Long field names option not supported in 4
  425. my_struct = np.zeros((1,1),dtype=[('my_fieldname',object)])
  426. assert_raises(ValueError, savemat, BytesIO(),
  427. {'my_struct':my_struct}, format='4', long_field_names=True)
  428. def test_long_field_names():
  429. # Test limit for length of field names in structs
  430. lim = 63
  431. fldname = 'a' * lim
  432. st1 = np.zeros((1,1), dtype=[(fldname, object)])
  433. savemat(BytesIO(), {'longstruct': st1}, format='5',long_field_names=True)
  434. fldname = 'a' * (lim+1)
  435. st1 = np.zeros((1,1), dtype=[(fldname, object)])
  436. assert_raises(ValueError, savemat, BytesIO(),
  437. {'longstruct': st1}, format='5',long_field_names=True)
  438. def test_long_field_names_in_struct():
  439. # Regression test - long_field_names was erased if you passed a struct
  440. # within a struct
  441. lim = 63
  442. fldname = 'a' * lim
  443. cell = np.ndarray((1,2),dtype=object)
  444. st1 = np.zeros((1,1), dtype=[(fldname, object)])
  445. cell[0,0] = st1
  446. cell[0,1] = st1
  447. savemat(BytesIO(), {'longstruct': cell}, format='5',long_field_names=True)
  448. #
  449. # Check to make sure it fails with long field names off
  450. #
  451. assert_raises(ValueError, savemat, BytesIO(),
  452. {'longstruct': cell}, format='5', long_field_names=False)
  453. def test_cell_with_one_thing_in_it():
  454. # Regression test - make a cell array that's 1 x 2 and put two
  455. # strings in it. It works. Make a cell array that's 1 x 1 and put
  456. # a string in it. It should work but, in the old days, it didn't.
  457. cells = np.ndarray((1,2),dtype=object)
  458. cells[0,0] = 'Hello'
  459. cells[0,1] = 'World'
  460. savemat(BytesIO(), {'x': cells}, format='5')
  461. cells = np.ndarray((1,1),dtype=object)
  462. cells[0,0] = 'Hello, world'
  463. savemat(BytesIO(), {'x': cells}, format='5')
  464. def test_writer_properties():
  465. # Tests getting, setting of properties of matrix writer
  466. mfw = MatFile5Writer(BytesIO())
  467. assert_equal(mfw.global_vars, [])
  468. mfw.global_vars = ['avar']
  469. assert_equal(mfw.global_vars, ['avar'])
  470. assert_equal(mfw.unicode_strings, False)
  471. mfw.unicode_strings = True
  472. assert_equal(mfw.unicode_strings, True)
  473. assert_equal(mfw.long_field_names, False)
  474. mfw.long_field_names = True
  475. assert_equal(mfw.long_field_names, True)
  476. def test_use_small_element():
  477. # Test whether we're using small data element or not
  478. sio = BytesIO()
  479. wtr = MatFile5Writer(sio)
  480. # First check size for no sde for name
  481. arr = np.zeros(10)
  482. wtr.put_variables({'aaaaa': arr})
  483. w_sz = len(sio.getvalue())
  484. # Check small name results in largish difference in size
  485. sio.truncate(0)
  486. sio.seek(0)
  487. wtr.put_variables({'aaaa': arr})
  488. assert_(w_sz - len(sio.getvalue()) > 4)
  489. # Whereas increasing name size makes less difference
  490. sio.truncate(0)
  491. sio.seek(0)
  492. wtr.put_variables({'aaaaaa': arr})
  493. assert_(len(sio.getvalue()) - w_sz < 4)
  494. def test_save_dict():
  495. # Test that both dict and OrderedDict can be saved (as recarray),
  496. # loaded as matstruct, and preserve order
  497. ab_exp = np.array([[(1, 2)]], dtype=[('a', object), ('b', object)])
  498. for dict_type in (dict, OrderedDict):
  499. # Initialize with tuples to keep order
  500. d = dict_type([('a', 1), ('b', 2)])
  501. stream = BytesIO()
  502. savemat(stream, {'dict': d})
  503. stream.seek(0)
  504. vals = loadmat(stream)['dict']
  505. assert_equal(vals.dtype.names, ('a', 'b'))
  506. assert_array_equal(vals, ab_exp)
  507. def test_1d_shape():
  508. # New 5 behavior is 1D -> row vector
  509. arr = np.arange(5)
  510. for format in ('4', '5'):
  511. # Column is the default
  512. stream = BytesIO()
  513. savemat(stream, {'oned': arr}, format=format)
  514. vals = loadmat(stream)
  515. assert_equal(vals['oned'].shape, (1, 5))
  516. # can be explicitly 'column' for oned_as
  517. stream = BytesIO()
  518. savemat(stream, {'oned':arr},
  519. format=format,
  520. oned_as='column')
  521. vals = loadmat(stream)
  522. assert_equal(vals['oned'].shape, (5,1))
  523. # but different from 'row'
  524. stream = BytesIO()
  525. savemat(stream, {'oned':arr},
  526. format=format,
  527. oned_as='row')
  528. vals = loadmat(stream)
  529. assert_equal(vals['oned'].shape, (1,5))
  530. def test_compression():
  531. arr = np.zeros(100).reshape((5,20))
  532. arr[2,10] = 1
  533. stream = BytesIO()
  534. savemat(stream, {'arr':arr})
  535. raw_len = len(stream.getvalue())
  536. vals = loadmat(stream)
  537. assert_array_equal(vals['arr'], arr)
  538. stream = BytesIO()
  539. savemat(stream, {'arr':arr}, do_compression=True)
  540. compressed_len = len(stream.getvalue())
  541. vals = loadmat(stream)
  542. assert_array_equal(vals['arr'], arr)
  543. assert_(raw_len > compressed_len)
  544. # Concatenate, test later
  545. arr2 = arr.copy()
  546. arr2[0,0] = 1
  547. stream = BytesIO()
  548. savemat(stream, {'arr':arr, 'arr2':arr2}, do_compression=False)
  549. vals = loadmat(stream)
  550. assert_array_equal(vals['arr2'], arr2)
  551. stream = BytesIO()
  552. savemat(stream, {'arr':arr, 'arr2':arr2}, do_compression=True)
  553. vals = loadmat(stream)
  554. assert_array_equal(vals['arr2'], arr2)
  555. def test_single_object():
  556. stream = BytesIO()
  557. savemat(stream, {'A':np.array(1, dtype=object)})
  558. def test_skip_variable():
  559. # Test skipping over the first of two variables in a MAT file
  560. # using mat_reader_factory and put_variables to read them in.
  561. #
  562. # This is a regression test of a problem that's caused by
  563. # using the compressed file reader seek instead of the raw file
  564. # I/O seek when skipping over a compressed chunk.
  565. #
  566. # The problem arises when the chunk is large: this file has
  567. # a 256x256 array of random (uncompressible) doubles.
  568. #
  569. filename = pjoin(test_data_path,'test_skip_variable.mat')
  570. #
  571. # Prove that it loads with loadmat
  572. #
  573. d = loadmat(filename, struct_as_record=True)
  574. assert_('first' in d)
  575. assert_('second' in d)
  576. #
  577. # Make the factory
  578. #
  579. factory, file_opened = mat_reader_factory(filename, struct_as_record=True)
  580. #
  581. # This is where the factory breaks with an error in MatMatrixGetter.to_next
  582. #
  583. d = factory.get_variables('second')
  584. assert_('second' in d)
  585. factory.mat_stream.close()
  586. def test_empty_struct():
  587. # ticket 885
  588. filename = pjoin(test_data_path,'test_empty_struct.mat')
  589. # before ticket fix, this would crash with ValueError, empty data
  590. # type
  591. d = loadmat(filename, struct_as_record=True)
  592. a = d['a']
  593. assert_equal(a.shape, (1,1))
  594. assert_equal(a.dtype, np.dtype(object))
  595. assert_(a[0,0] is None)
  596. stream = BytesIO()
  597. arr = np.array((), dtype='U')
  598. # before ticket fix, this used to give data type not understood
  599. savemat(stream, {'arr':arr})
  600. d = loadmat(stream)
  601. a2 = d['arr']
  602. assert_array_equal(a2, arr)
  603. def test_save_empty_dict():
  604. # saving empty dict also gives empty struct
  605. stream = BytesIO()
  606. savemat(stream, {'arr': {}})
  607. d = loadmat(stream)
  608. a = d['arr']
  609. assert_equal(a.shape, (1,1))
  610. assert_equal(a.dtype, np.dtype(object))
  611. assert_(a[0,0] is None)
  612. def assert_any_equal(output, alternatives):
  613. """ Assert `output` is equal to at least one element in `alternatives`
  614. """
  615. one_equal = False
  616. for expected in alternatives:
  617. if np.all(output == expected):
  618. one_equal = True
  619. break
  620. assert_(one_equal)
  621. def test_to_writeable():
  622. # Test to_writeable function
  623. res = to_writeable(np.array([1])) # pass through ndarrays
  624. assert_equal(res.shape, (1,))
  625. assert_array_equal(res, 1)
  626. # Dict fields can be written in any order
  627. expected1 = np.array([(1, 2)], dtype=[('a', '|O8'), ('b', '|O8')])
  628. expected2 = np.array([(2, 1)], dtype=[('b', '|O8'), ('a', '|O8')])
  629. alternatives = (expected1, expected2)
  630. assert_any_equal(to_writeable({'a':1,'b':2}), alternatives)
  631. # Fields with underscores discarded with a warning message.
  632. with pytest.warns(MatWriteWarning, match='Starting field name with'):
  633. assert_any_equal(to_writeable({'a':1, 'b':2, '_c':3}), alternatives)
  634. # Not-string fields discarded
  635. assert_any_equal(to_writeable({'a':1,'b':2, 100:3}), alternatives)
  636. # String fields that are valid Python identifiers discarded
  637. with pytest.warns(MatWriteWarning, match='Starting field name with'):
  638. assert_any_equal(to_writeable({'a':1, 'b':2, '99':3}), alternatives)
  639. # Object with field names is equivalent
  640. class klass:
  641. pass
  642. c = klass
  643. c.a = 1
  644. c.b = 2
  645. assert_any_equal(to_writeable(c), alternatives)
  646. # empty list and tuple go to empty array
  647. res = to_writeable([])
  648. assert_equal(res.shape, (0,))
  649. assert_equal(res.dtype.type, np.float64)
  650. res = to_writeable(())
  651. assert_equal(res.shape, (0,))
  652. assert_equal(res.dtype.type, np.float64)
  653. # None -> None
  654. assert_(to_writeable(None) is None)
  655. # String to strings
  656. assert_equal(to_writeable('a string').dtype.type, np.str_)
  657. # Scalars to numpy to NumPy scalars
  658. res = to_writeable(1)
  659. assert_equal(res.shape, ())
  660. assert_equal(res.dtype.type, np.array(1).dtype.type)
  661. assert_array_equal(res, 1)
  662. # Empty dict returns EmptyStructMarker
  663. assert_(to_writeable({}) is EmptyStructMarker)
  664. # Object does not have (even empty) __dict__
  665. assert_(to_writeable(object()) is None)
  666. # Custom object does have empty __dict__, returns EmptyStructMarker
  667. class C:
  668. pass
  669. assert_(to_writeable(c()) is EmptyStructMarker)
  670. # dict keys with legal characters are convertible
  671. res = to_writeable({'a': 1})['a']
  672. assert_equal(res.shape, (1,))
  673. assert_equal(res.dtype.type, np.object_)
  674. # Only fields with illegal characters, falls back to EmptyStruct
  675. with pytest.warns(MatWriteWarning, match='Starting field name with'):
  676. assert_(to_writeable({'1':1}) is EmptyStructMarker)
  677. with pytest.warns(MatWriteWarning, match='Starting field name with'):
  678. assert_(to_writeable({'_a':1}) is EmptyStructMarker)
  679. # Unless there are valid fields, in which case structured array
  680. with pytest.warns(MatWriteWarning, match='Starting field name with'):
  681. assert_equal(to_writeable({'1':1, 'f': 2}),
  682. np.array([(2,)], dtype=[('f', '|O8')]))
  683. def test_recarray():
  684. # check roundtrip of structured array
  685. dt = [('f1', 'f8'),
  686. ('f2', 'S10')]
  687. arr = np.zeros((2,), dtype=dt)
  688. arr[0]['f1'] = 0.5
  689. arr[0]['f2'] = 'python'
  690. arr[1]['f1'] = 99
  691. arr[1]['f2'] = 'not perl'
  692. stream = BytesIO()
  693. savemat(stream, {'arr': arr})
  694. d = loadmat(stream, struct_as_record=False)
  695. a20 = d['arr'][0,0]
  696. assert_equal(a20.f1, 0.5)
  697. assert_equal(a20.f2, 'python')
  698. d = loadmat(stream, struct_as_record=True)
  699. a20 = d['arr'][0,0]
  700. assert_equal(a20['f1'], 0.5)
  701. assert_equal(a20['f2'], 'python')
  702. # structs always come back as object types
  703. assert_equal(a20.dtype, np.dtype([('f1', 'O'),
  704. ('f2', 'O')]))
  705. a21 = d['arr'].flat[1]
  706. assert_equal(a21['f1'], 99)
  707. assert_equal(a21['f2'], 'not perl')
  708. def test_save_object():
  709. class C:
  710. pass
  711. c = C()
  712. c.field1 = 1
  713. c.field2 = 'a string'
  714. stream = BytesIO()
  715. savemat(stream, {'c': c})
  716. d = loadmat(stream, struct_as_record=False)
  717. c2 = d['c'][0,0]
  718. assert_equal(c2.field1, 1)
  719. assert_equal(c2.field2, 'a string')
  720. d = loadmat(stream, struct_as_record=True)
  721. c2 = d['c'][0,0]
  722. assert_equal(c2['field1'], 1)
  723. assert_equal(c2['field2'], 'a string')
  724. def test_read_opts():
  725. # tests if read is seeing option sets, at initialization and after
  726. # initialization
  727. arr = np.arange(6).reshape(1,6)
  728. stream = BytesIO()
  729. savemat(stream, {'a': arr})
  730. rdr = MatFile5Reader(stream)
  731. back_dict = rdr.get_variables()
  732. rarr = back_dict['a']
  733. assert_array_equal(rarr, arr)
  734. rdr = MatFile5Reader(stream, squeeze_me=True)
  735. assert_array_equal(rdr.get_variables()['a'], arr.reshape((6,)))
  736. rdr.squeeze_me = False
  737. assert_array_equal(rarr, arr)
  738. rdr = MatFile5Reader(stream, byte_order=boc.native_code)
  739. assert_array_equal(rdr.get_variables()['a'], arr)
  740. # inverted byte code leads to error on read because of swapped
  741. # header etc.
  742. rdr = MatFile5Reader(stream, byte_order=boc.swapped_code)
  743. assert_raises(Exception, rdr.get_variables)
  744. rdr.byte_order = boc.native_code
  745. assert_array_equal(rdr.get_variables()['a'], arr)
  746. arr = np.array(['a string'])
  747. stream.truncate(0)
  748. stream.seek(0)
  749. savemat(stream, {'a': arr})
  750. rdr = MatFile5Reader(stream)
  751. assert_array_equal(rdr.get_variables()['a'], arr)
  752. rdr = MatFile5Reader(stream, chars_as_strings=False)
  753. carr = np.atleast_2d(np.array(list(arr.item()), dtype='U1'))
  754. assert_array_equal(rdr.get_variables()['a'], carr)
  755. rdr.chars_as_strings = True
  756. assert_array_equal(rdr.get_variables()['a'], arr)
  757. def test_empty_string():
  758. # make sure reading empty string does not raise error
  759. estring_fname = pjoin(test_data_path, 'single_empty_string.mat')
  760. fp = open(estring_fname, 'rb')
  761. rdr = MatFile5Reader(fp)
  762. d = rdr.get_variables()
  763. fp.close()
  764. assert_array_equal(d['a'], np.array([], dtype='U1'))
  765. # Empty string round trip. Matlab cannot distinguish
  766. # between a string array that is empty, and a string array
  767. # containing a single empty string, because it stores strings as
  768. # arrays of char. There is no way of having an array of char that
  769. # is not empty, but contains an empty string.
  770. stream = BytesIO()
  771. savemat(stream, {'a': np.array([''])})
  772. rdr = MatFile5Reader(stream)
  773. d = rdr.get_variables()
  774. assert_array_equal(d['a'], np.array([], dtype='U1'))
  775. stream.truncate(0)
  776. stream.seek(0)
  777. savemat(stream, {'a': np.array([], dtype='U1')})
  778. rdr = MatFile5Reader(stream)
  779. d = rdr.get_variables()
  780. assert_array_equal(d['a'], np.array([], dtype='U1'))
  781. stream.close()
  782. def test_corrupted_data():
  783. import zlib
  784. for exc, fname in [(ValueError, 'corrupted_zlib_data.mat'),
  785. (zlib.error, 'corrupted_zlib_checksum.mat')]:
  786. with open(pjoin(test_data_path, fname), 'rb') as fp:
  787. rdr = MatFile5Reader(fp)
  788. assert_raises(exc, rdr.get_variables)
  789. def test_corrupted_data_check_can_be_disabled():
  790. with open(pjoin(test_data_path, 'corrupted_zlib_data.mat'), 'rb') as fp:
  791. rdr = MatFile5Reader(fp, verify_compressed_data_integrity=False)
  792. rdr.get_variables()
  793. def test_read_both_endian():
  794. # make sure big- and little- endian data is read correctly
  795. for fname in ('big_endian.mat', 'little_endian.mat'):
  796. fp = open(pjoin(test_data_path, fname), 'rb')
  797. rdr = MatFile5Reader(fp)
  798. d = rdr.get_variables()
  799. fp.close()
  800. assert_array_equal(d['strings'],
  801. np.array([['hello'],
  802. ['world']], dtype=object))
  803. assert_array_equal(d['floats'],
  804. np.array([[2., 3.],
  805. [3., 4.]], dtype=np.float32))
  806. def test_write_opposite_endian():
  807. # We don't support writing opposite endian .mat files, but we need to behave
  808. # correctly if the user supplies an other-endian NumPy array to write out.
  809. float_arr = np.array([[2., 3.],
  810. [3., 4.]])
  811. int_arr = np.arange(6).reshape((2, 3))
  812. uni_arr = np.array(['hello', 'world'], dtype='U')
  813. stream = BytesIO()
  814. savemat(stream, {
  815. 'floats': float_arr.byteswap().view(float_arr.dtype.newbyteorder()),
  816. 'ints': int_arr.byteswap().view(int_arr.dtype.newbyteorder()),
  817. 'uni_arr': uni_arr.byteswap().view(uni_arr.dtype.newbyteorder()),
  818. })
  819. rdr = MatFile5Reader(stream)
  820. d = rdr.get_variables()
  821. assert_array_equal(d['floats'], float_arr)
  822. assert_array_equal(d['ints'], int_arr)
  823. assert_array_equal(d['uni_arr'], uni_arr)
  824. stream.close()
  825. def test_logical_array():
  826. # The roundtrip test doesn't verify that we load the data up with the
  827. # correct (bool) dtype
  828. with open(pjoin(test_data_path, 'testbool_8_WIN64.mat'), 'rb') as fobj:
  829. rdr = MatFile5Reader(fobj, mat_dtype=True)
  830. d = rdr.get_variables()
  831. x = np.array([[True], [False]], dtype=np.bool_)
  832. assert_array_equal(d['testbools'], x)
  833. assert_equal(d['testbools'].dtype, x.dtype)
  834. def test_logical_out_type():
  835. # Confirm that bool type written as uint8, uint8 class
  836. # See gh-4022
  837. stream = BytesIO()
  838. barr = np.array([False, True, False])
  839. savemat(stream, {'barray': barr})
  840. stream.seek(0)
  841. reader = MatFile5Reader(stream)
  842. reader.initialize_read()
  843. reader.read_file_header()
  844. hdr, _ = reader.read_var_header()
  845. assert_equal(hdr.mclass, mio5p.mxUINT8_CLASS)
  846. assert_equal(hdr.is_logical, True)
  847. var = reader.read_var_array(hdr, False)
  848. assert_equal(var.dtype.type, np.uint8)
  849. def test_roundtrip_zero_dimensions():
  850. stream = BytesIO()
  851. savemat(stream, {'d':np.empty((10, 0))})
  852. d = loadmat(stream)
  853. assert d['d'].shape == (10, 0)
  854. def test_mat4_3d():
  855. # test behavior when writing 3-D arrays to matlab 4 files
  856. stream = BytesIO()
  857. arr = np.arange(24).reshape((2,3,4))
  858. assert_raises(ValueError, savemat, stream, {'a': arr}, True, '4')
  859. def test_func_read():
  860. func_eg = pjoin(test_data_path, 'testfunc_7.4_GLNX86.mat')
  861. fp = open(func_eg, 'rb')
  862. rdr = MatFile5Reader(fp)
  863. d = rdr.get_variables()
  864. fp.close()
  865. assert isinstance(d['testfunc'], MatlabFunction)
  866. stream = BytesIO()
  867. wtr = MatFile5Writer(stream)
  868. # This test mat file has `__header__` field.
  869. with pytest.warns(MatWriteWarning, match='Starting field name with'):
  870. assert_raises(MatWriteError, wtr.put_variables, d)
  871. def test_mat_dtype():
  872. double_eg = pjoin(test_data_path, 'testmatrix_6.1_SOL2.mat')
  873. fp = open(double_eg, 'rb')
  874. rdr = MatFile5Reader(fp, mat_dtype=False)
  875. d = rdr.get_variables()
  876. fp.close()
  877. assert_equal(d['testmatrix'].dtype.kind, 'u')
  878. fp = open(double_eg, 'rb')
  879. rdr = MatFile5Reader(fp, mat_dtype=True)
  880. d = rdr.get_variables()
  881. fp.close()
  882. assert_equal(d['testmatrix'].dtype.kind, 'f')
  883. def test_sparse_in_struct():
  884. # reproduces bug found by DC where Cython code was insisting on
  885. # ndarray return type, but getting sparse matrix
  886. st = {'sparsefield': eye_array(4)}
  887. stream = BytesIO()
  888. savemat(stream, {'a':st})
  889. d = loadmat(stream, struct_as_record=True)
  890. assert_array_equal(d['a'][0, 0]['sparsefield'].toarray(), np.eye(4))
  891. def test_mat_struct_squeeze():
  892. stream = BytesIO()
  893. in_d = {'st':{'one':1, 'two':2}}
  894. savemat(stream, in_d)
  895. # no error without squeeze
  896. loadmat(stream, struct_as_record=False)
  897. # previous error was with squeeze, with mat_struct
  898. loadmat(stream, struct_as_record=False, squeeze_me=True)
  899. def test_scalar_squeeze():
  900. stream = BytesIO()
  901. in_d = {'scalar': [[0.1]], 'string': 'my name', 'st':{'one':1, 'two':2}}
  902. savemat(stream, in_d)
  903. out_d = loadmat(stream, squeeze_me=True)
  904. assert_(isinstance(out_d['scalar'], float))
  905. assert_(isinstance(out_d['string'], str))
  906. assert_(isinstance(out_d['st'], np.ndarray))
  907. def test_str_round():
  908. # from report by Angus McMorland on mailing list 3 May 2010
  909. stream = BytesIO()
  910. in_arr = np.array(['Hello', 'Foob'])
  911. out_arr = np.array(['Hello', 'Foob '])
  912. savemat(stream, dict(a=in_arr))
  913. res = loadmat(stream)
  914. # resulted in ['HloolFoa', 'elWrdobr']
  915. assert_array_equal(res['a'], out_arr)
  916. stream.truncate(0)
  917. stream.seek(0)
  918. # Make Fortran ordered version of string
  919. in_str = in_arr.tobytes(order='F')
  920. in_from_str = np.ndarray(shape=a.shape,
  921. dtype=in_arr.dtype,
  922. order='F',
  923. buffer=in_str)
  924. savemat(stream, dict(a=in_from_str))
  925. assert_array_equal(res['a'], out_arr)
  926. # unicode save did lead to buffer too small error
  927. stream.truncate(0)
  928. stream.seek(0)
  929. in_arr_u = in_arr.astype('U')
  930. out_arr_u = out_arr.astype('U')
  931. savemat(stream, {'a': in_arr_u})
  932. res = loadmat(stream)
  933. assert_array_equal(res['a'], out_arr_u)
  934. def test_fieldnames():
  935. # Check that field names are as expected
  936. stream = BytesIO()
  937. savemat(stream, {'a': {'a':1, 'b':2}})
  938. res = loadmat(stream)
  939. field_names = res['a'].dtype.names
  940. assert_equal(set(field_names), {'a', 'b'})
  941. def test_loadmat_varnames():
  942. # Test that we can get just one variable from a mat file using loadmat
  943. mat5_sys_names = ['__globals__',
  944. '__header__',
  945. '__version__']
  946. for eg_file, sys_v_names in (
  947. (pjoin(test_data_path, 'testmulti_4.2c_SOL2.mat'), []), (pjoin(
  948. test_data_path, 'testmulti_7.4_GLNX86.mat'), mat5_sys_names)):
  949. vars = loadmat(eg_file)
  950. assert_equal(set(vars.keys()), set(['a', 'theta'] + sys_v_names))
  951. vars = loadmat(eg_file, variable_names='a')
  952. assert_equal(set(vars.keys()), set(['a'] + sys_v_names))
  953. vars = loadmat(eg_file, variable_names=['a'])
  954. assert_equal(set(vars.keys()), set(['a'] + sys_v_names))
  955. vars = loadmat(eg_file, variable_names=['theta'])
  956. assert_equal(set(vars.keys()), set(['theta'] + sys_v_names))
  957. vars = loadmat(eg_file, variable_names=('theta',))
  958. assert_equal(set(vars.keys()), set(['theta'] + sys_v_names))
  959. vars = loadmat(eg_file, variable_names=[])
  960. assert_equal(set(vars.keys()), set(sys_v_names))
  961. vnames = ['theta']
  962. vars = loadmat(eg_file, variable_names=vnames)
  963. assert_equal(vnames, ['theta'])
  964. def test_round_types():
  965. # Check that saving, loading preserves dtype in most cases
  966. arr = np.arange(10)
  967. stream = BytesIO()
  968. for dts in ('f8','f4','i8','i4','i2','i1',
  969. 'u8','u4','u2','u1','c16','c8'):
  970. stream.truncate(0)
  971. stream.seek(0) # needed for BytesIO in Python 3
  972. savemat(stream, {'arr': arr.astype(dts)})
  973. vars = loadmat(stream)
  974. assert_equal(np.dtype(dts), vars['arr'].dtype)
  975. def test_varmats_from_mat():
  976. # Make a mat file with several variables, write it, read it back
  977. names_vars = (('arr', mlarr(np.arange(10))),
  978. ('mystr', mlarr('a string')),
  979. ('mynum', mlarr(10)))
  980. # Dict like thing to give variables in defined order
  981. class C:
  982. def items(self):
  983. return names_vars
  984. stream = BytesIO()
  985. savemat(stream, C())
  986. varmats = varmats_from_mat(stream)
  987. assert_equal(len(varmats), 3)
  988. for i in range(3):
  989. name, var_stream = varmats[i]
  990. exp_name, exp_res = names_vars[i]
  991. assert_equal(name, exp_name)
  992. res = loadmat(var_stream)
  993. assert_array_equal(res[name], exp_res)
  994. def test_one_by_zero():
  995. # Test 1x0 chars get read correctly
  996. func_eg = pjoin(test_data_path, 'one_by_zero_char.mat')
  997. fp = open(func_eg, 'rb')
  998. rdr = MatFile5Reader(fp)
  999. d = rdr.get_variables()
  1000. fp.close()
  1001. assert_equal(d['var'].shape, (0,))
  1002. def test_load_mat4_le():
  1003. # We were getting byte order wrong when reading little-endian floa64 dense
  1004. # matrices on big-endian platforms
  1005. mat4_fname = pjoin(test_data_path, 'test_mat4_le_floats.mat')
  1006. vars = loadmat(mat4_fname)
  1007. assert_array_equal(vars['a'], [[0.1, 1.2]])
  1008. def test_unicode_mat4():
  1009. # Mat4 should save unicode as latin1
  1010. bio = BytesIO()
  1011. var = {'second_cat': 'Schrödinger'}
  1012. savemat(bio, var, format='4')
  1013. var_back = loadmat(bio)
  1014. assert_equal(var_back['second_cat'], var['second_cat'])
  1015. def test_logical_sparse():
  1016. # Test we can read logical sparse stored in mat file as bytes.
  1017. # See https://github.com/scipy/scipy/issues/3539.
  1018. # In some files saved by MATLAB, the sparse data elements (Real Part
  1019. # Subelement in MATLAB speak) are stored with apparent type double
  1020. # (miDOUBLE) but are in fact single bytes.
  1021. filename = pjoin(test_data_path,'logical_sparse.mat')
  1022. # Before fix, this would crash with:
  1023. # ValueError: indices and data should have the same size
  1024. d = loadmat(filename, struct_as_record=True, spmatrix=False)
  1025. log_sp = d['sp_log_5_4']
  1026. assert_(issparse(log_sp) and log_sp.format == "csc")
  1027. assert_equal(log_sp.dtype.type, np.bool_)
  1028. assert_array_equal(log_sp.toarray(),
  1029. [[True, True, True, False],
  1030. [False, False, True, False],
  1031. [False, False, True, False],
  1032. [False, False, False, False],
  1033. [False, False, False, False]])
  1034. def test_empty_sparse():
  1035. # Can we read empty sparse matrices?
  1036. sio = BytesIO()
  1037. import scipy.sparse
  1038. empty_sparse = scipy.sparse.csr_array([[0,0],[0,0]])
  1039. savemat(sio, dict(x=empty_sparse))
  1040. sio.seek(0)
  1041. res = loadmat(sio, spmatrix=False)
  1042. assert not scipy.sparse.isspmatrix(res['x'])
  1043. res = loadmat(sio, spmatrix=True)
  1044. assert scipy.sparse.isspmatrix(res['x'])
  1045. res = loadmat(sio) # chk default
  1046. assert scipy.sparse.isspmatrix(res['x'])
  1047. assert_array_equal(res['x'].shape, empty_sparse.shape)
  1048. assert_array_equal(res['x'].toarray(), 0)
  1049. # Do empty sparse matrices get written with max nnz 1?
  1050. # See https://github.com/scipy/scipy/issues/4208
  1051. sio.seek(0)
  1052. reader = MatFile5Reader(sio)
  1053. reader.initialize_read()
  1054. reader.read_file_header()
  1055. hdr, _ = reader.read_var_header()
  1056. assert_equal(hdr.nzmax, 1)
  1057. def test_empty_mat_error():
  1058. # Test we get a specific warning for an empty mat file
  1059. sio = BytesIO()
  1060. assert_raises(MatReadError, loadmat, sio)
  1061. def test_miuint32_compromise():
  1062. # Reader should accept miUINT32 for miINT32, but check signs
  1063. # mat file with miUINT32 for miINT32, but OK values
  1064. filename = pjoin(test_data_path, 'miuint32_for_miint32.mat')
  1065. res = loadmat(filename)
  1066. assert_equal(res['an_array'], np.arange(10)[None, :])
  1067. # mat file with miUINT32 for miINT32, with negative value
  1068. filename = pjoin(test_data_path, 'bad_miuint32.mat')
  1069. with assert_raises(ValueError):
  1070. loadmat(filename)
  1071. def test_miutf8_for_miint8_compromise():
  1072. # Check reader accepts ascii as miUTF8 for array names
  1073. filename = pjoin(test_data_path, 'miutf8_array_name.mat')
  1074. res = loadmat(filename)
  1075. assert_equal(res['array_name'], [[1]])
  1076. # mat file with non-ascii utf8 name raises error
  1077. filename = pjoin(test_data_path, 'bad_miutf8_array_name.mat')
  1078. with assert_raises(ValueError):
  1079. loadmat(filename)
  1080. def test_bad_utf8():
  1081. # Check that reader reads bad UTF with 'replace' option
  1082. filename = pjoin(test_data_path,'broken_utf8.mat')
  1083. res = loadmat(filename)
  1084. assert_equal(res['bad_string'],
  1085. b'\x80 am broken'.decode('utf8', 'replace'))
  1086. def test_save_unicode_field(tmpdir):
  1087. filename = os.path.join(str(tmpdir), 'test.mat')
  1088. test_dict = {'a':{'b':1,'c':'test_str'}}
  1089. savemat(filename, test_dict)
  1090. def test_save_custom_array_type(tmpdir):
  1091. class CustomArray:
  1092. def __array__(self, dtype=None, copy=None):
  1093. return np.arange(6.0).reshape(2, 3)
  1094. a = CustomArray()
  1095. filename = os.path.join(str(tmpdir), 'test.mat')
  1096. savemat(filename, {'a': a})
  1097. out = loadmat(filename)
  1098. assert_array_equal(out['a'], np.array(a))
  1099. def test_filenotfound():
  1100. # Check the correct error is thrown
  1101. assert_raises(OSError, loadmat, "NotExistentFile00.mat")
  1102. assert_raises(OSError, loadmat, "NotExistentFile00")
  1103. def test_simplify_cells():
  1104. # Test output when simplify_cells=True
  1105. filename = pjoin(test_data_path, 'testsimplecell.mat')
  1106. res1 = loadmat(filename, simplify_cells=True)
  1107. res2 = loadmat(filename, simplify_cells=False)
  1108. assert_(isinstance(res1["s"], dict))
  1109. assert_(isinstance(res2["s"], np.ndarray))
  1110. assert_array_equal(res1["s"]["mycell"], np.array(["a", "b", "c"]))
  1111. @pytest.mark.parametrize('version, filt, regex', [
  1112. (0, '_4*_*', None),
  1113. (1, '_5*_*', None),
  1114. (1, '_6*_*', None),
  1115. (1, '_7*_*', '^((?!hdf5).)*$'), # not containing hdf5
  1116. (2, '_7*_*', '.*hdf5.*'),
  1117. (1, '8*_*', None),
  1118. ])
  1119. def test_matfile_version(version, filt, regex):
  1120. use_filt = pjoin(test_data_path, f'test*{filt}.mat')
  1121. files = glob(use_filt)
  1122. if regex is not None:
  1123. files = [file for file in files if re.match(regex, file) is not None]
  1124. assert len(files) > 0, \
  1125. f"No files for version {version} using filter {filt}"
  1126. for file in files:
  1127. got_version = matfile_version(file)
  1128. assert got_version[0] == version
  1129. def test_opaque():
  1130. """Test that we can read a MatlabOpaque object."""
  1131. data = loadmat(pjoin(test_data_path, 'parabola.mat'))
  1132. assert isinstance(data['parabola'], MatlabFunction)
  1133. assert isinstance(data['parabola'].item()[3].item()[3], MatlabOpaque)
  1134. def test_opaque_simplify():
  1135. """Test that we can read a MatlabOpaque object when simplify_cells=True."""
  1136. data = loadmat(pjoin(test_data_path, 'parabola.mat'), simplify_cells=True)
  1137. assert isinstance(data['parabola'], MatlabFunction)
  1138. def test_deprecation():
  1139. """Test that access to previous attributes still works."""
  1140. # This should be accessible immediately from scipy.io import
  1141. with assert_warns(DeprecationWarning):
  1142. scipy.io.matlab.mio5_params.MatlabOpaque
  1143. # These should be importable but warn as well
  1144. with assert_warns(DeprecationWarning):
  1145. from scipy.io.matlab.miobase import MatReadError # noqa: F401
  1146. def test_gh_17992(tmp_path):
  1147. rng = np.random.default_rng(12345)
  1148. outfile = tmp_path / "lists.mat"
  1149. array_one = rng.random((5,3))
  1150. array_two = rng.random((6,3))
  1151. list_of_arrays = [array_one, array_two]
  1152. savemat(outfile,
  1153. {'data': list_of_arrays},
  1154. long_field_names=True,
  1155. do_compression=True)
  1156. # round trip check
  1157. new_dict = {}
  1158. loadmat(outfile,
  1159. new_dict)
  1160. assert_allclose(new_dict["data"][0][0], array_one)
  1161. assert_allclose(new_dict["data"][0][1], array_two)
  1162. def test_gh_19659(tmp_path):
  1163. d = {
  1164. "char_array": np.array([list("char"), list("char")], dtype="U1"),
  1165. "string_array": np.array(["string", "string"]),
  1166. }
  1167. outfile = tmp_path / "tmp.mat"
  1168. # should not error:
  1169. savemat(outfile, d, format="4")
  1170. def test_large_m4():
  1171. # Test we can read a Matlab 4 file with array > 2GB.
  1172. # (In fact, test we get the correct error from reading a truncated
  1173. # version).
  1174. # See https://github.com/scipy/scipy/issues/21256
  1175. # Data file is first 1024 bytes of:
  1176. # >>> a = np.zeros((134217728, 3))
  1177. # >>> siom.savemat('big_m4.mat', {'a': a}, format='4')
  1178. truncated_mat = pjoin(test_data_path, 'debigged_m4.mat')
  1179. match = ("Not enough bytes to read matrix 'a';"
  1180. if np.intp == np.int64 else
  1181. "Variable 'a' has byte length longer than largest possible")
  1182. with pytest.raises(ValueError, match=match):
  1183. loadmat(truncated_mat)
  1184. def test_gh_19223():
  1185. from scipy.io.matlab import varmats_from_mat # noqa: F401
  1186. def test_invalid_field_name_warning():
  1187. names_vars = (
  1188. ('_1', mlarr(np.arange(10))),
  1189. ('mystr', mlarr('a string')))
  1190. check_mat_write_warning(names_vars)
  1191. names_vars = (('mymap', {"a": 1, "_b": 2}),)
  1192. check_mat_write_warning(names_vars)
  1193. names_vars = (('mymap', {"a": 1, "1a": 2}),)
  1194. check_mat_write_warning(names_vars)
  1195. def check_mat_write_warning(names_vars):
  1196. class C:
  1197. def items(self):
  1198. return names_vars
  1199. stream = BytesIO()
  1200. with pytest.warns(MatWriteWarning, match='Starting field name with'):
  1201. savemat(stream, C())
  1202. def test_corrupt_files():
  1203. # Test we can detect truncated or corrupt (all zero) files.
  1204. for n in (2, 4, 10, 19):
  1205. with pytest.raises(MatReadError,
  1206. match="Mat file appears to be truncated"):
  1207. loadmat(BytesIO(b'\x00' * n))
  1208. with pytest.raises(MatReadError,
  1209. match="Mat file appears to be corrupt"):
  1210. loadmat(BytesIO(b'\x00' * 20))