lib.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005
  1. # Copyright 2015 Google Inc. All rights reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import enum
  15. import math
  16. import re
  17. from typing import (
  18. Any,
  19. Callable,
  20. IO,
  21. Iterable,
  22. Mapping,
  23. Optional,
  24. Set,
  25. Tuple,
  26. Type,
  27. Union,
  28. )
  29. import unicodedata
  30. from json5.parser import Parser
  31. # Used when encoding keys, below.
  32. _reserved_word_re: Optional[re.Pattern] = None
  33. class QuoteStyle(enum.Enum):
  34. """Controls how strings will be quoted during encoding.
  35. By default, for compatibility with the `json` module and older versions of
  36. `json5`, strings (not being used as keys and that are legal identifiers)
  37. will always be double-quoted, and any double quotes in the string will be
  38. escaped. This is `QuoteStyle.ALWAYS_DOUBLE`. If you pass
  39. `QuoteStyle.ALWAYS_SINGLE`, then strings will always be single-quoted, and
  40. any single quotes in the string will be escaped. If you pass
  41. `QuoteStyle.PREFER_DOUBLE`, then the behavior is the same as ALWAYS_DOUBLE
  42. and strings will be double-quoted *unless* the string contains more double
  43. quotes than single quotes, in which case the string will be single-quoted
  44. and single quotes will be escaped. If you pass `QuoteStyle.PREFER_SINGLE`,
  45. then the behavior is the same as ALWAYS_SINGLE and strings will be
  46. single-quoted *unless* the string contains more single quotes than double
  47. quotes, in which case the string will be double-quoted and any double
  48. quotes will be escaped.
  49. *Note:* PREFER_DOUBLE and PREFER_SINGLE can impact performance, since in
  50. order to know which encoding to use you have to iterate over the entire
  51. string to count the number of single and double quotes. The codes guesses
  52. at an encoding while doing so, but if it guess wrong, the entire string has
  53. to be re-encoded, which will slow things down. If you are very concerned
  54. about performance (a) you probably shouldn't be using this library in the
  55. first place, because it just isn't very fast, and (b) you should use
  56. ALWAYS_DOUBLE or ALWAYS_SINGLE, which won't have this issue.
  57. """
  58. ALWAYS_DOUBLE = 'always_double'
  59. ALWAYS_SINGLE = 'always_single'
  60. PREFER_DOUBLE = 'prefer_double'
  61. PREFER_SINGLE = 'prefer_single'
  62. def load(
  63. fp: IO,
  64. *,
  65. encoding: Optional[str] = None,
  66. cls: Any = None,
  67. object_hook: Optional[Callable[[Mapping[str, Any]], Any]] = None,
  68. parse_float: Optional[Callable[[str], Any]] = None,
  69. parse_int: Optional[Callable[[str], Any]] = None,
  70. parse_constant: Optional[Callable[[str], Any]] = None,
  71. strict: bool = True,
  72. object_pairs_hook: Optional[
  73. Callable[[Iterable[Tuple[str, Any]]], Any]
  74. ] = None,
  75. allow_duplicate_keys: bool = True,
  76. consume_trailing: bool = True,
  77. start: Optional[int] = None,
  78. ) -> Any:
  79. """Deserialize ``fp`` (a ``.read()``-supporting file-like object
  80. containing a JSON document) to a Python object.
  81. Supports almost the same arguments as ``json.load()`` except that:
  82. - the `cls` keyword is ignored.
  83. - an extra `allow_duplicate_keys` parameter supports checking for
  84. duplicate keys in a object; by default, this is True for
  85. compatibility with ``json.load()``, but if set to False and
  86. the object contains duplicate keys, a ValueError will be raised.
  87. - an extra `consume_trailing` parameter specifies whether to
  88. consume any trailing characters after a valid object has been
  89. parsed. By default, this value is True and the only legal
  90. trailing characters are whitespace. If this value is set to False,
  91. parsing will stop when a valid object has been parsed and any
  92. trailing characters in the string will be ignored.
  93. - an extra `start` parameter specifies the zero-based offset into the
  94. file to start parsing at. If `start` is None, parsing will
  95. start at the current position in the file, and line number
  96. and column values will be reported as if starting from the
  97. beginning of the file; If `start` is not None,
  98. `load` will seek to zero and then read (and discard) the
  99. appropriate number of characters before beginning parsing;
  100. the file must be seekable for this to work correctly.
  101. You can use `load(..., consume_trailing=False)` to repeatedly read
  102. values from a file. However, in the current implementation `load` does
  103. this by reading the entire file into memory before doing anything, so
  104. it is not very efficient.
  105. Raises
  106. - `ValueError` if given an invalid document. This is different
  107. from the `json` module, which raises `json.JSONDecodeError`.
  108. - `UnicodeDecodeError` if given a byte string that is not a
  109. legal UTF-8 document (or the equivalent, if using a different
  110. `encoding`). This matches the `json` module.
  111. """
  112. s = fp.read()
  113. val, err, _ = parse(
  114. s,
  115. encoding=encoding,
  116. cls=cls,
  117. object_hook=object_hook,
  118. parse_float=parse_float,
  119. parse_int=parse_int,
  120. parse_constant=parse_constant,
  121. strict=strict,
  122. object_pairs_hook=object_pairs_hook,
  123. allow_duplicate_keys=allow_duplicate_keys,
  124. consume_trailing=consume_trailing,
  125. start=start,
  126. )
  127. if err:
  128. raise ValueError(err)
  129. return val
  130. def loads(
  131. s: str,
  132. *,
  133. encoding: Optional[str] = None,
  134. cls: Any = None,
  135. object_hook: Optional[Callable[[Mapping[str, Any]], Any]] = None,
  136. parse_float: Optional[Callable[[str], Any]] = None,
  137. parse_int: Optional[Callable[[str], Any]] = None,
  138. parse_constant: Optional[Callable[[str], Any]] = None,
  139. strict: bool = True,
  140. object_pairs_hook: Optional[
  141. Callable[[Iterable[Tuple[str, Any]]], Any]
  142. ] = None,
  143. allow_duplicate_keys: bool = True,
  144. consume_trailing: bool = True,
  145. start: Optional[int] = None,
  146. ):
  147. """Deserialize ``s`` (a string containing a JSON5 document) to a Python
  148. object.
  149. Supports the same arguments as ``json.load()`` except that:
  150. - the `cls` keyword is ignored.
  151. - an extra `allow_duplicate_keys` parameter supports checking for
  152. duplicate keys in a object; by default, this is True for
  153. compatibility with ``json.load()``, but if set to False and
  154. the object contains duplicate keys, a ValueError will be raised.
  155. - an extra `consume_trailing` parameter specifies whether to
  156. consume any trailing characters after a valid object has been
  157. parsed. By default, this value is True and the only legal
  158. trailing characters are whitespace. If this value is set to False,
  159. parsing will stop when a valid object has been parsed and any
  160. trailing characters in the string will be ignored.
  161. - an extra `start` parameter specifies the zero-based offset into the
  162. string to start parsing at.
  163. Raises
  164. - `ValueError` if given an invalid document. This is different
  165. from the `json` module, which raises `json.JSONDecodeError`.
  166. - `UnicodeDecodeError` if given a byte string that is not a
  167. legal UTF-8 document (or the equivalent, if using a different
  168. `encoding`). This matches the `json` module.
  169. """
  170. val, err, _ = parse(
  171. s=s,
  172. encoding=encoding,
  173. cls=cls,
  174. object_hook=object_hook,
  175. parse_float=parse_float,
  176. parse_int=parse_int,
  177. parse_constant=parse_constant,
  178. strict=strict,
  179. object_pairs_hook=object_pairs_hook,
  180. allow_duplicate_keys=allow_duplicate_keys,
  181. consume_trailing=consume_trailing,
  182. start=start,
  183. )
  184. if err:
  185. raise ValueError(err)
  186. return val
  187. def parse(
  188. s: str,
  189. *,
  190. encoding: Optional[str] = None,
  191. cls: Any = None,
  192. object_hook: Optional[Callable[[Mapping[str, Any]], Any]] = None,
  193. parse_float: Optional[Callable[[str], Any]] = None,
  194. parse_int: Optional[Callable[[str], Any]] = None,
  195. parse_constant: Optional[Callable[[str], Any]] = None,
  196. strict: bool = True,
  197. object_pairs_hook: Optional[
  198. Callable[[Iterable[Tuple[str, Any]]], Any]
  199. ] = None,
  200. allow_duplicate_keys: bool = True,
  201. consume_trailing: bool = True,
  202. start: Optional[int] = None,
  203. ):
  204. """Parse ```s``, returning positional information along with a value.
  205. This works exactly like `loads()`, except that (a) it returns the
  206. position in the string where the parsing stopped (either due to
  207. hitting an error or parsing a valid value) and any error as a string,
  208. (b) it takes an optional `consume_trailing` parameter that says whether
  209. to keep parsing the string after a valid value has been parsed; if True
  210. (the default), any trailing characters must be whitespace. If False,
  211. parsing stops when a valid value has been reached, (c) it takes an
  212. optional `start` parameter that specifies a zero-based offset to start
  213. parsing from in the string, and (d) the return value is different, as
  214. described below.
  215. `parse()` is useful if you have a string that might contain multiple
  216. values and you need to extract all of them; you can do so by repeatedly
  217. calling `parse`, setting `start` to the value returned in `position`
  218. from the previous call.
  219. Returns a tuple of (value, error_string, position). If the string
  220. was a legal value, `value` will be the deserialized value,
  221. `error_string` will be `None`, and `position` will be one
  222. past the zero-based offset where the parser stopped reading.
  223. If the string was not a legal value,
  224. `value` will be `None`, `error_string` will be the string value
  225. of the exception that would've been raised, and `position` will
  226. be the zero-based farthest offset into the string where the parser
  227. hit an error.
  228. Raises:
  229. - `UnicodeDecodeError` if given a byte string that is not a
  230. legal UTF-8 document (or the equivalent, if using a different
  231. `encoding`). This matches the `json` module.
  232. Note that this does *not* raise a `ValueError`; instead any error is
  233. returned as the second value in the tuple.
  234. You can use this method to read in a series of values from a string
  235. `s` as follows:
  236. >>> import json5
  237. >>> s = '1 2 3 4'
  238. >>> values = []
  239. >>> start = 0
  240. >>> while True:
  241. ... v, err, pos = json5.parse(s, start=start, consume_trailing=False)
  242. ... if v:
  243. ... values.append(v)
  244. ... start = pos
  245. ... if start == len(s) or s[start:].isspace():
  246. ... # Reached the end of the string (ignoring trailing
  247. ... # whitespace
  248. ... break
  249. ... continue
  250. ... raise ValueError(err)
  251. >>> values
  252. [1, 2, 3, 4]
  253. """
  254. assert cls is None, 'Custom decoders are not supported'
  255. if isinstance(s, bytes):
  256. encoding = encoding or 'utf-8'
  257. s = s.decode(encoding)
  258. if not s:
  259. raise ValueError('Empty strings are not legal JSON5')
  260. start = start or 0
  261. parser = Parser(s, '<string>', pos=start)
  262. ast, err, pos = parser.parse(
  263. global_vars={'_strict': strict, '_consume_trailing': consume_trailing}
  264. )
  265. if err:
  266. return None, err, pos
  267. try:
  268. value = _convert(
  269. ast,
  270. object_hook=object_hook,
  271. parse_float=parse_float,
  272. parse_int=parse_int,
  273. parse_constant=parse_constant,
  274. object_pairs_hook=object_pairs_hook,
  275. allow_duplicate_keys=allow_duplicate_keys,
  276. )
  277. return value, None, pos
  278. except ValueError as e:
  279. return None, str(e), pos
  280. def _convert(
  281. ast,
  282. object_hook,
  283. parse_float,
  284. parse_int,
  285. parse_constant,
  286. object_pairs_hook,
  287. allow_duplicate_keys,
  288. ):
  289. def _fp_constant_parser(s):
  290. return float(s.replace('Infinity', 'inf').replace('NaN', 'nan'))
  291. def _dictify(pairs):
  292. if not allow_duplicate_keys:
  293. keys = set()
  294. for key, _ in pairs:
  295. if key in keys:
  296. raise ValueError(f'Duplicate key "{key}" found in object')
  297. keys.add(key)
  298. if object_pairs_hook:
  299. return object_pairs_hook(pairs)
  300. if object_hook:
  301. return object_hook(dict(pairs))
  302. return dict(pairs)
  303. parse_float = parse_float or float
  304. parse_int = parse_int or int
  305. parse_constant = parse_constant or _fp_constant_parser
  306. return _walk_ast(ast, _dictify, parse_float, parse_int, parse_constant)
  307. def _walk_ast(
  308. el,
  309. dictify: Callable[[Iterable[Tuple[str, Any]]], Any],
  310. parse_float,
  311. parse_int,
  312. parse_constant,
  313. ):
  314. if el == 'None':
  315. return None
  316. if el == 'True':
  317. return True
  318. if el == 'False':
  319. return False
  320. ty, v = el
  321. if ty == 'number':
  322. if v.startswith('0x') or v.startswith('0X'):
  323. return parse_int(v, base=16)
  324. if '.' in v or 'e' in v or 'E' in v:
  325. return parse_float(v)
  326. if 'Infinity' in v or 'NaN' in v:
  327. return parse_constant(v)
  328. return parse_int(v)
  329. if ty == 'string':
  330. return v
  331. if ty == 'object':
  332. pairs = []
  333. for key, val_expr in v:
  334. val = _walk_ast(
  335. val_expr, dictify, parse_float, parse_int, parse_constant
  336. )
  337. pairs.append((key, val))
  338. return dictify(pairs)
  339. if ty == 'array':
  340. return [
  341. _walk_ast(el, dictify, parse_float, parse_int, parse_constant)
  342. for el in v
  343. ]
  344. raise ValueError('unknown el: ' + el) # pragma: no cover
  345. def dump(
  346. obj: Any,
  347. fp: IO,
  348. *,
  349. skipkeys: bool = False,
  350. ensure_ascii: bool = True,
  351. check_circular: bool = True,
  352. allow_nan: bool = True,
  353. cls: Optional[Type['JSON5Encoder']] = None,
  354. indent: Optional[Union[int, str]] = None,
  355. separators: Optional[Tuple[str, str]] = None,
  356. default: Optional[Callable[[Any], Any]] = None,
  357. sort_keys: bool = False,
  358. quote_keys: bool = False,
  359. trailing_commas: bool = True,
  360. allow_duplicate_keys: bool = True,
  361. quote_style: QuoteStyle = QuoteStyle.ALWAYS_DOUBLE,
  362. **kw,
  363. ):
  364. """Serialize ``obj`` to a JSON5-formatted stream to ``fp``,
  365. a ``.write()``-supporting file-like object.
  366. Supports the same arguments as ``dumps()``, below.
  367. Calling ``dump(obj, fp, quote_keys=True, trailing_commas=False, \
  368. allow_duplicate_keys=True)``
  369. should produce exactly the same output as ``json.dump(obj, fp).``
  370. """
  371. fp.write(
  372. dumps(
  373. obj=obj,
  374. skipkeys=skipkeys,
  375. ensure_ascii=ensure_ascii,
  376. check_circular=check_circular,
  377. allow_nan=allow_nan,
  378. cls=cls,
  379. indent=indent,
  380. separators=separators,
  381. default=default,
  382. sort_keys=sort_keys,
  383. quote_keys=quote_keys,
  384. trailing_commas=trailing_commas,
  385. allow_duplicate_keys=allow_duplicate_keys,
  386. quote_style=quote_style,
  387. **kw,
  388. )
  389. )
  390. def dumps(
  391. obj: Any,
  392. *,
  393. skipkeys: bool = False,
  394. ensure_ascii: bool = True,
  395. check_circular: bool = True,
  396. allow_nan: bool = True,
  397. cls: Optional[Type['JSON5Encoder']] = None,
  398. indent: Optional[Union[int, str]] = None,
  399. separators: Optional[Tuple[str, str]] = None,
  400. default: Optional[Callable[[Any], Any]] = None,
  401. sort_keys: bool = False,
  402. quote_keys: bool = False,
  403. trailing_commas: bool = True,
  404. allow_duplicate_keys: bool = True,
  405. quote_style: QuoteStyle = QuoteStyle.ALWAYS_DOUBLE,
  406. **kw: Any,
  407. ):
  408. """Serialize ``obj`` to a JSON5-formatted string.
  409. Supports the same arguments as ``json.dumps()``, except that:
  410. - The ``encoding`` keyword is ignored; Unicode strings are always written.
  411. - By default, object keys that are legal identifiers are not quoted; if you
  412. pass ``quote_keys=True``, they will be.
  413. - By default, if lists and objects span multiple lines of output (i.e.,
  414. when ``indent`` >=0), the last item will have a trailing comma after it.
  415. If you pass ``trailing_commas=False``, it will not.
  416. - If you use a number, a boolean, or ``None`` as a key value in a dict, it
  417. will be converted to the corresponding JSON string value, e.g. "1",
  418. "true", or "null". By default, ``dump()`` will match the `json` modules
  419. behavior and produce malformed JSON if you mix keys of different types
  420. that have the same converted value; e.g., ``{1: "foo", "1": "bar"}``
  421. produces '{"1": "foo", "1": "bar"}', an object with duplicated keys. If
  422. you pass ``allow_duplicate_keys=False``, an exception will be raised
  423. instead.
  424. - If `quote_keys` is true, then keys of objects will be enclosed in quotes,
  425. as in regular JSON. Otheriwse, keys will not be enclosed in quotes unless
  426. they contain whitespace.
  427. - If `trailing_commas` is false, then commas will not be inserted after the
  428. final elements of objects and arrays, as in regular JSON. Otherwise,
  429. such commas will be inserted.
  430. - If `allow_duplicate_keys` is false, then only the last entry with a given
  431. key will be written. Otherwise, all entries with the same key will be
  432. written.
  433. - `quote_style` controls how strings are encoded. See the documentation
  434. for the `QuoteStyle` class, above, for how this is used.
  435. *Note*: Strings that are being used as unquoted keys are not affected
  436. by this parameter and remain unquoted.
  437. *`quote_style` was added in version 0.10.0*.
  438. Other keyword arguments are allowed and will be passed to the
  439. encoder so custom encoders can get them, but otherwise they will
  440. be ignored in an attempt to provide some amount of forward-compatibility.
  441. *Note:* the standard JSON module explicitly calls `int.__repr(obj)__`
  442. and `float.__repr(obj)__` to encode ints and floats, thereby bypassing
  443. any custom representations you might have for objects that are subclasses
  444. of ints and floats, and, for compatibility, JSON5 does the same thing.
  445. To override this behavior, create a subclass of JSON5Encoder
  446. that overrides `encode()` and handles your custom representation.
  447. For example:
  448. ```
  449. >>> import json5
  450. >>> from typing import Any, Set
  451. >>>
  452. >>> class Hex(int):
  453. ... def __repr__(self):
  454. ... return hex(self)
  455. >>>
  456. >>> class CustomEncoder(json5.JSON5Encoder):
  457. ... def encode(
  458. ... self, obj: Any, seen: Set, level: int, *, as_key: bool
  459. ... ) -> str:
  460. ... if isinstance(obj, Hex):
  461. ... return repr(obj)
  462. ... return super().encode(obj, seen, level, as_key=as_key)
  463. ...
  464. >>> json5.dumps([20, Hex(20)], cls=CustomEncoder)
  465. '[20, 0x14]'
  466. ```
  467. *Note:* calling ``dumps(obj, quote_keys=True, trailing_commas=False, \
  468. allow_duplicate_keys=True)``
  469. should produce exactly the same output as ``json.dumps(obj).``
  470. """
  471. cls = cls or JSON5Encoder
  472. enc = cls(
  473. skipkeys=skipkeys,
  474. ensure_ascii=ensure_ascii,
  475. check_circular=check_circular,
  476. allow_nan=allow_nan,
  477. indent=indent,
  478. separators=separators,
  479. default=default,
  480. sort_keys=sort_keys,
  481. quote_keys=quote_keys,
  482. trailing_commas=trailing_commas,
  483. allow_duplicate_keys=allow_duplicate_keys,
  484. quote_style=quote_style,
  485. **kw,
  486. )
  487. return enc.encode(obj, seen=set(), level=0, as_key=False)
  488. class JSON5Encoder:
  489. def __init__(
  490. self,
  491. *,
  492. skipkeys: bool = False,
  493. ensure_ascii: bool = True,
  494. check_circular: bool = True,
  495. allow_nan: bool = True,
  496. indent: Optional[Union[int, str]] = None,
  497. separators: Optional[Tuple[str, str]] = None,
  498. default: Optional[Callable[[Any], Any]] = None,
  499. sort_keys: bool = False,
  500. quote_keys: bool = False,
  501. trailing_commas: bool = True,
  502. allow_duplicate_keys: bool = True,
  503. quote_style: QuoteStyle = QuoteStyle.ALWAYS_DOUBLE,
  504. **kw,
  505. ):
  506. """Provides a class that may be overridden to customize the behavior
  507. of `dumps()`. The keyword args are the same as for that function.
  508. *Added in version 0.10.0"""
  509. # Ignore unrecognized keyword arguments in the hope of providing
  510. # some level of backwards- and forwards-compatibility.
  511. del kw
  512. self.skipkeys = skipkeys
  513. self.ensure_ascii = ensure_ascii
  514. self.check_circular = check_circular
  515. self.allow_nan = allow_nan
  516. self.indent = indent
  517. self.separators = separators
  518. if separators is None:
  519. separators = (', ', ': ') if indent is None else (',', ': ')
  520. self.item_separator, self.kv_separator = separators
  521. self.default_fn = default or _raise_type_error
  522. self.sort_keys = sort_keys
  523. self.quote_keys = quote_keys
  524. self.trailing_commas = trailing_commas
  525. self.allow_duplicate_keys = allow_duplicate_keys
  526. self.quote_style = quote_style
  527. def default(self, obj: Any) -> Any:
  528. """Provides a last-ditch option to encode a value that the encoder
  529. doesn't otherwise recognize, by converting `obj` to a value that
  530. *can* (and will) be serialized by the other methods in the class.
  531. Note: this must not return a serialized value (i.e., string)
  532. directly, as that'll result in a doubly-encoded value."""
  533. return self.default_fn(obj)
  534. def encode(
  535. self,
  536. obj: Any,
  537. seen: Set,
  538. level: int,
  539. *,
  540. as_key: bool,
  541. ) -> str:
  542. """Returns an JSON5-encoded version of an arbitrary object. This can
  543. be used to provide customized serialization of objects. Overridden
  544. methods of this class should handle their custom objects and then
  545. fall back to super.encode() if they've been passed a normal object.
  546. `seen` is used for duplicate object tracking when `check_circular`
  547. is True.
  548. `level` represents the current indentation level, which increases
  549. by one for each recursive invocation of encode (i.e., whenever
  550. we're encoding the values of a dict or a list).
  551. May raise `TypeError` if the object is the wrong type to be
  552. encoded (i.e., your custom routine can't handle it either), and
  553. `ValueError` if there's something wrong with the value, e.g.
  554. a float value of NaN when `allow_nan` is false.
  555. If `as_key` is true, the return value should be a double-quoted string
  556. representation of the object, unless obj is a string that can be an
  557. identifier (and quote_keys is false and obj isn't a reserved word).
  558. If the object should not be used as a key, `TypeError` should be
  559. raised; that allows the base implementation to implement `skipkeys`
  560. properly.
  561. """
  562. seen = seen or set()
  563. s = self._encode_basic_type(obj, as_key=as_key)
  564. if s is not None:
  565. return s
  566. if as_key:
  567. raise TypeError(f'Invalid key f{obj}')
  568. return self._encode_non_basic_type(obj, seen, level)
  569. def _encode_basic_type(self, obj: Any, *, as_key: bool) -> Optional[str]:
  570. """Returns None if the object is not a basic type."""
  571. if isinstance(obj, str):
  572. return self._encode_str(obj, as_key=as_key)
  573. # Check for True/False before ints because True and False are
  574. # also considered ints and so would be represented as 1 and 0
  575. # if we did ints first.
  576. if obj is True:
  577. return '"true"' if as_key else 'true'
  578. if obj is False:
  579. return '"false"' if as_key else 'false'
  580. if obj is None:
  581. return '"null"' if as_key else 'null'
  582. if isinstance(obj, int):
  583. return self._encode_int(obj, as_key=as_key)
  584. if isinstance(obj, float):
  585. return self._encode_float(obj, as_key=as_key)
  586. return None
  587. def _encode_int(self, obj: int, *, as_key: bool) -> str:
  588. s = int.__repr__(obj)
  589. return f'"{s}"' if as_key else s
  590. def _encode_float(self, obj: float, *, as_key: bool) -> str:
  591. if obj == float('inf'):
  592. allowed = self.allow_nan
  593. s = 'Infinity'
  594. elif obj == float('-inf'):
  595. allowed = self.allow_nan
  596. s = '-Infinity'
  597. elif math.isnan(obj):
  598. allowed = self.allow_nan
  599. s = 'NaN'
  600. else:
  601. allowed = True
  602. s = float.__repr__(obj)
  603. if not allowed:
  604. raise ValueError('Illegal JSON5 value: f{obj}')
  605. return f'"{s}"' if as_key else s
  606. def _encode_str(self, obj: str, *, as_key: bool) -> str:
  607. if (
  608. as_key
  609. and self.is_identifier(obj)
  610. and not self.quote_keys
  611. and not self.is_reserved_word(obj)
  612. ):
  613. return obj
  614. return self._encode_quoted_str(obj, self.quote_style)
  615. def _encode_quoted_str(self, obj: str, quote_style: QuoteStyle) -> str:
  616. """Returns a quoted string with a minimal number of escaped quotes."""
  617. ret = []
  618. double_quotes_seen = 0
  619. single_quotes_seen = 0
  620. sq = "'"
  621. dq = '"'
  622. for ch in obj:
  623. if ch == dq:
  624. # At first we will guess at which quotes to escape. If
  625. # we guess wrong, we reencode the string below.
  626. double_quotes_seen += 1
  627. if quote_style in (
  628. QuoteStyle.ALWAYS_DOUBLE,
  629. QuoteStyle.PREFER_DOUBLE,
  630. ):
  631. encoded_ch = self._escape_ch(dq)
  632. else:
  633. encoded_ch = dq
  634. elif ch == sq:
  635. single_quotes_seen += 1
  636. if quote_style in (
  637. QuoteStyle.ALWAYS_SINGLE,
  638. QuoteStyle.PREFER_SINGLE,
  639. ):
  640. encoded_ch = self._escape_ch(sq)
  641. else:
  642. encoded_ch = sq
  643. elif ch == '\\':
  644. encoded_ch = self._escape_ch(ch)
  645. else:
  646. o = ord(ch)
  647. if o < 32:
  648. encoded_ch = self._escape_ch(ch)
  649. elif o < 128:
  650. encoded_ch = ch
  651. elif not self.ensure_ascii and ch not in ('\u2028', '\u2029'):
  652. encoded_ch = ch
  653. else:
  654. encoded_ch = self._escape_ch(ch)
  655. ret.append(encoded_ch)
  656. # We may have guessed wrong and need to reencode the string.
  657. if (
  658. double_quotes_seen > single_quotes_seen
  659. and quote_style == QuoteStyle.PREFER_DOUBLE
  660. ):
  661. return self._encode_quoted_str(obj, QuoteStyle.ALWAYS_SINGLE)
  662. if (
  663. single_quotes_seen > double_quotes_seen
  664. and quote_style == QuoteStyle.PREFER_SINGLE
  665. ):
  666. return self._encode_quoted_str(obj, QuoteStyle.ALWAYS_DOUBLE)
  667. if quote_style in (QuoteStyle.ALWAYS_DOUBLE, QuoteStyle.PREFER_DOUBLE):
  668. return '"' + ''.join(ret) + '"'
  669. return "'" + ''.join(ret) + "'"
  670. def _escape_ch(self, ch: str) -> str:
  671. """Returns the backslash-escaped representation of the char."""
  672. if ch == '\\':
  673. return '\\\\'
  674. if ch == "'":
  675. return r'\''
  676. if ch == '"':
  677. return r'\"'
  678. if ch == '\n':
  679. return r'\n'
  680. if ch == '\r':
  681. return r'\r'
  682. if ch == '\t':
  683. return r'\t'
  684. if ch == '\b':
  685. return r'\b'
  686. if ch == '\f':
  687. return r'\f'
  688. if ch == '\v':
  689. return r'\v'
  690. if ch == '\0':
  691. return r'\0'
  692. o = ord(ch)
  693. if o < 65536:
  694. return rf'\u{o:04x}'
  695. val = o - 0x10000
  696. high = 0xD800 + (val >> 10)
  697. low = 0xDC00 + (val & 0x3FF)
  698. return rf'\u{high:04x}\u{low:04x}'
  699. def _encode_non_basic_type(self, obj, seen: Set, level: int) -> str:
  700. # Basic types can't be recursive so we only check for circularity
  701. # on non-basic types. If for some reason the caller was using a
  702. # subclass of a basic type and wanted to check circularity on it,
  703. # it'd have to do so directly in a subclass of JSON5Encoder.
  704. if self.check_circular:
  705. i = id(obj)
  706. if i in seen:
  707. raise ValueError('Circular reference detected.')
  708. seen.add(i)
  709. # Ideally we'd use collections.abc.Mapping and collections.abc.Sequence
  710. # here, but for backwards-compatibility with potential old callers,
  711. # we only check for the two attributes we need in each case.
  712. if hasattr(obj, 'keys') and hasattr(obj, '__getitem__'):
  713. s = self._encode_dict(obj, seen, level + 1)
  714. elif hasattr(obj, '__getitem__') and hasattr(obj, '__iter__'):
  715. s = self._encode_array(obj, seen, level + 1)
  716. else:
  717. s = self.encode(self.default(obj), seen, level, as_key=False)
  718. assert s is not None
  719. if self.check_circular:
  720. seen.remove(i)
  721. return s
  722. def _encode_dict(self, obj: Any, seen: set, level: int) -> str:
  723. if not obj:
  724. return '{}'
  725. indent_str, end_str = self._spacers(level)
  726. item_sep = self.item_separator + indent_str
  727. kv_sep = self.kv_separator
  728. if self.sort_keys:
  729. keys = sorted(obj.keys())
  730. else:
  731. keys = obj.keys()
  732. s = '{' + indent_str
  733. first_key = True
  734. new_keys = set()
  735. for key in keys:
  736. try:
  737. key_str = self.encode(key, seen, level, as_key=True)
  738. except TypeError:
  739. if self.skipkeys:
  740. continue
  741. raise
  742. if not self.allow_duplicate_keys:
  743. if key_str in new_keys:
  744. raise ValueError(f'duplicate key {repr(key)}')
  745. new_keys.add(key_str)
  746. if first_key:
  747. first_key = False
  748. else:
  749. s += item_sep
  750. val_str = self.encode(obj[key], seen, level, as_key=False)
  751. s += key_str + kv_sep + val_str
  752. s += end_str + '}'
  753. return s
  754. def _encode_array(self, obj: Any, seen: Set, level: int) -> str:
  755. if not obj:
  756. return '[]'
  757. indent_str, end_str = self._spacers(level)
  758. item_sep = self.item_separator + indent_str
  759. return (
  760. '['
  761. + indent_str
  762. + item_sep.join(
  763. self.encode(el, seen, level, as_key=False) for el in obj
  764. )
  765. + end_str
  766. + ']'
  767. )
  768. def _spacers(self, level: int) -> Tuple[str, str]:
  769. if self.indent is not None:
  770. end_str = ''
  771. if self.trailing_commas:
  772. end_str = ','
  773. if isinstance(self.indent, int):
  774. if self.indent > 0:
  775. indent_str = '\n' + ' ' * self.indent * level
  776. end_str += '\n' + ' ' * self.indent * (level - 1)
  777. else:
  778. indent_str = '\n'
  779. end_str += '\n'
  780. else:
  781. indent_str = '\n' + self.indent * level
  782. end_str += '\n' + self.indent * (level - 1)
  783. else:
  784. indent_str = ''
  785. end_str = ''
  786. return indent_str, end_str
  787. def is_identifier(self, key: str) -> bool:
  788. """Returns whether the string could be used as a legal
  789. EcmaScript/JavaScript identifier.
  790. There should normally be no reason to override this, unless
  791. the definition of identifiers change in later versions of the
  792. JSON5 spec and this implementation hasn't been updated to handle
  793. the changes yet."""
  794. if (
  795. not key
  796. or not self._is_id_start(key[0])
  797. and key[0] not in ('$', '_')
  798. ):
  799. return False
  800. for ch in key[1:]:
  801. if not self._is_id_continue(ch) and ch not in ('$', '_'):
  802. return False
  803. return True
  804. def _is_id_start(self, ch: str) -> bool:
  805. return unicodedata.category(ch) in (
  806. 'Lu',
  807. 'Ll',
  808. 'Li',
  809. 'Lt',
  810. 'Lm',
  811. 'Lo',
  812. 'Nl',
  813. )
  814. def _is_id_continue(self, ch: str) -> bool:
  815. return unicodedata.category(ch) in (
  816. 'Lu',
  817. 'Ll',
  818. 'Li',
  819. 'Lt',
  820. 'Lm',
  821. 'Lo',
  822. 'Nl',
  823. 'Nd',
  824. 'Mn',
  825. 'Mc',
  826. 'Pc',
  827. )
  828. def is_reserved_word(self, key: str) -> bool:
  829. """Returns whether the key is a reserved word.
  830. There should normally be no need to override this, unless there
  831. have been reserved words added in later versions of the JSON5
  832. spec and this implementation has not yet been updated to handle
  833. the changes yet."""
  834. global _reserved_word_re
  835. if _reserved_word_re is None:
  836. # List taken from section 7.6.1 of ECMA-262, version 5.1.
  837. # https://262.ecma-international.org/5.1/#sec-7.6.1.
  838. # This includes currently reserved words, words reserved
  839. # for future use (both as of 5.1), null, true, and false.
  840. _reserved_word_re = re.compile(
  841. '('
  842. + '|'.join(
  843. [
  844. 'break',
  845. 'case',
  846. 'catch',
  847. 'class',
  848. 'const',
  849. 'continue',
  850. 'debugger',
  851. 'default',
  852. 'delete',
  853. 'do',
  854. 'else',
  855. 'enum',
  856. 'export',
  857. 'extends',
  858. 'false',
  859. 'finally',
  860. 'for',
  861. 'function',
  862. 'if',
  863. 'implements',
  864. 'import',
  865. 'in',
  866. 'instanceof',
  867. 'interface',
  868. 'let',
  869. 'new',
  870. 'null',
  871. 'package',
  872. 'private',
  873. 'protected',
  874. 'public',
  875. 'return',
  876. 'static',
  877. 'super',
  878. 'switch',
  879. 'this',
  880. 'throw',
  881. 'true',
  882. 'try',
  883. 'typeof',
  884. 'var',
  885. 'void',
  886. 'while',
  887. 'with',
  888. 'yield',
  889. ]
  890. )
  891. + ')$'
  892. )
  893. return _reserved_word_re.match(key) is not None
  894. def _raise_type_error(obj) -> Any:
  895. raise TypeError(f'{repr(obj)} is not JSON5 serializable')