test_jsonschema_test_suite.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. """
  2. Test runner for the JSON Schema official test suite
  3. Tests comprehensive correctness of each draft's validator.
  4. See https://github.com/json-schema-org/JSON-Schema-Test-Suite for details.
  5. """
  6. from jsonschema.tests._suite import Suite
  7. import jsonschema
  8. SUITE = Suite()
  9. DRAFT3 = SUITE.version(name="draft3")
  10. DRAFT4 = SUITE.version(name="draft4")
  11. DRAFT6 = SUITE.version(name="draft6")
  12. DRAFT7 = SUITE.version(name="draft7")
  13. DRAFT201909 = SUITE.version(name="draft2019-09")
  14. DRAFT202012 = SUITE.version(name="draft2020-12")
  15. def skip(message, **kwargs):
  16. def skipper(test):
  17. if all(value == getattr(test, attr) for attr, value in kwargs.items()):
  18. return message
  19. return skipper
  20. def ecmascript_regex(test):
  21. if test.subject == "ecmascript-regex":
  22. return "ECMA regex support will be added in #1142."
  23. def missing_format(Validator):
  24. def missing_format(test): # pragma: no cover
  25. schema = test.schema
  26. if (
  27. schema is True
  28. or schema is False
  29. or "format" not in schema
  30. or schema["format"] in Validator.FORMAT_CHECKER.checkers
  31. or test.valid
  32. ):
  33. return
  34. return f"Format checker {schema['format']!r} not found."
  35. return missing_format
  36. def complex_email_validation(test):
  37. if test.subject != "email":
  38. return
  39. message = "Complex email validation is (intentionally) unsupported."
  40. return skip(
  41. message=message,
  42. description="an invalid domain",
  43. )(test) or skip(
  44. message=message,
  45. description="an invalid IPv4-address-literal",
  46. )(test) or skip(
  47. message=message,
  48. description="dot after local part is not valid",
  49. )(test) or skip(
  50. message=message,
  51. description="dot before local part is not valid",
  52. )(test) or skip(
  53. message=message,
  54. description="two subsequent dots inside local part are not valid",
  55. )(test)
  56. def leap_second(test):
  57. message = "Leap seconds are unsupported."
  58. return skip(
  59. message=message,
  60. subject="time",
  61. description="a valid time string with leap second",
  62. )(test) or skip(
  63. message=message,
  64. subject="time",
  65. description="a valid time string with leap second, Zulu",
  66. )(test) or skip(
  67. message=message,
  68. subject="time",
  69. description="a valid time string with leap second with offset",
  70. )(test) or skip(
  71. message=message,
  72. subject="time",
  73. description="valid leap second, positive time-offset",
  74. )(test) or skip(
  75. message=message,
  76. subject="time",
  77. description="valid leap second, negative time-offset",
  78. )(test) or skip(
  79. message=message,
  80. subject="time",
  81. description="valid leap second, large positive time-offset",
  82. )(test) or skip(
  83. message=message,
  84. subject="time",
  85. description="valid leap second, large negative time-offset",
  86. )(test) or skip(
  87. message=message,
  88. subject="time",
  89. description="valid leap second, zero time-offset",
  90. )(test) or skip(
  91. message=message,
  92. subject="date-time",
  93. description="a valid date-time with a leap second, UTC",
  94. )(test) or skip(
  95. message=message,
  96. subject="date-time",
  97. description="a valid date-time with a leap second, with minus offset",
  98. )(test)
  99. TestDraft3 = DRAFT3.to_unittest_testcase(
  100. DRAFT3.cases(),
  101. DRAFT3.format_cases(),
  102. DRAFT3.optional_cases_of(name="bignum"),
  103. DRAFT3.optional_cases_of(name="non-bmp-regex"),
  104. DRAFT3.optional_cases_of(name="zeroTerminatedFloats"),
  105. Validator=jsonschema.Draft3Validator,
  106. format_checker=jsonschema.Draft3Validator.FORMAT_CHECKER,
  107. skip=lambda test: (
  108. ecmascript_regex(test)
  109. or missing_format(jsonschema.Draft3Validator)(test)
  110. or complex_email_validation(test)
  111. ),
  112. )
  113. TestDraft4 = DRAFT4.to_unittest_testcase(
  114. DRAFT4.cases(),
  115. DRAFT4.format_cases(),
  116. DRAFT4.optional_cases_of(name="bignum"),
  117. DRAFT4.optional_cases_of(name="float-overflow"),
  118. DRAFT4.optional_cases_of(name="id"),
  119. DRAFT4.optional_cases_of(name="non-bmp-regex"),
  120. DRAFT4.optional_cases_of(name="zeroTerminatedFloats"),
  121. Validator=jsonschema.Draft4Validator,
  122. format_checker=jsonschema.Draft4Validator.FORMAT_CHECKER,
  123. skip=lambda test: (
  124. ecmascript_regex(test)
  125. or leap_second(test)
  126. or missing_format(jsonschema.Draft4Validator)(test)
  127. or complex_email_validation(test)
  128. ),
  129. )
  130. TestDraft6 = DRAFT6.to_unittest_testcase(
  131. DRAFT6.cases(),
  132. DRAFT6.format_cases(),
  133. DRAFT6.optional_cases_of(name="bignum"),
  134. DRAFT6.optional_cases_of(name="float-overflow"),
  135. DRAFT6.optional_cases_of(name="id"),
  136. DRAFT6.optional_cases_of(name="non-bmp-regex"),
  137. Validator=jsonschema.Draft6Validator,
  138. format_checker=jsonschema.Draft6Validator.FORMAT_CHECKER,
  139. skip=lambda test: (
  140. ecmascript_regex(test)
  141. or leap_second(test)
  142. or missing_format(jsonschema.Draft6Validator)(test)
  143. or complex_email_validation(test)
  144. ),
  145. )
  146. TestDraft7 = DRAFT7.to_unittest_testcase(
  147. DRAFT7.cases(),
  148. DRAFT7.format_cases(),
  149. DRAFT7.optional_cases_of(name="bignum"),
  150. DRAFT7.optional_cases_of(name="cross-draft"),
  151. DRAFT7.optional_cases_of(name="float-overflow"),
  152. DRAFT6.optional_cases_of(name="id"),
  153. DRAFT7.optional_cases_of(name="non-bmp-regex"),
  154. DRAFT7.optional_cases_of(name="unknownKeyword"),
  155. Validator=jsonschema.Draft7Validator,
  156. format_checker=jsonschema.Draft7Validator.FORMAT_CHECKER,
  157. skip=lambda test: (
  158. ecmascript_regex(test)
  159. or leap_second(test)
  160. or missing_format(jsonschema.Draft7Validator)(test)
  161. or complex_email_validation(test)
  162. ),
  163. )
  164. TestDraft201909 = DRAFT201909.to_unittest_testcase(
  165. DRAFT201909.cases(),
  166. DRAFT201909.optional_cases_of(name="anchor"),
  167. DRAFT201909.optional_cases_of(name="bignum"),
  168. DRAFT201909.optional_cases_of(name="cross-draft"),
  169. DRAFT201909.optional_cases_of(name="float-overflow"),
  170. DRAFT201909.optional_cases_of(name="id"),
  171. DRAFT201909.optional_cases_of(name="no-schema"),
  172. DRAFT201909.optional_cases_of(name="non-bmp-regex"),
  173. DRAFT201909.optional_cases_of(name="refOfUnknownKeyword"),
  174. DRAFT201909.optional_cases_of(name="unknownKeyword"),
  175. Validator=jsonschema.Draft201909Validator,
  176. skip=skip(
  177. message="Vocabulary support is still in-progress.",
  178. subject="vocabulary",
  179. description=(
  180. "no validation: invalid number, but it still validates"
  181. ),
  182. ),
  183. )
  184. TestDraft201909Format = DRAFT201909.to_unittest_testcase(
  185. DRAFT201909.format_cases(),
  186. name="TestDraft201909Format",
  187. Validator=jsonschema.Draft201909Validator,
  188. format_checker=jsonschema.Draft201909Validator.FORMAT_CHECKER,
  189. skip=lambda test: (
  190. complex_email_validation(test)
  191. or ecmascript_regex(test)
  192. or leap_second(test)
  193. or missing_format(jsonschema.Draft201909Validator)(test)
  194. or complex_email_validation(test)
  195. ),
  196. )
  197. TestDraft202012 = DRAFT202012.to_unittest_testcase(
  198. DRAFT202012.cases(),
  199. DRAFT201909.optional_cases_of(name="anchor"),
  200. DRAFT202012.optional_cases_of(name="bignum"),
  201. DRAFT202012.optional_cases_of(name="cross-draft"),
  202. DRAFT202012.optional_cases_of(name="float-overflow"),
  203. DRAFT202012.optional_cases_of(name="id"),
  204. DRAFT202012.optional_cases_of(name="no-schema"),
  205. DRAFT202012.optional_cases_of(name="non-bmp-regex"),
  206. DRAFT202012.optional_cases_of(name="refOfUnknownKeyword"),
  207. DRAFT202012.optional_cases_of(name="unknownKeyword"),
  208. Validator=jsonschema.Draft202012Validator,
  209. skip=skip(
  210. message="Vocabulary support is still in-progress.",
  211. subject="vocabulary",
  212. description=(
  213. "no validation: invalid number, but it still validates"
  214. ),
  215. ),
  216. )
  217. TestDraft202012Format = DRAFT202012.to_unittest_testcase(
  218. DRAFT202012.format_cases(),
  219. name="TestDraft202012Format",
  220. Validator=jsonschema.Draft202012Validator,
  221. format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER,
  222. skip=lambda test: (
  223. complex_email_validation(test)
  224. or ecmascript_regex(test)
  225. or leap_second(test)
  226. or missing_format(jsonschema.Draft202012Validator)(test)
  227. or complex_email_validation(test)
  228. ),
  229. )