Add net/JSONTestSuite tests. Fix JsonPointer OOME on "[[[[..." (#106)

readme.md

@@ -367,3 +367,5 @@ uPickle: a simple Scala JSON and Binary (MessagePack) serialization library
 If you use uPickle/weePickle and like it, please support it by donating to lihaoyi's Patreon:
 
 - [https://www.patreon.com/lihaoyi](https://www.patreon.com/lihaoyi)
+
+Thanks to [JSONTestSuite](https://github.com/nst/JSONTestSuite) for the comprehensive collection of interesting JSON test files.

weejson-jackson/src/main/scala/com/rallyhealth/weejson/v1/jackson/JsonParserException.scala

@@ -1,16 +1,19 @@
 package com.rallyhealth.weejson.v1.jackson
 
-import com.fasterxml.jackson.core.JsonParser
+import com.fasterxml.jackson.core.{JsonParser, JsonStreamContext}
 import com.fasterxml.jackson.core.base.ParserBase
 import com.rallyhealth.weepickle.v1.core.TransformException
 
+import scala.collection.compat._
+import scala.collection.mutable.ArrayBuffer
+
 object JsonParserException {
 
   def apply(msg: String, parser: JsonParser, t: Throwable = null): Throwable = {
     val parserBase = Option(parser).collect { case pb: ParserBase => pb }
     new TransformException(
       shortMsg = msg,
-      jsonPointer = splunkFriendly(parser.getParsingContext.pathAsPointer().toString),
+      jsonPointer = splunkFriendly(toJsonPointer(parser.getParsingContext)),
       index = parserBase.map(_.getTokenCharacterOffset),
       line = parserBase.map(_.getTokenLineNr),
       col = parserBase.map(_.getTokenColumnNr),
@@ -19,6 +22,43 @@ object JsonParserException {
     )
   }
 
+  private def toJsonPointer(context: JsonStreamContext): String = {
+    // context.pathAsPointer().toString requires O(depth^2) memory because of
+    // JsonPointer._asString, and OOMEs on FromJson("[" * 100000).transform().
+    // For our own, we collect the nodes from leaf to root, reverse them,
+    // then build our string in order.
+    val leafToRoot =
+    Iterator
+      .iterate(Option(context))(_.flatMap(c => Option(c.getParent)))
+      .takeWhile(_.nonEmpty)
+      .flatten
+      .filter(_.hasPathSegment)
+      .to(ArrayBuffer)
+
+    leafToRoot
+      .reverseIterator
+      .foldLeft(new StringBuilder()) { (sb, ctx) =>
+        if (ctx.inArray()) {
+          sb.append('/').append(ctx.getCurrentIndex)
+        }
+        else if (ctx.inObject()) {
+          sb.append('/')
+          for {
+            name <- Option(ctx.getCurrentName)
+              c <- name
+          } {
+            c match {
+              case '/' => sb.append("~1")
+              case '~' => sb.append("~0")
+              case _ => sb.append(c)
+            }
+          }
+        }
+        sb
+      }
+      .result()
+  }
+
   private def splunkFriendly(value: String): String = {
     if (value.contains('"') || value.contains(" ")) {
       "\"" + value.replace("\"", "\\\"") + "\""

weepickle-tests/src/test/scala/com/rallyhealth/weepickle/v1/JsonPointerSpec.scala

@@ -0,0 +1,56 @@
+package com.rallyhealth.weepickle.v1
+
+import com.rallyhealth.weejson.v1.jackson.FromJson
+import com.rallyhealth.weepickle.v1.core.{FromInput, NoOpVisitor, TransformException}
+import org.scalactic.TypeCheckedTripleEquals
+import org.scalatest.freespec.AnyFreeSpec
+import org.scalatest.matchers.should.Matchers
+
+import java.io.{ByteArrayInputStream, StringReader}
+import java.nio.file.Files
+import scala.language.{existentials, implicitConversions}
+
+abstract class JsonPointerSpec(parse: Array[Byte] => FromInput, depthLimit: Int = 100)
+  extends AnyFreeSpec
+    with Matchers
+    with TypeCheckedTripleEquals {
+
+  private implicit def toBytes(s: String): Array[Byte] = s.getBytes
+
+  def assertPath(json: String, expectedPointer: String) = {
+    val t = intercept[TransformException](parse(json).transform(NoOpVisitor))
+    t.jsonPointer shouldBe expectedPointer
+  }
+
+  val scenarios = Seq(
+    ("", ""),
+    ("unquoted", ""),
+    (""" {"a": """, "/a"),
+    (""" {"a": 1, "b"  """, "/b"),
+    (""" {"foo/bar": """, "/foo~1bar"),
+    (""" {"foo~bar": """, "/foo~0bar"),
+    (""" [0,1,2 """, "/2"),
+    (""" [[[ """, "/0/0"),
+    (""" [[0,[0,1,[ """, "/0/1/2")
+  )
+  for ((json, expected) <- scenarios) {
+    json in assertPath(json, expected)
+  }
+}
+
+class FromJsonBytesJsonPointerSpec extends JsonPointerSpec(FromJson(_))
+
+class FromJsonStringJsonPointerSpec extends JsonPointerSpec(b => FromJson(new String(b)))
+
+class FromJsonInputStreamJsonPointerSpec extends JsonPointerSpec(b => FromJson(new ByteArrayInputStream(b)))
+
+class FromJsonReaderJsonPointerSpec extends JsonPointerSpec(b => FromJson(new StringReader(new String(b))))
+
+class FromJsonPathJsonPointerSpec extends JsonPointerSpec(
+  b => FromJson(Files.write(Files.createTempFile("JsonPointerSpec", ".json"), b))
+)
+
+class FromJsonFileJsonPointerSpec extends JsonPointerSpec(
+  b => FromJson(Files.write(Files.createTempFile("JsonPointerSpec", ".json"), b).toFile)
+)
+

weepickle-tests/src/test/scala/com/rallyhealth/weepickle/v1/ParserSpec.scala

@@ -3,14 +3,16 @@ package com.rallyhealth.weepickle.v1
 import com.rallyhealth.weejson.v1.CanonicalizeNumsVisitor._
 import com.rallyhealth.weejson.v1.jackson.{FromJson, ToJson, ToPrettyJson}
 import com.rallyhealth.weejson.v1.{BufferedValue, GenBufferedValue}
-import com.rallyhealth.weepickle.v1.core.FromInput
+import com.rallyhealth.weepickle.v1.core.{FromInput, NoOpVisitor}
 import org.scalactic.TypeCheckedTripleEquals
 import org.scalatest.freespec.AnyFreeSpec
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
 
-import java.io.{ByteArrayInputStream, StringReader}
+import java.io.{ByteArrayInputStream, File, StringReader}
 import java.nio.file.Files
+import scala.concurrent.duration._
 import scala.language.{existentials, implicitConversions}
+import scala.util.Try
 
 abstract class ParserSpec(parse: Array[Byte] => FromInput, depthLimit: Int = 100)
   extends AnyFreeSpec
@@ -31,6 +33,29 @@ abstract class ParserSpec(parse: Array[Byte] => FromInput, depthLimit: Int = 100
   "deep arr" in testDepth(Arr(_))
   "deep obj" in testDepth(b => Obj("k" -> b))
 
+  "net/JSONTestSuite" - {
+    for {
+      file <- new File("weepickle-tests/src/test/test_parsing").listFiles()
+        name = file.getName
+        if name.endsWith(".json")
+    } {
+      def parse() = FromJson(file).transform(NoOpVisitor)
+
+      name in {
+        val start = System.currentTimeMillis()
+
+        def duration = (System.currentTimeMillis() - start).nanos
+
+        name.head match {
+          case 'i' => Try(parse())
+          case 'y' => parse()
+          case 'n' => intercept[Exception](parse())
+        }
+        assert(duration < 5.seconds, s"parsing $name exceeded than the 5s time limit")
+      }
+    }
+  }
+
   private def testJson(tweak: BufferedValue => BufferedValue = identity) = {
     forAll { (b: BufferedValue) =>
       val value = tweak(b)

weepickle-tests/src/test/test_parsing/i_number_double_huge_neg_exp.json

@@ -0,0 +1 @@
+[123.456e-789]

weepickle-tests/src/test/test_parsing/i_number_huge_exp.json

@@ -0,0 +1 @@
+[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]

weepickle-tests/src/test/test_parsing/i_number_neg_int_huge_exp.json

@@ -0,0 +1 @@
+[-1e+9999]

weepickle-tests/src/test/test_parsing/i_number_pos_double_huge_exp.json

@@ -0,0 +1 @@
+[1.5e+9999]

weepickle-tests/src/test/test_parsing/i_number_real_neg_overflow.json

@@ -0,0 +1 @@
+[-123123e100000]

weepickle-tests/src/test/test_parsing/i_number_real_pos_overflow.json

@@ -0,0 +1 @@
+[123123e100000]

weepickle-tests/src/test/test_parsing/i_number_real_underflow.json

@@ -0,0 +1 @@
+[123e-10000000]

weepickle-tests/src/test/test_parsing/i_number_too_big_neg_int.json

@@ -0,0 +1 @@
+[-123123123123123123123123123123]

weepickle-tests/src/test/test_parsing/i_number_too_big_pos_int.json

@@ -0,0 +1 @@
+[100000000000000000000]

weepickle-tests/src/test/test_parsing/i_number_very_big_negative_int.json

@@ -0,0 +1 @@
+[-237462374673276894279832749832423479823246327846]

weepickle-tests/src/test/test_parsing/i_object_key_lone_2nd_surrogate.json

@@ -0,0 +1 @@
+{"\uDFAA":0}

weepickle-tests/src/test/test_parsing/i_string_1st_surrogate_but_2nd_missing.json

@@ -0,0 +1 @@
+["\uDADA"]

weepickle-tests/src/test/test_parsing/i_string_1st_valid_surrogate_2nd_invalid.json

@@ -0,0 +1 @@
+["\uD888\u1234"]

weepickle-tests/src/test/test_parsing/i_string_UTF-8_invalid_sequence.json

@@ -0,0 +1 @@
+["日ш�"]

weepickle-tests/src/test/test_parsing/i_string_UTF8_surrogate_U+D800.json

@@ -0,0 +1 @@
+["���"]

weepickle-tests/src/test/test_parsing/i_string_incomplete_surrogate_and_escape_valid.json

@@ -0,0 +1 @@
+["\uD800\n"]

weepickle-tests/src/test/test_parsing/i_string_incomplete_surrogate_pair.json

@@ -0,0 +1 @@
+["\uDd1ea"]

weepickle-tests/src/test/test_parsing/i_string_incomplete_surrogates_escape_valid.json

@@ -0,0 +1 @@
+["\uD800\uD800\n"]

weepickle-tests/src/test/test_parsing/i_string_invalid_lonely_surrogate.json

@@ -0,0 +1 @@
+["\ud800"]

weepickle-tests/src/test/test_parsing/i_string_invalid_surrogate.json

@@ -0,0 +1 @@
+["\ud800abc"]

weepickle-tests/src/test/test_parsing/i_string_invalid_utf-8.json

@@ -0,0 +1 @@
+["�"]

weepickle-tests/src/test/test_parsing/i_string_inverted_surrogates_U+1D11E.json

@@ -0,0 +1 @@
+["\uDd1e\uD834"]

weepickle-tests/src/test/test_parsing/i_string_iso_latin_1.json

@@ -0,0 +1 @@
+["�"]

weepickle-tests/src/test/test_parsing/i_string_lone_second_surrogate.json

@@ -0,0 +1 @@
+["\uDFAA"]

weepickle-tests/src/test/test_parsing/i_string_lone_utf8_continuation_byte.json

@@ -0,0 +1 @@
+["�"]

weepickle-tests/src/test/test_parsing/i_string_not_in_unicode_range.json

@@ -0,0 +1 @@
+["����"]

weepickle-tests/src/test/test_parsing/i_string_overlong_sequence_2_bytes.json

@@ -0,0 +1 @@
+["��"]

weepickle-tests/src/test/test_parsing/i_string_overlong_sequence_6_bytes.json

@@ -0,0 +1 @@
+["������"]

weepickle-tests/src/test/test_parsing/i_string_overlong_sequence_6_bytes_null.json

@@ -0,0 +1 @@
+["������"]

weepickle-tests/src/test/test_parsing/i_string_truncated-utf-8.json

@@ -0,0 +1 @@
+["��"]

weepickle-tests/src/test/test_parsing/i_structure_500_nested_arrays.json

@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]

weepickle-tests/src/test/test_parsing/i_structure_UTF-8_BOM_empty_object.json

@@ -0,0 +1 @@
+{}

weepickle-tests/src/test/test_parsing/n_array_1_true_without_comma.json

@@ -0,0 +1 @@
+[1 true]

weepickle-tests/src/test/test_parsing/n_array_a_invalid_utf8.json

@@ -0,0 +1 @@
+[a�]

weepickle-tests/src/test/test_parsing/n_array_colon_instead_of_comma.json

@@ -0,0 +1 @@
+["": 1]

weepickle-tests/src/test/test_parsing/n_array_comma_after_close.json

@@ -0,0 +1 @@
+[""],

weepickle-tests/src/test/test_parsing/n_array_comma_and_number.json

@@ -0,0 +1 @@
+[,1]

weepickle-tests/src/test/test_parsing/n_array_double_comma.json

@@ -0,0 +1 @@
+[1,,2]

weepickle-tests/src/test/test_parsing/n_array_double_extra_comma.json

@@ -0,0 +1 @@
+["x",,]

weepickle-tests/src/test/test_parsing/n_array_extra_close.json

@@ -0,0 +1 @@
+["x"]]

weepickle-tests/src/test/test_parsing/n_array_extra_comma.json

@@ -0,0 +1 @@
+["",]

weepickle-tests/src/test/test_parsing/n_array_incomplete.json

@@ -0,0 +1 @@
+["x"

weepickle-tests/src/test/test_parsing/n_array_incomplete_invalid_value.json

@@ -0,0 +1 @@
+[x

weepickle-tests/src/test/test_parsing/n_array_inner_array_no_comma.json

@@ -0,0 +1 @@
+[3[4]]

weepickle-tests/src/test/test_parsing/n_array_invalid_utf8.json

@@ -0,0 +1 @@
+[�]

weepickle-tests/src/test/test_parsing/n_array_items_separated_by_semicolon.json

@@ -0,0 +1 @@
+[1:2]

weepickle-tests/src/test/test_parsing/n_array_just_comma.json

@@ -0,0 +1 @@
+[,]

weepickle-tests/src/test/test_parsing/n_array_just_minus.json

@@ -0,0 +1 @@
+[-]

weepickle-tests/src/test/test_parsing/n_array_missing_value.json

@@ -0,0 +1 @@
+[   , ""]

weepickle-tests/src/test/test_parsing/n_array_newlines_unclosed.json

@@ -0,0 +1,3 @@
+["a",
+4
+,1,

weepickle-tests/src/test/test_parsing/n_array_number_and_comma.json

@@ -0,0 +1 @@
+[1,]

weepickle-tests/src/test/test_parsing/n_array_number_and_several_commas.json

@@ -0,0 +1 @@
+[1,,]

weepickle-tests/src/test/test_parsing/n_array_spaces_vertical_tab_formfeed.json

@@ -0,0 +1 @@
+["a"\f]

weepickle-tests/src/test/test_parsing/n_array_star_inside.json

@@ -0,0 +1 @@
+[*]

weepickle-tests/src/test/test_parsing/n_array_unclosed.json

@@ -0,0 +1 @@
+[""

weepickle-tests/src/test/test_parsing/n_array_unclosed_trailing_comma.json

@@ -0,0 +1 @@
+[1,

weepickle-tests/src/test/test_parsing/n_array_unclosed_with_new_lines.json

@@ -0,0 +1,3 @@
+[1,
+1
+,1

weepickle-tests/src/test/test_parsing/n_array_unclosed_with_object_inside.json

@@ -0,0 +1 @@
+[{}

weepickle-tests/src/test/test_parsing/n_incomplete_false.json

@@ -0,0 +1 @@
+[fals]

weepickle-tests/src/test/test_parsing/n_incomplete_null.json

@@ -0,0 +1 @@
+[nul]

weepickle-tests/src/test/test_parsing/n_incomplete_true.json

@@ -0,0 +1 @@
+[tru]

weepickle-tests/src/test/test_parsing/n_number_++.json

@@ -0,0 +1 @@
+[++1234]

weepickle-tests/src/test/test_parsing/n_number_+1.json

@@ -0,0 +1 @@
+[+1]

weepickle-tests/src/test/test_parsing/n_number_+Inf.json

@@ -0,0 +1 @@
+[+Inf]

weepickle-tests/src/test/test_parsing/n_number_-01.json

@@ -0,0 +1 @@
+[-01]

weepickle-tests/src/test/test_parsing/n_number_-1.0..json

@@ -0,0 +1 @@
+[-1.0.]

weepickle-tests/src/test/test_parsing/n_number_-2..json

@@ -0,0 +1 @@
+[-2.]

weepickle-tests/src/test/test_parsing/n_number_-NaN.json

@@ -0,0 +1 @@
+[-NaN]

weepickle-tests/src/test/test_parsing/n_number_.-1.json

@@ -0,0 +1 @@
+[.-1]

weepickle-tests/src/test/test_parsing/n_number_.2e-3.json

@@ -0,0 +1 @@
+[.2e-3]

weepickle-tests/src/test/test_parsing/n_number_0.1.2.json

@@ -0,0 +1 @@
+[0.1.2]

weepickle-tests/src/test/test_parsing/n_number_0.3e+.json

@@ -0,0 +1 @@
+[0.3e+]