Pre-registering certain standard classes to better support long term persistence scenarios

.travis.yml

@@ -2,16 +2,16 @@ language: scala
 sudo: false
 matrix:
   include:
-    - scala: 2.10.6
+    - scala: 2.10.7
       script: ./sbt ++$TRAVIS_SCALA_VERSION clean test mimaReportBinaryIssues
 
-    - scala: 2.11.8
+    - scala: 2.11.12
       jdk: oraclejdk8
       script: ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport mimaReportBinaryIssues
       after_success:
         - bash <(curl -s https://codecov.io/bash)
 
-    - scala: 2.12.1
+    - scala: 2.12.4
       jdk: oraclejdk8
       script: ./sbt "+++$TRAVIS_SCALA_VERSION clean" "+++$TRAVIS_SCALA_VERSION test" # "+++$TRAVIS_SCALA_VERSION mimaReportBinaryIssues"
 

build.sbt

@@ -9,8 +9,8 @@ val scroogeVersion = "4.12.0"
 
 val sharedSettings = mimaDefaultSettings ++ scalariformSettings ++ Seq(
   organization := "com.twitter",
-  scalaVersion := "2.11.8",
-  crossScalaVersions := Seq("2.10.6", "2.11.8", "2.12.1"),
+  scalaVersion := "2.11.12",
+  crossScalaVersions := Seq("2.10.7", "2.11.12", "2.12.4"),
   scalacOptions ++= Seq("-unchecked", "-deprecation"),
   ScalariformKeys.preferences := formattingPreferences,
 

chill-scala/src/main/scala/com/twitter/chill/ClassManifestSerializer.scala

@@ -19,7 +19,7 @@ package com.twitter.chill
 class ClassManifestSerializer[T] extends KSerializer[ClassManifest[T]] {
 
   def write(kser: Kryo, out: Output, obj: ClassManifest[T]) {
-    kser.writeObject(out, obj.erasure)
+    kser.writeObject(out, obj.runtimeClass)
   }
 
   def read(kser: Kryo, in: Input, cls: Class[ClassManifest[T]]): ClassManifest[T] = {

chill-scala/src/main/scala/com/twitter/chill/ScalaKryoInstantiator.scala

@@ -16,37 +16,15 @@ limitations under the License.
 
 package com.twitter.chill
 
-import scala.collection.immutable.{
-  BitSet,
-  HashSet,
-  ListSet,
-  NumericRange,
-  Range,
-  SortedSet,
-  SortedMap,
-  ListMap,
-  HashMap,
-  Queue
-}
-
-import scala.collection.mutable.{
-  WrappedArray,
-  BitSet => MBitSet,
-  Map => MMap,
-  HashMap => MHashMap,
-  Set => MSet,
-  HashSet => MHashSet,
-  ListBuffer,
-  Queue => MQueue,
-  Buffer
-}
-
+import scala.collection.immutable.{ BitSet, HashMap, HashSet, ListMap, ListSet, NumericRange, Queue, Range, SortedMap, SortedSet, TreeMap, TreeSet, WrappedString }
+import scala.collection.mutable.{ Buffer, ListBuffer, WrappedArray, BitSet => MBitSet, HashMap => MHashMap, HashSet => MHashSet, Map => MMap, Queue => MQueue, Set => MSet }
 import scala.util.matching.Regex
 
 import com.twitter.chill.java.{ Java8ClosureRegistrar, PackageRegistrar }
 import _root_.java.io.Serializable
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 import scala.reflect.ClassTag
 
 /**
@@ -99,6 +77,10 @@ class ScalaKryoInstantiator extends EmptyScalaKryoInstantiator {
   }
 }
 
+/**
+ * Note that additional scala collections registrations are provided by [[AllScalaRegistrar]]. They have not been
+ * included in this registrar for backwards compatibility reasons.
+ */
 class ScalaCollectionsRegistrar extends IKryoRegistrar {
   def apply(newK: Kryo) {
     // for binary compat this is here, but could be moved to RichKryo
@@ -177,14 +159,11 @@ class JavaWrapperCollectionRegistrar extends IKryoRegistrar {
   }
 }
 
-/** Registers all the scala (and java) serializers we have */
-class AllScalaRegistrar extends IKryoRegistrar {
+/** Registrar for everything that was registered in chill 0.9.2 - included for backwards compatibility. */
+class AllScalaRegistrar_0_9_2 extends IKryoRegistrar {
   def apply(k: Kryo) {
-    val col = new ScalaCollectionsRegistrar
-    col(k)
-
-    val jcol = new JavaWrapperCollectionRegistrar
-    jcol(k)
+    new ScalaCollectionsRegistrar()(k)
+    new JavaWrapperCollectionRegistrar()(k)
 
     // Register all 22 tuple serializers and specialized serializers
     ScalaTupleSerialization.register(k)
@@ -205,3 +184,67 @@ class AllScalaRegistrar extends IKryoRegistrar {
     new Java8ClosureRegistrar()(k)
   }
 }
+
+/**
+ * Registers all the scala (and java) serializers we have. The registrations are designed to cover most of
+ * scala.collecion.immutable, so they can be used in long term persistence scenarios that run with
+ * setRegistrationRequired(true).
+ */
+class AllScalaRegistrar extends IKryoRegistrar {
+  def apply(k: Kryo) {
+    new AllScalaRegistrar_0_9_2()(k)
+
+    k.registerClasses(Seq(
+      classOf[Array[Byte]],
+      classOf[Array[Short]],
+      classOf[Array[Int]],
+      classOf[Array[Long]],
+      classOf[Array[Float]],
+      classOf[Array[Double]],
+      classOf[Array[Boolean]],
+      classOf[Array[Char]],
+      classOf[Array[String]],
+      classOf[Array[Any]],
+      classOf[Class[_]], // needed for the WrappedArraySerializer
+      classOf[Any], // needed for scala.collection.mutable.WrappedArray$ofRef
+      mutable.WrappedArray.make(Array[Byte]()).getClass,
+      mutable.WrappedArray.make(Array[Short]()).getClass,
+      mutable.WrappedArray.make(Array[Int]()).getClass,
+      mutable.WrappedArray.make(Array[Long]()).getClass,
+      mutable.WrappedArray.make(Array[Float]()).getClass,
+      mutable.WrappedArray.make(Array[Double]()).getClass,
+      mutable.WrappedArray.make(Array[Boolean]()).getClass,
+      mutable.WrappedArray.make(Array[Char]()).getClass,
+      mutable.WrappedArray.make(Array[String]()).getClass,
+      None.getClass,
+      classOf[Queue[_]],
+      Nil.getClass,
+      classOf[::[_]],
+      classOf[Range],
+      classOf[WrappedString],
+      classOf[TreeSet[_]],
+      classOf[TreeMap[_, _]],
+      // The most common orderings for TreeSet and TreeMap
+      Ordering.Byte.getClass,
+      Ordering.Short.getClass,
+      Ordering.Int.getClass,
+      Ordering.Long.getClass,
+      Ordering.Float.getClass,
+      Ordering.Double.getClass,
+      Ordering.Boolean.getClass,
+      Ordering.Char.getClass,
+      Ordering.String.getClass))
+      .forConcreteTraversableClass(Set[Any]())
+      .forConcreteTraversableClass(ListSet[Any]())
+      .forConcreteTraversableClass(ListSet[Any]('a))
+      .forConcreteTraversableClass(HashSet[Any]())
+      .forConcreteTraversableClass(HashSet[Any]('a))
+      .forConcreteTraversableClass(Map[Any, Any]())
+      .forConcreteTraversableClass(HashMap())
+      .forConcreteTraversableClass(HashMap('a -> 'a))
+      .forConcreteTraversableClass(ListMap())
+      .forConcreteTraversableClass(ListMap('a -> 'a))
+    k.register(classOf[Stream.Cons[_]], new StreamSerializer[Any])
+    k.register(Stream.empty[Any].getClass)
+  }
+}

chill-scala/src/main/scala/com/twitter/chill/StreamSerializer.scala

@@ -0,0 +1,11 @@
+package com.twitter.chill
+
+class StreamSerializer[T]() extends KSerializer[Stream[T]] {
+  def write(kser: Kryo, out: Output, stream: Stream[T]) {
+    kser.writeClassAndObject(out, stream.toList)
+  }
+
+  def read(kser: Kryo, in: Input, cls: Class[Stream[T]]): Stream[T] = {
+    kser.readClassAndObject(in).asInstanceOf[List[T]].toStream
+  }
+}

chill-scala/src/test/scala/com/twitter/chill/KryoSpec.scala

@@ -22,7 +22,6 @@ import org.scalatest.matchers.{ Matcher, MatchResult }
 import scala.collection.immutable.{ SortedSet, BitSet, ListSet, HashSet, SortedMap, ListMap, HashMap }
 import scala.collection.mutable.{ ArrayBuffer => MArrayBuffer, BitSet => MBitSet, HashMap => MHashMap }
 import _root_.java.util.PriorityQueue
-import _root_.java.util.Locale
 import scala.collection.mutable
 import scala.collection.JavaConverters._
 import scala.reflect._
@@ -55,7 +54,7 @@ case class Foo(m1: Map[String, Int], m2: Map[String, Seq[String]])
 class KryoSpec extends WordSpec with Matchers with BaseProperties {
 
   def roundtrip[T] = new Matcher[T] {
-    def apply(t: T) = MatchResult(rtEquiv(t), "successfull serialization roundtrip for " + t, "failed serialization roundtrip for " + t)
+    def apply(t: T) = MatchResult(rtEquiv(t), "successful serialization roundtrip for " + t, "failed serialization roundtrip for " + t)
   }
 
   def getKryo = KryoSerializer.registered.newKryo