diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala new file mode 100644 index 00000000..885c9a9e --- /dev/null +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala @@ -0,0 +1,130 @@ +/* +Copyright 2012 Twitter, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.chill.backwardcomp + +import com.esotericsoftware.kryo.util.Util.{className, log} +import com.esotericsoftware.kryo.util.{DefaultClassResolver, IdentityObjectIntMap, IntMap} +import com.esotericsoftware.kryo.{Kryo, KryoException, Registration} +import com.esotericsoftware.minlog.Log.{trace, DEBUG, TRACE} +import com.twitter.chill.{Input, Output} + +class BackwardCompatibleClassResolver extends DefaultClassResolver { + import BackwardCompatibleClassResolver._ + + private val fallbackByClass: Map[Class[_], Int] = + Map( + classOf[FallbackToHashSet1] -> 23, + classOf[FallbackToHashSet2] -> 23, + classOf[FallbackToHashMap1] -> 28, + classOf[FallbackToHashMap2] -> 28, + classOf[FallbackToMap11] -> 24, + classOf[FallbackToMap12] -> 24, + classOf[Range.Exclusive] -> 118 + ) + + override def getRegistration(`type`: Class[_]): Registration = + if (`type` == classOf[Range.Exclusive]) { + getRegistration(118) + } else { + super.getRegistration(`type`) + } + + override def getRegistration(classID: Int): Registration = { + val registration = super.getRegistration(classID) + if (registration != null) { + fallbackByClass.get(registration.getType) match { + case Some(value) => super.getRegistration(value) + case None => + if (registration.getType == classOf[Range]) { + new Registration( + classOf[Range.Exclusive], + registration.getSerializer, + registration.getId + ) + } else { + registration + } + } + } else { + null + } + } + + override def readClass(input: Input): Registration = { + val classID = input.readVarInt(true) + classID match { + case Kryo.NULL => + if (TRACE || (DEBUG && kryo.getDepth == 1)) log("Read", null) + null + case 1 => // Offset for NAME and NULL. + readName(input) + case _ => + val registration = getRegistration(classID - 2) + if (registration == null) + throw new KryoException("Encountered unregistered class ID: " + (classID - 2)) + if (TRACE) trace("kryo", "Read class " + (classID - 2) + ": " + className(registration.getType)) + + registration + } + } + + def readIgnoredClass(input: Input): Unit = { + val classId = input.readVarInt(true) + if (classId == 1) { + val nameId = input.readVarInt(true) + if (nameIdToClass == null) { + nameIdToClass = new IntMap[Class[_]] + } + if (nameIdToClass.get(nameId) == null) { + val name = input.readString() + nameIdToClass.put(nameId, IgnoredClassPlaceholder.getClass) + } + } + } + + def writeFakeName[T](output: Output, name: String, placeholderType: Class[_]): Unit = { + output.writeVarInt(1, true) + if (classToNameId != null) { + val nameId = classToNameId.get(placeholderType, -1) + if (nameId != -1) { + output.writeVarInt(nameId, true) + } else { + writeClassNameOnFirstEncounter(output, name, placeholderType) + } + } else { + writeClassNameOnFirstEncounter(output, name, placeholderType) + } + } + + private def writeClassNameOnFirstEncounter[T](output: Output, name: String, placeholderType: Class[_]) = { + // Only write the class name the first time encountered in object graph. + val nameId = nextNameId + nextNameId += 1 + if (classToNameId == null) classToNameId = new IdentityObjectIntMap[Class[_]] + classToNameId.put(placeholderType, nameId) + output.writeVarInt(nameId, true) + output.writeString(name) + } +} + +object BackwardCompatibleClassResolver { + abstract class FallbackToHashSet1 + abstract class FallbackToHashSet2 + abstract class FallbackToHashMap1 + abstract class FallbackToHashMap2 + abstract class FallbackToMap11 + abstract class FallbackToMap12 + + object IgnoredClassPlaceholder +} diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala new file mode 100644 index 00000000..eca60338 --- /dev/null +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala @@ -0,0 +1,85 @@ +/* +Copyright 2012 Twitter, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.chill.backwardcomp + +import com.esotericsoftware.kryo.serializers.FieldSerializer +import com.twitter.chill.{Input, Kryo, Output} + +class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Class[_]) + extends FieldSerializer[T](kryo, typ) { + + override def read(kryo: Kryo, input: Input, typ: Class[T]): T = { + val result = create(kryo, input, typ) + kryo.reference(result) + + val bitmap0 = input.readByte() + val end = kryo.readClassAndObject(input) + val hashCode = input.readVarInt(false) + val isInclusive = input.readBoolean() + val last = kryo.readClassAndObject(input) + val num1 = kryo.readClassAndObject(input) + val num2 = kryo.readClassAndObject(input) + val numRangeElements = input.readVarInt(false) + val start = kryo.readClassAndObject(input) + val step = kryo.readClassAndObject(input) + + var idx = 0 + val fields = getFields + while (idx < fields.length) { + val field = fields(idx) + idx = idx + 1 + + field.getField.getName match { + case "bitmap$0" => field.getField.set(result, bitmap0) + case "end" => field.getField.set(result, end) + case "hashCode" => field.getField.set(result, hashCode) + case "isInclusive" => field.getField.set(result, isInclusive) + case "length" => field.getField.set(result, numRangeElements) + case "num" => field.getField.set(result, num1) + case "scala$collection$immutable$NumericRange$$num" => field.getField.set(result, num2) + case "start" => field.getField.set(result, start) + case "step" => field.getField.set(result, step) + } + } + + result + } + + override def write(kryo: Kryo, output: Output, `object`: T): Unit = { + var length: Option[Int] = None + var idx = 0 + val fields = getFields + while (idx < fields.length) { + val field = fields(idx) + idx = idx + 1 + + val value = field.getField.get(`object`) + field.getField.getName match { + case "bitmap$0" => output.writeByte(value.asInstanceOf[Byte]) + case "end" => kryo.writeClassAndObject(output, value) + case "hashCode" => output.writeVarInt(value.asInstanceOf[Int], false) + case "isInclusive" => output.writeBoolean(value.asInstanceOf[Boolean]) + case "length" => + kryo.writeClassAndObject(output, null) // last + length = Some(value.asInstanceOf[Int]) // storing length to be emitted after the implicits + case "num" => kryo.writeClassAndObject(output, value) + case "scala$collection$immutable$NumericRange$$num" => + kryo.writeClassAndObject(output, value) + output.writeVarInt(length.get, false) + case "start" => kryo.writeClassAndObject(output, value) + case "step" => kryo.writeClassAndObject(output, value) + } + } + } +} diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala new file mode 100644 index 00000000..4a9f8d20 --- /dev/null +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala @@ -0,0 +1,42 @@ +/* +Copyright 2012 Twitter, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.chill.backwardcomp + +import com.esotericsoftware.kryo.serializers.FieldSerializer +import com.twitter.chill.{Input, Kryo, ObjectSerializer, Output} + +class BackwardCompatibleJavaWrappersSerializer[T](kryo: Kryo, typ: Class[_]) + extends FieldSerializer[T](kryo, typ) { + import BackwardCompatibleJavaWrappersSerializer._ + + override def read(kryo: Kryo, input: Input, typ: Class[T]): T = { + // Skipping the first serialized field which is assumed to be an $outer object reference + kryo.getClassResolver.asInstanceOf[BackwardCompatibleClassResolver].readIgnoredClass(input) + val refId = input.readVarInt(true) + // Assuming ObjectSerializer which reads no data, so it does not mean if this is the first occurrence or not + super.read(kryo, input, typ) + } + + override def write(kryo: Kryo, output: Output, `object`: T): Unit = { + kryo.getClassResolver + .asInstanceOf[BackwardCompatibleClassResolver] + .writeFakeName(output, "scala.collection.convert.Wrappers$", OuterWrapper.getClass) + kryo.writeObjectOrNull(output, OuterWrapper, new ObjectSerializer) + super.write(kryo, output, `object`) + } +} + +object BackwardCompatibleJavaWrappersSerializer { + object OuterWrapper +} diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala new file mode 100644 index 00000000..82d50e9f --- /dev/null +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala @@ -0,0 +1,254 @@ +/* +Copyright 2012 Twitter, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.chill.backwardcomp + +import com.esotericsoftware.kryo.util.MapReferenceResolver +import com.twitter.chill.java.{Java8ClosureRegistrar, PackageRegistrar} +import com.twitter.chill._ + +import scala.collection.JavaConverters._ +import scala.collection.immutable.{ + BitSet, + HashMap, + HashSet, + ListMap, + ListSet, + NumericRange, + Queue, + Range, + SortedMap, + SortedSet, + TreeMap, + TreeSet, + WrappedString +} +import scala.collection.mutable +import scala.collection.mutable.{ + Buffer, + ListBuffer, + WrappedArray, + BitSet => MBitSet, + HashMap => MHashMap, + HashSet => MHashSet, + Map => MMap, + Queue => MQueue, + Set => MSet +} +import scala.reflect.ClassTag +import scala.util.matching.Regex + +/** Makes an empty instantiator then registers everything */ +class BackwardCompatibleScalaKryoInstantiator extends EmptyScalaKryoInstantiator { + override def newKryo: KryoBase = { + val k = new KryoBase(new BackwardCompatibleClassResolver, new MapReferenceResolver) + k.setRegistrationRequired(false) + k.setInstantiatorStrategy(new org.objenesis.strategy.StdInstantiatorStrategy) + + // Handle cases where we may have an odd classloader setup like with libjars + // for hadoop + val classLoader = Thread.currentThread.getContextClassLoader + k.setClassLoader(classLoader) + + val reg = new BackwardCompatibleAllScalaRegistrar + reg(k) + k + } +} + +/** + * Note that additional scala collections registrations are provided by [[AllScalaRegistrar]]. They have not been + * included in this registrar for backwards compatibility reasons. + */ +class BackwardCompatibleScalaCollectionsRegistrar extends IKryoRegistrar { + def apply(newK: Kryo): Unit = { + // for binary compat this is here, but could be moved to RichKryo + def useField[T](cls: Class[T]): Unit = { + val fs = new BackwardCompatibleJavaWrappersSerializer(newK, cls) + fs.setIgnoreSyntheticFields(false) // scala generates a lot of these attributes + newK.register(cls, fs) + } + // The wrappers are private classes: + useField(List(1, 2, 3).asJava.getClass) + useField(List(1, 2, 3).iterator.asJava.getClass) + useField(Map(1 -> 2, 4 -> 3).asJava.getClass) + useField(new _root_.java.util.ArrayList().asScala.getClass) + useField(new _root_.java.util.HashMap().asScala.getClass) + + /* + * Note that subclass-based use: addDefaultSerializers, else: register + * You should go from MOST specific, to least to specific when using + * default serializers. The FIRST one found is the one used + */ + newK + // wrapper array is abstract + .forSubclass[WrappedArray[Any]](new WrappedArraySerializer[Any]) + .forSubclass[BitSet](new BitSetSerializer) + .forSubclass[SortedSet[Any]](new SortedSetSerializer) + .forClass[Some[Any]](new SomeSerializer[Any]) + .forClass[Left[Any, Any]](new LeftSerializer[Any, Any]) + .forClass[Right[Any, Any]](new RightSerializer[Any, Any]) + .forTraversableSubclass(Queue.empty[Any]) + // List is a sealed class, so there are only two subclasses: + .forTraversableSubclass(List.empty[Any]) + // Add ListBuffer subclass before Buffer to prevent the more general case taking precedence + .forTraversableSubclass(ListBuffer.empty[Any], isImmutable = false) + // add mutable Buffer before Vector, otherwise Vector is used + .forTraversableSubclass(Buffer.empty[Any], isImmutable = false) + // Vector is a final class + .forTraversableClass(Vector.empty[Any]) + .forTraversableSubclass(ListSet.empty[Any]) + // specifically register small sets since Scala represents them differently + .forConcreteTraversableClass(Set[Any]('a)) + .forConcreteTraversableClass(Set[Any]('a, 'b)) + .forConcreteTraversableClass(Set[Any]('a, 'b, 'c)) + .forConcreteTraversableClass(Set[Any]('a, 'b, 'c, 'd)) + // default set implementation + .forConcreteTraversableClass(HashSet[Any]('a, 'b, 'c, 'd, 'e)) + // specifically register small maps since Scala represents them differently + .forConcreteTraversableClass(Map[Any, Any]('a -> 'a)) + .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b)) + .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c)) + .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c, 'd -> 'd)) + // default map implementation + .forConcreteTraversableClass(HashMap[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c, 'd -> 'd, 'e -> 'e)) + // The normal fields serializer works for ranges + .registerClasses( + Seq(classOf[Range.Inclusive], classOf[NumericRange.Inclusive[_]]) + ) + + newK.register( + classOf[NumericRange.Exclusive[_]], + new BackwardCompatibleExclusiveNumericRangeSerializer(newK, classOf[NumericRange.Exclusive[_]]) + ) + // Add some maps + + newK + .forSubclass[SortedMap[Any, Any]](new SortedMapSerializer) + .forTraversableSubclass(ListMap.empty[Any, Any]) + .forTraversableSubclass(HashMap.empty[Any, Any]) + // The above ListMap/HashMap must appear before this: + .forTraversableSubclass(Map.empty[Any, Any]) + // here are the mutable ones: + .forTraversableClass(MBitSet.empty, isImmutable = false) + .forTraversableClass(MHashMap.empty[Any, Any], isImmutable = false) + .forTraversableClass(MHashSet.empty[Any], isImmutable = false) + .forTraversableSubclass(MQueue.empty[Any], isImmutable = false) + .forTraversableSubclass(MMap.empty[Any, Any], isImmutable = false) + .forTraversableSubclass(MSet.empty[Any], isImmutable = false) + } +} + +/** Registrar for everything that was registered in chill 0.9.2 - included for backwards compatibility. */ +class BackwardCompatibleAllScalaRegistrar_0_9_2 extends IKryoRegistrar { + def apply(k: Kryo): Unit = { + new BackwardCompatibleScalaCollectionsRegistrar()(k) + new JavaWrapperCollectionRegistrar()(k) + + // Register all 22 tuple serializers and specialized serializers + ScalaTupleSerialization.register(k) + k.forClass[Symbol](new KSerializer[Symbol] { + override def isImmutable = true + def write(k: Kryo, out: Output, obj: Symbol): Unit = out.writeString(obj.name) + def read(k: Kryo, in: Input, cls: Class[Symbol]): Symbol = Symbol(in.readString) + }).forSubclass[Regex](new RegexSerializer) + .forClass[ClassTag[Any]](new ClassTagSerializer[Any]) + .forSubclass[Manifest[Any]](new ManifestSerializer[Any]) + .forSubclass[scala.Enumeration#Value](new EnumerationSerializer) + + // use the singleton serializer for boxed Unit + val boxedUnit = scala.runtime.BoxedUnit.UNIT + k.register(boxedUnit.getClass, new SingletonSerializer(boxedUnit)) + PackageRegistrar.all()(k) + new Java8ClosureRegistrar()(k) + } +} + +/** + * Registers all the scala (and java) serializers we have. The registrations are designed to cover most of + * scala.collecion.immutable, so they can be used in long term persistence scenarios that run with + * setRegistrationRequired(true). + * + * When adding new serializers, add them to the end of the list, so compatibility is not broken needlessly + * for projects using chill for long term persistence - see com.twitter.chill.RegistrationIdsSpec. + */ +class BackwardCompatibleAllScalaRegistrar extends IKryoRegistrar { + def apply(k: Kryo): Unit = { + new BackwardCompatibleAllScalaRegistrar_0_9_2()(k) + + k.registerClasses( + Seq( + classOf[Array[Byte]], + classOf[Array[Short]], + classOf[Array[Int]], + classOf[Array[Long]], + classOf[Array[Float]], + classOf[Array[Double]], + classOf[Array[Boolean]], + classOf[Array[Char]], + classOf[Array[String]], + classOf[Array[Any]], + classOf[Class[_]], // needed for the WrappedArraySerializer + classOf[Any], // needed for scala.collection.mutable.WrappedArray$ofRef + mutable.WrappedArray.make(Array[Byte]()).getClass, + mutable.WrappedArray.make(Array[Short]()).getClass, + mutable.WrappedArray.make(Array[Int]()).getClass, + mutable.WrappedArray.make(Array[Long]()).getClass, + mutable.WrappedArray.make(Array[Float]()).getClass, + mutable.WrappedArray.make(Array[Double]()).getClass, + mutable.WrappedArray.make(Array[Boolean]()).getClass, + mutable.WrappedArray.make(Array[Char]()).getClass, + mutable.WrappedArray.make(Array[String]()).getClass, + None.getClass, + classOf[Queue[_]], + Nil.getClass, + classOf[::[_]], + classOf[Range], + classOf[WrappedString], + classOf[TreeSet[_]], + classOf[TreeMap[_, _]], + // The most common orderings for TreeSet and TreeMap + Ordering.Byte.getClass, + Ordering.Short.getClass, + Ordering.Int.getClass, + Ordering.Long.getClass, + Ordering.Float.getClass, + Ordering.Double.getClass, + Ordering.Boolean.getClass, + Ordering.Char.getClass, + Ordering.String.getClass + ) + ).forConcreteTraversableClass(Set[Any]()) + .forConcreteTraversableClass(ListSet[Any]()) + .forConcreteTraversableClass(ListSet[Any]('a)) + + k.register(classOf[BackwardCompatibleClassResolver.FallbackToHashSet1]) + k.register(classOf[BackwardCompatibleClassResolver.FallbackToHashSet2]) + + k.forConcreteTraversableClass(Map[Any, Any]()) + + k.register(classOf[BackwardCompatibleClassResolver.FallbackToHashMap1]) + k.register(classOf[BackwardCompatibleClassResolver.FallbackToHashMap2]) + + k.forConcreteTraversableClass(ListMap[Any, Any]()) + .forConcreteTraversableClass(ListMap('a -> 'a)) + k.register(classOf[Stream.Cons[_]], new StreamSerializer[Any]) + k.register(Stream.empty[Any].getClass) + k.forClass[scala.runtime.VolatileByteRef](new VolatileByteRefSerializer) + k.forClass[BigDecimal](new BigDecimalSerializer) + k.register(Queue.empty[Any].getClass) + k.register(classOf[BackwardCompatibleClassResolver.FallbackToMap11]) + k.register(classOf[BackwardCompatibleClassResolver.FallbackToMap12]) + k.forConcreteTraversableClass(Map(1 -> 2).keySet) + } +}