From e54e1f592e4a2eb59d335b171e5ca310ee51bc50 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Mon, 31 Aug 2020 10:44:45 +0200 Subject: [PATCH 1/4] Backward compatible variant for migration from Scala 2.12 to 2.13 --- .../BackwardCompatibleClassResolver.scala | 124 ++++++++++ ...tibleExclusiveNumericRangeSerializer.scala | 74 ++++++ ...wardCompatibleJavaWrappersSerializer.scala | 39 +++ ...kwardCompatibleScalaKryoInstantiator.scala | 228 ++++++++++++++++++ 4 files changed, 465 insertions(+) create mode 100644 chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala create mode 100644 chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala create mode 100644 chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala create mode 100644 chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala new file mode 100644 index 00000000..ca19f9e0 --- /dev/null +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala @@ -0,0 +1,124 @@ +/* +Copyright 2012 Twitter, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.chill.backwardcomp + +import com.esotericsoftware.kryo.util.Util.{className, log} +import com.esotericsoftware.kryo.util.{DefaultClassResolver, IdentityObjectIntMap, IntMap} +import com.esotericsoftware.kryo.{Kryo, KryoException, Registration} +import com.esotericsoftware.minlog.Log.{DEBUG, TRACE, trace} +import com.twitter.chill.{Input, Output} + +class BackwardCompatibleClassResolver extends DefaultClassResolver { + import BackwardCompatibleClassResolver._ + + private val fallbackByClass: Map[Class[_], Int] = + Map( + classOf[FallbackToHashSet1] -> 23, + classOf[FallbackToHashSet2] -> 23, + classOf[FallbackToHashMap1] -> 28, + classOf[FallbackToHashMap2] -> 28, + classOf[FallbackToMap11] -> 24, + classOf[FallbackToMap12] -> 24, + classOf[Range.Exclusive] -> 118 + ) + + override def getRegistration(`type`: Class[_]): Registration = { + if (`type` == classOf[Range.Exclusive]) { + getRegistration(118) + } else { + super.getRegistration(`type`) + } + } + + override def getRegistration(classID: Int): Registration = { + val registration = super.getRegistration(classID) + if (registration != null) { + fallbackByClass.get(registration.getType) match { + case Some(value) => super.getRegistration(value) + case None => + if (registration.getType == classOf[Range]) { + new Registration( + classOf[Range.Exclusive], + registration.getSerializer, + registration.getId + ) + } else { + registration + } + } + } else { + null + } + } + + override def readClass(input: Input): Registration = { + val classID = input.readVarInt(true) + classID match { + case Kryo.NULL => + if (TRACE || (DEBUG && kryo.getDepth == 1)) log("Read", null) + null + case 1 => // Offset for NAME and NULL. + readName(input) + case _ => + val registration = getRegistration(classID - 2) + if (registration == null) throw new KryoException("Encountered unregistered class ID: " + (classID - 2)) + if (TRACE) trace("kryo", "Read class " + (classID - 2) + ": " + className(registration.getType)) + + registration + } + } + + def readIgnoredClass(input: Input): Unit = { + val classId = input.readVarInt(true) + if (classId == 1) { + val nameId = input.readVarInt(true) + if (nameIdToClass == null) { + nameIdToClass = new IntMap[Class[_]] + } + if (nameIdToClass.get(nameId) == null) { + val name = input.readString() + nameIdToClass.put(nameId, IgnoredClassPlaceholder.getClass) + } + } + } + + def writeFakeName[T](output: Output, name: String, placeholderType: Class[_]): Unit = { + output.writeVarInt(1, true) + if (classToNameId != null) { + val nameId = classToNameId.get(placeholderType, -1) + if (nameId != -1) { + output.writeVarInt(nameId, true) + return + } + } + // Only write the class name the first time encountered in object graph. + val nameId = nextNameId + nextNameId += 1 + if (classToNameId == null) classToNameId = new IdentityObjectIntMap[Class[_]] + classToNameId.put(placeholderType, nameId) + output.writeVarInt(nameId, true) + output.writeString(name) + } +} + +object BackwardCompatibleClassResolver { + abstract class FallbackToHashSet1 + abstract class FallbackToHashSet2 + abstract class FallbackToHashMap1 + abstract class FallbackToHashMap2 + abstract class FallbackToMap11 + abstract class FallbackToMap12 + + object IgnoredClassPlaceholder +} \ No newline at end of file diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala new file mode 100644 index 00000000..67280904 --- /dev/null +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala @@ -0,0 +1,74 @@ +/* +Copyright 2012 Twitter, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.chill.backwardcomp + +import com.esotericsoftware.kryo.serializers.FieldSerializer +import com.twitter.chill.{Input, Kryo, Output} + +class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Class[_]) extends FieldSerializer[T](kryo, typ) { + + override def read(kryo: Kryo, input: Input, typ: Class[T]): T = { + val result = create(kryo, input, typ) + kryo.reference(result) + + val bitmap0 = input.readByte() + val end = kryo.readClassAndObject(input) + val hashCode = input.readVarInt(false) + val isInclusive = input.readBoolean() + val last = kryo.readClassAndObject(input) + val num1 = kryo.readClassAndObject(input) + val num2 = kryo.readClassAndObject(input) + val numRangeElements = input.readVarInt(false) + val start = kryo.readClassAndObject(input) + val step = kryo.readClassAndObject(input) + + for (field <- getFields) { + field.getField.getName match { + case "bitmap$0" => field.getField.set(result, bitmap0) + case "end" => field.getField.set(result, end) + case "hashCode" => field.getField.set(result, hashCode) + case "isInclusive" => field.getField.set(result, isInclusive) + case "length" => field.getField.set(result, numRangeElements) + case "num" => field.getField.set(result, num1) + case "scala$collection$immutable$NumericRange$$num" => field.getField.set(result, num2) + case "start" => field.getField.set(result, start) + case "step" => field.getField.set(result, step) + } + } + + result + } + + override def write(kryo: Kryo, output: Output, `object`: T): Unit = { + var length: Option[Int] = None + for (field <- getFields) { + val value = field.getField.get(`object`) + field.getField.getName match { + case "bitmap$0" => output.writeByte(value.asInstanceOf[Byte]) + case "end" => kryo.writeClassAndObject(output, value) + case "hashCode" => output.writeVarInt(value.asInstanceOf[Int], false) + case "isInclusive" => output.writeBoolean(value.asInstanceOf[Boolean]) + case "length" => + kryo.writeClassAndObject(output, null) // last + length = Some(value.asInstanceOf[Int]) // storing length to be emitted after the implicits + case "num" => kryo.writeClassAndObject(output, value) + case "scala$collection$immutable$NumericRange$$num" => + kryo.writeClassAndObject(output, value) + output.writeVarInt(length.get, false) + case "start" => kryo.writeClassAndObject(output, value) + case "step" => kryo.writeClassAndObject(output, value) + } + } + } +} \ No newline at end of file diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala new file mode 100644 index 00000000..f5537e97 --- /dev/null +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala @@ -0,0 +1,39 @@ +/* +Copyright 2012 Twitter, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.chill.backwardcomp + +import com.esotericsoftware.kryo.serializers.FieldSerializer +import com.twitter.chill.{Input, Kryo, ObjectSerializer, Output} + +class BackwardCompatibleJavaWrappersSerializer[T](kryo: Kryo, typ: Class[_]) extends FieldSerializer[T](kryo, typ) { + import BackwardCompatibleJavaWrappersSerializer._ + + override def read(kryo: Kryo, input: Input, typ: Class[T]): T = { + // Skipping the first serialized field which is assumed to be an $outer object reference + kryo.getClassResolver.asInstanceOf[BackwardCompatibleClassResolver].readIgnoredClass(input) + val refId = input.readVarInt(true) + // Assuming ObjectSerializer which reads no data, so it does not mean if this is the first occurrence or not + super.read(kryo, input, typ) + } + + override def write(kryo: Kryo, output: Output, `object`: T): Unit = { + kryo.getClassResolver.asInstanceOf[BackwardCompatibleClassResolver].writeFakeName(output, "scala.collection.convert.Wrappers$", OuterWrapper.getClass) + kryo.writeObjectOrNull(output, OuterWrapper, new ObjectSerializer) + super.write(kryo, output, `object`) + } +} + +object BackwardCompatibleJavaWrappersSerializer { + object OuterWrapper +} diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala new file mode 100644 index 00000000..64903534 --- /dev/null +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala @@ -0,0 +1,228 @@ +/* +Copyright 2012 Twitter, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.twitter.chill.backwardcomp + +import com.esotericsoftware.kryo.util.MapReferenceResolver +import com.twitter.chill.java.{Java8ClosureRegistrar, PackageRegistrar} +import com.twitter.chill._ + +import scala.collection.JavaConverters._ +import scala.collection.immutable.{BitSet, HashMap, HashSet, ListMap, ListSet, NumericRange, Queue, Range, SortedMap, SortedSet, TreeMap, TreeSet, WrappedString} +import scala.collection.mutable +import scala.collection.mutable.{Buffer, ListBuffer, WrappedArray, BitSet => MBitSet, HashMap => MHashMap, HashSet => MHashSet, Map => MMap, Queue => MQueue, Set => MSet} +import scala.reflect.ClassTag +import scala.util.matching.Regex + +/** Makes an empty instantiator then registers everything */ +class BackwardCompatibleScalaKryoInstantiator extends EmptyScalaKryoInstantiator { + override def newKryo: KryoBase = { + val k = new KryoBase(new BackwardCompatibleClassResolver, new MapReferenceResolver) + k.setRegistrationRequired(false) + k.setInstantiatorStrategy(new org.objenesis.strategy.StdInstantiatorStrategy) + + // Handle cases where we may have an odd classloader setup like with libjars + // for hadoop + val classLoader = Thread.currentThread.getContextClassLoader + k.setClassLoader(classLoader) + + val reg = new BackwardCompatibleAllScalaRegistrar + reg(k) + k + } +} + +/** + * Note that additional scala collections registrations are provided by [[AllScalaRegistrar]]. They have not been + * included in this registrar for backwards compatibility reasons. + */ +class BackwardCompatibleScalaCollectionsRegistrar extends IKryoRegistrar { + def apply(newK: Kryo): Unit = { + // for binary compat this is here, but could be moved to RichKryo + def useField[T](cls: Class[T]): Unit = { + val fs = new BackwardCompatibleJavaWrappersSerializer(newK, cls) + fs.setIgnoreSyntheticFields(false) // scala generates a lot of these attributes + newK.register(cls, fs) + } + // The wrappers are private classes: + useField(List(1, 2, 3).asJava.getClass) + useField(List(1, 2, 3).iterator.asJava.getClass) + useField(Map(1 -> 2, 4 -> 3).asJava.getClass) + useField(new _root_.java.util.ArrayList().asScala.getClass) + useField(new _root_.java.util.HashMap().asScala.getClass) + + /* + * Note that subclass-based use: addDefaultSerializers, else: register + * You should go from MOST specific, to least to specific when using + * default serializers. The FIRST one found is the one used + */ + newK + // wrapper array is abstract + .forSubclass[WrappedArray[Any]](new WrappedArraySerializer[Any]) + .forSubclass[BitSet](new BitSetSerializer) + .forSubclass[SortedSet[Any]](new SortedSetSerializer) + .forClass[Some[Any]](new SomeSerializer[Any]) + .forClass[Left[Any, Any]](new LeftSerializer[Any, Any]) + .forClass[Right[Any, Any]](new RightSerializer[Any, Any]) + .forTraversableSubclass(Queue.empty[Any]) + // List is a sealed class, so there are only two subclasses: + .forTraversableSubclass(List.empty[Any]) + // Add ListBuffer subclass before Buffer to prevent the more general case taking precedence + .forTraversableSubclass(ListBuffer.empty[Any], isImmutable = false) + // add mutable Buffer before Vector, otherwise Vector is used + .forTraversableSubclass(Buffer.empty[Any], isImmutable = false) + // Vector is a final class + .forTraversableClass(Vector.empty[Any]) + .forTraversableSubclass(ListSet.empty[Any]) + // specifically register small sets since Scala represents them differently + .forConcreteTraversableClass(Set[Any]('a)) + .forConcreteTraversableClass(Set[Any]('a, 'b)) + .forConcreteTraversableClass(Set[Any]('a, 'b, 'c)) + .forConcreteTraversableClass(Set[Any]('a, 'b, 'c, 'd)) + // default set implementation + .forConcreteTraversableClass(HashSet[Any]('a, 'b, 'c, 'd, 'e)) + // specifically register small maps since Scala represents them differently + .forConcreteTraversableClass(Map[Any, Any]('a -> 'a)) + .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b)) + .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c)) + .forConcreteTraversableClass(Map[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c, 'd -> 'd)) + // default map implementation + .forConcreteTraversableClass(HashMap[Any, Any]('a -> 'a, 'b -> 'b, 'c -> 'c, 'd -> 'd, 'e -> 'e)) + // The normal fields serializer works for ranges + .registerClasses( + Seq(classOf[Range.Inclusive], classOf[NumericRange.Inclusive[_]]) + ) + + newK.register(classOf[NumericRange.Exclusive[_]], new BackwardCompatibleExclusiveNumericRangeSerializer(newK, classOf[NumericRange.Exclusive[_]])) + // Add some maps + + newK.forSubclass[SortedMap[Any, Any]](new SortedMapSerializer) + .forTraversableSubclass(ListMap.empty[Any, Any]) + .forTraversableSubclass(HashMap.empty[Any, Any]) + // The above ListMap/HashMap must appear before this: + .forTraversableSubclass(Map.empty[Any, Any]) + // here are the mutable ones: + .forTraversableClass(MBitSet.empty, isImmutable = false) + .forTraversableClass(MHashMap.empty[Any, Any], isImmutable = false) + .forTraversableClass(MHashSet.empty[Any], isImmutable = false) + .forTraversableSubclass(MQueue.empty[Any], isImmutable = false) + .forTraversableSubclass(MMap.empty[Any, Any], isImmutable = false) + .forTraversableSubclass(MSet.empty[Any], isImmutable = false) + } +} + +/** Registrar for everything that was registered in chill 0.9.2 - included for backwards compatibility. */ +class BackwardCompatibleAllScalaRegistrar_0_9_2 extends IKryoRegistrar { + def apply(k: Kryo): Unit = { + new BackwardCompatibleScalaCollectionsRegistrar()(k) + new JavaWrapperCollectionRegistrar()(k) + + // Register all 22 tuple serializers and specialized serializers + ScalaTupleSerialization.register(k) + k.forClass[Symbol](new KSerializer[Symbol] { + override def isImmutable = true + def write(k: Kryo, out: Output, obj: Symbol): Unit = out.writeString(obj.name) + def read(k: Kryo, in: Input, cls: Class[Symbol]): Symbol = Symbol(in.readString) + }) + .forSubclass[Regex](new RegexSerializer) + .forClass[ClassTag[Any]](new ClassTagSerializer[Any]) + .forSubclass[Manifest[Any]](new ManifestSerializer[Any]) + .forSubclass[scala.Enumeration#Value](new EnumerationSerializer) + + // use the singleton serializer for boxed Unit + val boxedUnit = scala.runtime.BoxedUnit.UNIT + k.register(boxedUnit.getClass, new SingletonSerializer(boxedUnit)) + PackageRegistrar.all()(k) + new Java8ClosureRegistrar()(k) + } +} + +/** + * Registers all the scala (and java) serializers we have. The registrations are designed to cover most of + * scala.collecion.immutable, so they can be used in long term persistence scenarios that run with + * setRegistrationRequired(true). + * + * When adding new serializers, add them to the end of the list, so compatibility is not broken needlessly + * for projects using chill for long term persistence - see com.twitter.chill.RegistrationIdsSpec. + */ +class BackwardCompatibleAllScalaRegistrar extends IKryoRegistrar { + def apply(k: Kryo): Unit = { + new BackwardCompatibleAllScalaRegistrar_0_9_2()(k) + + k.registerClasses( + Seq( + classOf[Array[Byte]], + classOf[Array[Short]], + classOf[Array[Int]], + classOf[Array[Long]], + classOf[Array[Float]], + classOf[Array[Double]], + classOf[Array[Boolean]], + classOf[Array[Char]], + classOf[Array[String]], + classOf[Array[Any]], + classOf[Class[_]], // needed for the WrappedArraySerializer + classOf[Any], // needed for scala.collection.mutable.WrappedArray$ofRef + mutable.WrappedArray.make(Array[Byte]()).getClass, + mutable.WrappedArray.make(Array[Short]()).getClass, + mutable.WrappedArray.make(Array[Int]()).getClass, + mutable.WrappedArray.make(Array[Long]()).getClass, + mutable.WrappedArray.make(Array[Float]()).getClass, + mutable.WrappedArray.make(Array[Double]()).getClass, + mutable.WrappedArray.make(Array[Boolean]()).getClass, + mutable.WrappedArray.make(Array[Char]()).getClass, + mutable.WrappedArray.make(Array[String]()).getClass, + None.getClass, + classOf[Queue[_]], + Nil.getClass, + classOf[::[_]], + classOf[Range], + classOf[WrappedString], + classOf[TreeSet[_]], + classOf[TreeMap[_, _]], + // The most common orderings for TreeSet and TreeMap + Ordering.Byte.getClass, + Ordering.Short.getClass, + Ordering.Int.getClass, + Ordering.Long.getClass, + Ordering.Float.getClass, + Ordering.Double.getClass, + Ordering.Boolean.getClass, + Ordering.Char.getClass, + Ordering.String.getClass + ) + ) + .forConcreteTraversableClass(Set[Any]()) + .forConcreteTraversableClass(ListSet[Any]()) + .forConcreteTraversableClass(ListSet[Any]('a)) + + k.register(classOf[BackwardCompatibleClassResolver.FallbackToHashSet1]) + k.register(classOf[BackwardCompatibleClassResolver.FallbackToHashSet2]) + + k.forConcreteTraversableClass(Map[Any, Any]()) + + k.register(classOf[BackwardCompatibleClassResolver.FallbackToHashMap1]) + k.register(classOf[BackwardCompatibleClassResolver.FallbackToHashMap2]) + + k.forConcreteTraversableClass(ListMap[Any, Any]()) + .forConcreteTraversableClass(ListMap('a -> 'a)) + k.register(classOf[Stream.Cons[_]], new StreamSerializer[Any]) + k.register(Stream.empty[Any].getClass) + k.forClass[scala.runtime.VolatileByteRef](new VolatileByteRefSerializer) + k.forClass[BigDecimal](new BigDecimalSerializer) + k.register(Queue.empty[Any].getClass) + k.register(classOf[BackwardCompatibleClassResolver.FallbackToMap11]) + k.register(classOf[BackwardCompatibleClassResolver.FallbackToMap12]) + k.forConcreteTraversableClass(Map(1 -> 2).keySet) + } +} \ No newline at end of file From ec9d6b84637c4d763db10be97ee7982703f8f17e Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Mon, 14 Sep 2020 09:27:25 +0200 Subject: [PATCH 2/4] Get rid of return --- .../backwardcomp/BackwardCompatibleClassResolver.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala index ca19f9e0..0d421878 100644 --- a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala @@ -99,9 +99,15 @@ class BackwardCompatibleClassResolver extends DefaultClassResolver { val nameId = classToNameId.get(placeholderType, -1) if (nameId != -1) { output.writeVarInt(nameId, true) - return + } else { + writeClassNameOnFirstEncounter(output, name, placeholderType) } + } else { + writeClassNameOnFirstEncounter(output, name, placeholderType) } + } + + private def writeClassNameOnFirstEncounter[T](output: Output, name: String, placeholderType: Class[_]) = { // Only write the class name the first time encountered in object graph. val nameId = nextNameId nextNameId += 1 From 2f04874706747a05373946562b357f29e4268627 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Mon, 14 Sep 2020 09:30:47 +0200 Subject: [PATCH 3/4] Use while in BackwardCompatibleExclusiveNumericRangeSerializer --- ...CompatibleExclusiveNumericRangeSerializer.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala index 67280904..53b525f1 100644 --- a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala @@ -33,7 +33,12 @@ class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Clas val start = kryo.readClassAndObject(input) val step = kryo.readClassAndObject(input) - for (field <- getFields) { + var idx = 0 + val fields = getFields + while (idx < fields.length) { + val field = fields(idx) + idx = idx + 1 + field.getField.getName match { case "bitmap$0" => field.getField.set(result, bitmap0) case "end" => field.getField.set(result, end) @@ -52,7 +57,12 @@ class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Clas override def write(kryo: Kryo, output: Output, `object`: T): Unit = { var length: Option[Int] = None - for (field <- getFields) { + var idx = 0 + val fields = getFields + while (idx < fields.length) { + val field = fields(idx) + idx = idx + 1 + val value = field.getField.get(`object`) field.getField.getName match { case "bitmap$0" => output.writeByte(value.asInstanceOf[Byte]) From 51ded442728d879065213a627f48c859d6896345 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Mon, 14 Sep 2020 09:33:14 +0200 Subject: [PATCH 4/4] scalaFmt --- .../BackwardCompatibleClassResolver.scala | 10 ++--- ...tibleExclusiveNumericRangeSerializer.scala | 29 ++++++------ ...wardCompatibleJavaWrappersSerializer.scala | 7 ++- ...kwardCompatibleScalaKryoInstantiator.scala | 44 +++++++++++++++---- 4 files changed, 60 insertions(+), 30 deletions(-) diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala index 0d421878..885c9a9e 100644 --- a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleClassResolver.scala @@ -16,7 +16,7 @@ package com.twitter.chill.backwardcomp import com.esotericsoftware.kryo.util.Util.{className, log} import com.esotericsoftware.kryo.util.{DefaultClassResolver, IdentityObjectIntMap, IntMap} import com.esotericsoftware.kryo.{Kryo, KryoException, Registration} -import com.esotericsoftware.minlog.Log.{DEBUG, TRACE, trace} +import com.esotericsoftware.minlog.Log.{trace, DEBUG, TRACE} import com.twitter.chill.{Input, Output} class BackwardCompatibleClassResolver extends DefaultClassResolver { @@ -33,13 +33,12 @@ class BackwardCompatibleClassResolver extends DefaultClassResolver { classOf[Range.Exclusive] -> 118 ) - override def getRegistration(`type`: Class[_]): Registration = { + override def getRegistration(`type`: Class[_]): Registration = if (`type` == classOf[Range.Exclusive]) { getRegistration(118) } else { super.getRegistration(`type`) } - } override def getRegistration(classID: Int): Registration = { val registration = super.getRegistration(classID) @@ -72,7 +71,8 @@ class BackwardCompatibleClassResolver extends DefaultClassResolver { readName(input) case _ => val registration = getRegistration(classID - 2) - if (registration == null) throw new KryoException("Encountered unregistered class ID: " + (classID - 2)) + if (registration == null) + throw new KryoException("Encountered unregistered class ID: " + (classID - 2)) if (TRACE) trace("kryo", "Read class " + (classID - 2) + ": " + className(registration.getType)) registration @@ -127,4 +127,4 @@ object BackwardCompatibleClassResolver { abstract class FallbackToMap12 object IgnoredClassPlaceholder -} \ No newline at end of file +} diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala index 53b525f1..eca60338 100644 --- a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleExclusiveNumericRangeSerializer.scala @@ -16,7 +16,8 @@ package com.twitter.chill.backwardcomp import com.esotericsoftware.kryo.serializers.FieldSerializer import com.twitter.chill.{Input, Kryo, Output} -class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Class[_]) extends FieldSerializer[T](kryo, typ) { +class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Class[_]) + extends FieldSerializer[T](kryo, typ) { override def read(kryo: Kryo, input: Input, typ: Class[T]): T = { val result = create(kryo, input, typ) @@ -40,15 +41,15 @@ class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Clas idx = idx + 1 field.getField.getName match { - case "bitmap$0" => field.getField.set(result, bitmap0) - case "end" => field.getField.set(result, end) - case "hashCode" => field.getField.set(result, hashCode) - case "isInclusive" => field.getField.set(result, isInclusive) - case "length" => field.getField.set(result, numRangeElements) - case "num" => field.getField.set(result, num1) + case "bitmap$0" => field.getField.set(result, bitmap0) + case "end" => field.getField.set(result, end) + case "hashCode" => field.getField.set(result, hashCode) + case "isInclusive" => field.getField.set(result, isInclusive) + case "length" => field.getField.set(result, numRangeElements) + case "num" => field.getField.set(result, num1) case "scala$collection$immutable$NumericRange$$num" => field.getField.set(result, num2) - case "start" => field.getField.set(result, start) - case "step" => field.getField.set(result, step) + case "start" => field.getField.set(result, start) + case "step" => field.getField.set(result, step) } } @@ -65,9 +66,9 @@ class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Clas val value = field.getField.get(`object`) field.getField.getName match { - case "bitmap$0" => output.writeByte(value.asInstanceOf[Byte]) - case "end" => kryo.writeClassAndObject(output, value) - case "hashCode" => output.writeVarInt(value.asInstanceOf[Int], false) + case "bitmap$0" => output.writeByte(value.asInstanceOf[Byte]) + case "end" => kryo.writeClassAndObject(output, value) + case "hashCode" => output.writeVarInt(value.asInstanceOf[Int], false) case "isInclusive" => output.writeBoolean(value.asInstanceOf[Boolean]) case "length" => kryo.writeClassAndObject(output, null) // last @@ -77,8 +78,8 @@ class BackwardCompatibleExclusiveNumericRangeSerializer[T](kryo: Kryo, typ: Clas kryo.writeClassAndObject(output, value) output.writeVarInt(length.get, false) case "start" => kryo.writeClassAndObject(output, value) - case "step" => kryo.writeClassAndObject(output, value) + case "step" => kryo.writeClassAndObject(output, value) } } } -} \ No newline at end of file +} diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala index f5537e97..4a9f8d20 100644 --- a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleJavaWrappersSerializer.scala @@ -16,7 +16,8 @@ package com.twitter.chill.backwardcomp import com.esotericsoftware.kryo.serializers.FieldSerializer import com.twitter.chill.{Input, Kryo, ObjectSerializer, Output} -class BackwardCompatibleJavaWrappersSerializer[T](kryo: Kryo, typ: Class[_]) extends FieldSerializer[T](kryo, typ) { +class BackwardCompatibleJavaWrappersSerializer[T](kryo: Kryo, typ: Class[_]) + extends FieldSerializer[T](kryo, typ) { import BackwardCompatibleJavaWrappersSerializer._ override def read(kryo: Kryo, input: Input, typ: Class[T]): T = { @@ -28,7 +29,9 @@ class BackwardCompatibleJavaWrappersSerializer[T](kryo: Kryo, typ: Class[_]) ext } override def write(kryo: Kryo, output: Output, `object`: T): Unit = { - kryo.getClassResolver.asInstanceOf[BackwardCompatibleClassResolver].writeFakeName(output, "scala.collection.convert.Wrappers$", OuterWrapper.getClass) + kryo.getClassResolver + .asInstanceOf[BackwardCompatibleClassResolver] + .writeFakeName(output, "scala.collection.convert.Wrappers$", OuterWrapper.getClass) kryo.writeObjectOrNull(output, OuterWrapper, new ObjectSerializer) super.write(kryo, output, `object`) } diff --git a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala index 64903534..82d50e9f 100644 --- a/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala +++ b/chill-scala/src/main/scala-2.13+/com/twitter/chill/backwardcomp/BackwardCompatibleScalaKryoInstantiator.scala @@ -18,9 +18,33 @@ import com.twitter.chill.java.{Java8ClosureRegistrar, PackageRegistrar} import com.twitter.chill._ import scala.collection.JavaConverters._ -import scala.collection.immutable.{BitSet, HashMap, HashSet, ListMap, ListSet, NumericRange, Queue, Range, SortedMap, SortedSet, TreeMap, TreeSet, WrappedString} +import scala.collection.immutable.{ + BitSet, + HashMap, + HashSet, + ListMap, + ListSet, + NumericRange, + Queue, + Range, + SortedMap, + SortedSet, + TreeMap, + TreeSet, + WrappedString +} import scala.collection.mutable -import scala.collection.mutable.{Buffer, ListBuffer, WrappedArray, BitSet => MBitSet, HashMap => MHashMap, HashSet => MHashSet, Map => MMap, Queue => MQueue, Set => MSet} +import scala.collection.mutable.{ + Buffer, + ListBuffer, + WrappedArray, + BitSet => MBitSet, + HashMap => MHashMap, + HashSet => MHashSet, + Map => MMap, + Queue => MQueue, + Set => MSet +} import scala.reflect.ClassTag import scala.util.matching.Regex @@ -103,10 +127,14 @@ class BackwardCompatibleScalaCollectionsRegistrar extends IKryoRegistrar { Seq(classOf[Range.Inclusive], classOf[NumericRange.Inclusive[_]]) ) - newK.register(classOf[NumericRange.Exclusive[_]], new BackwardCompatibleExclusiveNumericRangeSerializer(newK, classOf[NumericRange.Exclusive[_]])) + newK.register( + classOf[NumericRange.Exclusive[_]], + new BackwardCompatibleExclusiveNumericRangeSerializer(newK, classOf[NumericRange.Exclusive[_]]) + ) // Add some maps - newK.forSubclass[SortedMap[Any, Any]](new SortedMapSerializer) + newK + .forSubclass[SortedMap[Any, Any]](new SortedMapSerializer) .forTraversableSubclass(ListMap.empty[Any, Any]) .forTraversableSubclass(HashMap.empty[Any, Any]) // The above ListMap/HashMap must appear before this: @@ -133,8 +161,7 @@ class BackwardCompatibleAllScalaRegistrar_0_9_2 extends IKryoRegistrar { override def isImmutable = true def write(k: Kryo, out: Output, obj: Symbol): Unit = out.writeString(obj.name) def read(k: Kryo, in: Input, cls: Class[Symbol]): Symbol = Symbol(in.readString) - }) - .forSubclass[Regex](new RegexSerializer) + }).forSubclass[Regex](new RegexSerializer) .forClass[ClassTag[Any]](new ClassTagSerializer[Any]) .forSubclass[Manifest[Any]](new ManifestSerializer[Any]) .forSubclass[scala.Enumeration#Value](new EnumerationSerializer) @@ -201,8 +228,7 @@ class BackwardCompatibleAllScalaRegistrar extends IKryoRegistrar { Ordering.Char.getClass, Ordering.String.getClass ) - ) - .forConcreteTraversableClass(Set[Any]()) + ).forConcreteTraversableClass(Set[Any]()) .forConcreteTraversableClass(ListSet[Any]()) .forConcreteTraversableClass(ListSet[Any]('a)) @@ -225,4 +251,4 @@ class BackwardCompatibleAllScalaRegistrar extends IKryoRegistrar { k.register(classOf[BackwardCompatibleClassResolver.FallbackToMap12]) k.forConcreteTraversableClass(Map(1 -> 2).keySet) } -} \ No newline at end of file +}