Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

implementation for fields compatibility in default configuration #248

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

implementation for fields compatibility in default configuration #248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 116 additions & 37 deletions src/main/java/org/nustaq/serialization/FSTClazzInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.zip.CRC32;

/**
* Created with IntelliJ IDEA.
Expand Down Expand Up @@ -121,12 +122,19 @@ public void setDecoderAttached(Object decoderAttached) {
boolean crossPlatform;

public FSTClazzInfo(FSTConfiguration conf, Class clazz, FSTClazzInfoRegistry infoRegistry, boolean ignoreAnnotations) {
this(conf, clazz, infoRegistry, ignoreAnnotations, null);
}

/**
* new constructor with the serialized order of fields and the uniqueId for each field
*/
public FSTClazzInfo(FSTConfiguration conf, Class clazz, FSTClazzInfoRegistry infoRegistry, boolean ignoreAnnotations, long[] fieldsUniqueId) {
this.conf = conf; // fixme: historically was not bound to conf but now is. Remove redundant state + refs (note: may still be useful because of less pointerchasing)
crossPlatform = conf.isCrossPlatform();
this.clazz = clazz;
enumConstants = clazz.getEnumConstants();
ignoreAnn = ignoreAnnotations;
createFields(clazz);
createFields(clazz, fieldsUniqueId);

instantiator = conf.getInstantiator(clazz);
if (Externalizable.class.isAssignableFrom(clazz)) {
Expand Down Expand Up @@ -357,45 +365,53 @@ private FSTMap buildFieldMap() {
return res;
}

private void createFields(Class c) {
private void createFields(Class c, long[] fieldsUniqueId) {
if (c.isInterface() || c.isPrimitive()) {
return;
}
List<Field> fields = getAllFields(c, null);
fieldInfo = new FSTFieldInfo[fields.size()];
for (int i = 0; i < fields.size(); i++) {
Field field = fields.get(i);
fieldInfo[i] = createFieldInfo(field);
if(fieldsUniqueId != null){
// retains only the fields that are in the fields unique
// and keep the given order. for those in unique that
// are not present - just add a FSTFieldInfo with missing set to true
// in order to jump over the data and ignore it
List<FSTFieldInfo> fstFields = new ArrayList<FSTFieldInfo>(fields.size());
int fieldsLen = fields.size();
for (int i = 0; i < fieldsLen; i++) {
fstFields.add(createFieldInfo(fields.get(i)));
}
// create the fields list
final int lenFieldsUniqueId = fieldsUniqueId.length;
fieldInfo = new FSTFieldInfo[lenFieldsUniqueId];
long uid;
boolean foundIt;
for(int i=0;i<lenFieldsUniqueId;i++){
uid = fieldsUniqueId[i];
foundIt = false;
for(FSTFieldInfo ffi : fstFields){
if(ffi.getUniqueId() == uid){
// found it
fieldInfo[i] = ffi;
foundIt = true;
break;
}
}
if(foundIt) {
//
fstFields.remove(fieldInfo[i]);
}else{
//
(fieldInfo[i] = new FSTFieldInfo(null, null, true)).missing = true;
}
}
} else {
fieldInfo = new FSTFieldInfo[fields.size()];
int fieldsLen = fields.size();
for (int i = 0; i < fieldsLen; i++) {
fieldInfo[i] = createFieldInfo(fields.get(i));
}
}

// compatibility info sort order
Comparator<FSTFieldInfo> infocomp = new Comparator<FSTFieldInfo>() {
@Override
public int compare(FSTFieldInfo o1, FSTFieldInfo o2) {
int res = 0;
res = o1.getType().getSimpleName().compareTo(o2.getType().getSimpleName());
if (res == 0)
res = o1.getType().getName().compareTo(o2.getType().getName());
if (res == 0) {
Class declaringClass = o1.getType().getDeclaringClass();
Class declaringClass1 = o2.getType().getDeclaringClass();
if (declaringClass == null && declaringClass1 == null) {
return 0;
}
if (declaringClass != null && declaringClass1 == null) {
return 1;
}
if (declaringClass == null && declaringClass1 != null) {
return -1;
}
if (res == 0) {
return declaringClass.getName().compareTo(declaringClass1.getName());
}
}
return res;
}
};


// check if we actually need to build up compatibility info (memory intensive)
boolean requiresCompatibilityData = false;
if ( ! Externalizable.class.isAssignableFrom(c) && getSerNoStore() == null ) {
Expand Down Expand Up @@ -443,6 +459,33 @@ public int compare(FSTFieldInfo o1, FSTFieldInfo o2) {
}
}
}
// compatibility info sort order
Comparator<FSTFieldInfo> infocomp = new Comparator<FSTFieldInfo>() {
@Override
public int compare(FSTFieldInfo o1, FSTFieldInfo o2) {
int res = 0;
res = o1.getType().getSimpleName().compareTo(o2.getType().getSimpleName());
if (res == 0)
res = o1.getType().getName().compareTo(o2.getType().getName());
if (res == 0) {
Class declaringClass = o1.getType().getDeclaringClass();
Class declaringClass1 = o2.getType().getDeclaringClass();
if (declaringClass == null && declaringClass1 == null) {
return 0;
}
if (declaringClass != null && declaringClass1 == null) {
return 1;
}
if (declaringClass == null && declaringClass1 != null) {
return -1;
}
if (res == 0) {
return declaringClass.getName().compareTo(declaringClass1.getName());
}
}
return res;
}
};
Collections.sort(curClzFields, infocomp);
FSTCompatibilityInfo info = new FSTCompatibilityInfo(curClzFields, curCl);
getCompInfo().put(curCl, info);
Expand All @@ -456,12 +499,14 @@ public int compare(FSTFieldInfo o1, FSTFieldInfo o2) {

// default sort order
Comparator<FSTFieldInfo> comp = defFieldComparator;
if (!conf.isStructMode())
if (!conf.isStructMode() && (fieldsUniqueId == null))
Arrays.sort(fieldInfo, comp);
int off = 8; // object header: length + clzId
for (int i = 0; i < fieldInfo.length; i++) {
FSTFieldInfo fstFieldInfo = fieldInfo[i];
Align al = fstFieldInfo.getField().getAnnotation(Align.class);
// ARC: managing the missing fields
Field field;
Align al = (field = fstFieldInfo.getField())==null? null : field.getAnnotation(Align.class);
if (al != null) {
fstFieldInfo.align = al.value();
int alignOff = fstFieldInfo.align(off);
Expand Down Expand Up @@ -581,6 +626,40 @@ public final static class FSTFieldInfo {
// in rare cases, a field used in putField is not present as a real field
// in this case only these of a fieldinfo are set
public String fakeName;

// ARC: crc unique of the field name
Long uniqueId;
public boolean hasUniqueId() {
return (field!=null) || (fakeName == null);
}
public long getUniqueId(){
if(uniqueId == null){
if(field==null && fakeName == null){
// this should not happen
throw new UnsupportedOperationException("no unique id");
}else{
try{
CRC32 crc = new CRC32();
if(field==null){
crc.update(fakeName.getBytes("UTF-8"));
}else{
crc.update(field.getDeclaringClass().getName().getBytes("UTF-8"));
crc.update('.');
crc.update(field.getName().getBytes("UTF-8"));
}
uniqueId = crc.getValue();
}catch(UnsupportedEncodingException e){
throw new RuntimeException(e);
}
}
}
return uniqueId;
}
boolean missing = false;

public boolean isMissing() {
return missing;
}

public FSTFieldInfo(Class[] possibleClasses, Field fi, boolean ignoreAnnotations) {
this.possibleClasses = possibleClasses;
Expand Down
6 changes: 5 additions & 1 deletion src/main/java/org/nustaq/serialization/FSTClazzInfoRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ public FSTClazzInfoRegistry() {
}

public FSTClazzInfo getCLInfo(Class c, FSTConfiguration conf) {
return getCLInfo(c, conf, null);
}

public FSTClazzInfo getCLInfo(Class c, FSTConfiguration conf, long[] fieldsUniqueId) {
while(!rwLock.compareAndSet(false,true));
try {
FSTClazzInfo res = (FSTClazzInfo) mInfos.get(c);
Expand All @@ -126,7 +130,7 @@ public FSTClazzInfo getCLInfo(Class c, FSTConfiguration conf) {
if ( ! conf.getVerifier().allowClassDeserialization(c) )
throw new RuntimeException("tried to deserialize forbidden class "+c.getName() );
}
res = new FSTClazzInfo(conf, c, this, ignoreAnnotations);
res = new FSTClazzInfo(conf, c, this, ignoreAnnotations, fieldsUniqueId);
mInfos.put(c, res);
}
return res;
Expand Down
59 changes: 51 additions & 8 deletions src/main/java/org/nustaq/serialization/FSTClazzNameRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package org.nustaq.serialization;

import org.nustaq.offheap.structs.unsafeimpl.FSTStructFactory;
import org.nustaq.serialization.FSTClazzInfo.FSTFieldInfo;
import org.nustaq.serialization.util.FSTIdentity2IdMap;
import org.nustaq.serialization.util.FSTObject2IntMap;
import org.nustaq.serialization.util.FSTUtil;
Expand All @@ -42,7 +43,13 @@
*/
public class FSTClazzNameRegistry {

public static final int LOWEST_CLZ_ID = 3;
/**
* it seems that originally only bit 0 and 1 was used (<3)
*
* we will use the bit 2 to signal that the fields uniqueId are saved with the class info
*
*/
public static final int LOWEST_CLZ_ID = FSTConfiguration.FIELDS_FIX ? 4 : 3;
public static final int FIRST_USER_CLZ_ID = 1000;

FSTIdentity2IdMap clzToId;
Expand Down Expand Up @@ -133,29 +140,56 @@ public void encodeClass(FSTEncoder out, FSTClazzInfo ci) throws IOException {
// ugly hack, also making assumptions about
// on how the encoder works internally
final byte[] bufferedName = ci.getBufferedName();
out.writeFShort((short) 1); // no direct cl id ascii enc
if(FSTConfiguration.FIELDS_FIX)
out.writeFShort((short) 3); // no direct cl id ascii enc
else
out.writeFShort((short) 1); // no direct cl id ascii enc
out.writeFInt((char) bufferedName.length);
out.writeRawBytes(bufferedName,0,bufferedName.length);
//
if(FSTConfiguration.FIELDS_FIX){
FSTFieldInfo[] fields = ci.getFieldInfo();
int len = fields==null ? 0 : fields.length;
out.writeFShort((short)fields.length);
for(int i=0;i<len;i++){
out.writeFLong(fields[i].getUniqueId());
}
}
registerClassNoLookup(aClass,ci,ci.conf);
}
} else {
encodeClass(out,ci.getClazz());
encodeClass(out,ci.getClazz(),ci);
}
}
}

public void encodeClass(FSTEncoder out, Class c) throws IOException {
encodeClass(out, c, null);
}

private void encodeClass(FSTEncoder out, Class c, FSTClazzInfo ci) throws IOException {
int clid = getIdFromClazz(c);
if ( clid != Integer.MIN_VALUE ) {
out.writeFShort((short) clid); // > 2 !!
} else {
encodeClassName(out, c, out.getConf() );
encodeClassName(out, c, out.getConf(), ci );
}
}

private void encodeClassName(FSTEncoder out, Class c, FSTConfiguration conf) throws IOException {
out.writeFShort((short) 0); // no direct cl id
private void encodeClassName(FSTEncoder out, Class c, FSTConfiguration conf, FSTClazzInfo ci) throws IOException {
if(FSTConfiguration.FIELDS_FIX && ci!=null)
out.writeFShort((short) 2); // no direct cl id
else
out.writeFShort((short) 0); // no direct cl id
out.writeStringUTF(c.getName());
if(FSTConfiguration.FIELDS_FIX && ci!=null){
final FSTFieldInfo[] fields = ci.getFieldInfo();
final int len = fields==null ? 0 : fields.length;
out.writeFShort((short)fields.length);
for(int i=0;i<len;i++){
out.writeFLong(fields[i].getUniqueId());
}
}
registerClassNoLookup(c,null,conf);
}

Expand All @@ -164,14 +198,23 @@ public FSTClazzInfo decodeClass(FSTDecoder in, FSTConfiguration conf) throws IOE
if ( c < LOWEST_CLZ_ID ) {
// full class name
String clName;
if ( c==0) {
if (c==0 || c==2) {
clName = in.readStringUTF();
}
else {
clName = in.readStringAsc();
}
Class cl = classForName(clName,conf);
final FSTClazzInfo clInfo = conf.getCLInfoRegistry().getCLInfo(cl, conf);
long fieldsUnique[] = null;
if ((c & 2) == 2) {
/* read fields */
int len = in.readFShort();
fieldsUnique = new long[len];
for(int i=0;i<len;i++){
fieldsUnique[i] = in.readFLong();
}
}
final FSTClazzInfo clInfo = conf.getCLInfoRegistry().getCLInfo(cl, conf,fieldsUnique);
registerClassNoLookup(cl,clInfo,conf);
return clInfo;
} else {
Expand Down
30 changes: 30 additions & 0 deletions src/main/java/org/nustaq/serialization/FSTConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,36 @@
*
*/
public class FSTConfiguration {

/**
* ARC: a very simple hack in order to handle deserialisation of un-synched classes from the point of view of fields.
*
* The hack is handling removed/added fields on the receiver side or sending side. EQ: a field is removed/added at the
* serialisation VM or de-serialising VM.
*
* The hack is using an uniqueId computed as a CRC32 on the name of the declaring class and the field name. The uniqueId
* is used to find the removed or added fields.
*
* On the de-serialising side the removed fields are read but not set, the new fields are ignored. In order to "jump" over
* the field data we will prefix the data with a byte or 4byte information about the size of the serialized field.
*
* The hack is enforcing the order of de-serialisation as the one at the time of serialisation.
*
* Note: I did not implemented all the scenarios as we do not need it them.
*
* Thus the hack is working only with in default SHARED configuration with classes that are not registred and that are not in
* compatible mode! Also the hack is not working with Conditional or Version annotations
*
* Further work : make this configurable and not static as it is
* Cons: the resulting binary format is bigger.
* Pros: backward/forward compatibility between versions of classes in simple cases (adding/removing fields)
*/
public final static boolean FIELDS_FIX = true;
public final static boolean FIELDS_FIX_LENGTH = FIELDS_FIX;
/**
* ARC: store enum with names and not ordinal - as if the position changes then the deserialisation is not good.
*/
public static boolean FIELDS_FIX_ENUM_WITH_NAME = FIELDS_FIX;

static enum ConfType {
DEFAULT, UNSAFE, MINBIN, JSON, JSONPRETTY
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/org/nustaq/serialization/FSTEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ public interface FSTEncoder {
* @param v
*/
void writeInt32At(int position, int v);
/**
* used to write uncompressed int (guaranteed length = 1) at a (eventually recent) position
* @param position
* @param v
*/
void writeByteAt(int position, byte v);

/**
* if output stream is null, just encode into a byte array
Expand Down
Loading