Skip to content

Commit

Permalink
HPCC4J-650 DFSClient should retain integer subtypes
Browse files Browse the repository at this point in the history 
- Modified record translation code to maintain integer subtype
- Fixed record translation test case
- Added index record translation to test case

Signed-off-by: James McMullan [email protected]
  • Loading branch information
@jpmcmu
jpmcmu committed Oct 3, 2024
1 parent 434557e commit a5bc1c2
Show file tree
Hide file tree
Showing 9 changed files with 168 additions and 67 deletions.
39 changes: 31 additions & 8 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ public class FieldDef implements Serializable
private long len = 0;
private boolean fixedLength = false;
private boolean isUnsigned = false;
private boolean isBiased = false;
private int additionalFlags = 0;

/**
Expand All @@ -49,12 +48,17 @@ public FieldDef(FieldDef rhs)
this.fieldName = rhs.fieldName;
this.fieldType = rhs.fieldType;
this.typeName = rhs.typeName;
this.defs = rhs.defs;

this.defs = new FieldDef[rhs.defs.length];
for (int i = 0; i < rhs.defs.length; i++)
{
this.defs[i] = new FieldDef(rhs.defs[i]);
}

this.srcType = rhs.srcType;
this.len = rhs.len;
this.fixedLength = rhs.fixedLength;
this.isUnsigned = rhs.isUnsigned;
this.isBiased = rhs.isBiased;
this.additionalFlags = rhs.additionalFlags;
}

Expand Down Expand Up @@ -172,6 +176,16 @@ public HpccSrcType getSourceType()
return this.srcType;
}

/**
* Sets data type on the HPCC cluster.
*
* @param srcType the new source type
*/
public void setSourceType(HpccSrcType srcType)
{
this.srcType = srcType;
}

/**
* Length of data or minimum length if variable.
*
Expand Down Expand Up @@ -305,18 +319,27 @@ public boolean isUnsigned()
}

/**
* Is the underlying value biased?
* Is the underlying value biased? Deprecated in favor of isNonStandardInt.
*
* @return true when biased
* @return true when biased
*
* @deprecated
*/
public boolean isBiased()
{
return this.isBiased;
return isNonStandardInt();
}

void setIsBiased(boolean biased)
/**
*
*
* @return true when biased
*/
public boolean isNonStandardInt()
{
this.isBiased = biased;
return this.srcType == HpccSrcType.KEYED_INTEGER
|| this.srcType == HpccSrcType.SWAPPED_INTEGER
|| this.srcType == HpccSrcType.BIAS_SWAPPED_INTEGER;
}

/**
Expand Down
6 changes: 6 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/HpccSrcType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ public enum HpccSrcType
"little endian", false
), BINARY_CODED_DECIMAL (
"Binary coded decimal", false
), KEYED_INTEGER (
"Non-payload integer field within a key", false
), SWAPPED_INTEGER (
"Byte swapped integer, used within keys", false
), BIAS_SWAPPED_INTEGER (
"Byte swapped integer, deprecated", false
), UNKNOWN (
"Unkown", false
);
Expand Down
70 changes: 53 additions & 17 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public class RecordDefinitionTranslator
private static final String CHILD_KEY = "child";
private static final String FLAGS_KEY = "flags";

private static final String ESP_TYPE_NAME_PREFIX = "ty";

private static final int FLAG_UNSIGNED = 256;
private static final int FLAG_UNKNOWN_SIZE = 1024;
private static final int TYPE_ID_MASK = 0xff; // 0x7fff & ~FLAG_UNKNOWN_SIZE & ~FLAG_UNSIGNED;
Expand All @@ -55,6 +57,7 @@ public class RecordDefinitionTranslator
final private static int type_varunicode = 33;
final private static int type_utf8 = 41;


// FNoInitializer, // 0 means no initialiser - not a special virtual initialiser
// FVirtualFilePosition,
// FVirtualLocalFilePosition,
Expand All @@ -68,6 +71,8 @@ public class RecordDefinitionTranslator
final private static int type_char = 11; // Convert to string
final private static int type_qstring = 30; // Convert to string

final private static char XPATH_DELIMITER = 0x0001;

// Additional retained flags
final private static int FLAG_IS_PAYLOAD_FIELD = 0x00010000;

Expand Down Expand Up @@ -134,9 +139,12 @@ private static HpccSrcType getSourceType(int typeID)
case type_int:
case type_real:
return HpccSrcType.LITTLE_ENDIAN;
case type_swapint:
case type_biasedswapint:
return HpccSrcType.BIAS_SWAPPED_INTEGER;
case type_swapint:
return HpccSrcType.SWAPPED_INTEGER;
case type_keyedint:
return HpccSrcType.KEYED_INTEGER;
case type_filepos:
return HpccSrcType.BIG_ENDIAN;
case type_utf8:
Expand Down Expand Up @@ -272,7 +280,7 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
return "DATA" + field.getDataLen();
}

return "DATA";
}
case BOOLEAN:
Expand All @@ -298,7 +306,7 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
if (field.isUnsigned() == false)
{
throw new Exception("Error: Filepos must be unsigned");
throw new Exception("Error: Filepos must be unsigned");
}

if (field.getDataLen() != 8)
Expand Down Expand Up @@ -454,7 +462,7 @@ public static JSONObject toJsonRecord(FieldDef field) throws Exception
continue;
}

rootDefinition.put("ty" + (i + 1), typeDefinition);
rootDefinition.put(ESP_TYPE_NAME_PREFIX + (i + 1), typeDefinition);
}

return rootDefinition;
Expand All @@ -471,7 +479,7 @@ public static JSONObject toJsonRecord(FieldDef field) throws Exception
*/
private static int getTypeID(FieldDef field) throws Exception
{
int typeID = 0;
int typeID = -1;
switch (field.getFieldType())
{
case SET:
Expand Down Expand Up @@ -514,10 +522,25 @@ private static int getTypeID(FieldDef field) throws Exception
case INTEGER:
{
typeID = type_int;
HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.KEYED_INTEGER)
{
typeID = type_keyedint;
}
else if (srcType == HpccSrcType.SWAPPED_INTEGER)
{
typeID = type_swapint;
}
else if (srcType == HpccSrcType.BIAS_SWAPPED_INTEGER)
{
typeID = type_biasedswapint;
}

if (field.isUnsigned())
{
typeID |= FLAG_UNSIGNED;
}

break;
}
case DECIMAL:
Expand Down Expand Up @@ -625,7 +648,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
*/
private static int getTypeHash(FieldDef field) throws Exception
{
int numHashComponents = 2 + field.getNumDefs();
int numHashComponents = 4 + field.getNumDefs();
if (field.getFieldType() == FieldType.DECIMAL)
{
numHashComponents += 2;
Expand All @@ -634,8 +657,10 @@ private static int getTypeHash(FieldDef field) throws Exception
long[] hashComponents = new long[numHashComponents];
hashComponents[0] = getTypeID(field);
hashComponents[1] = field.getDataLen();
hashComponents[2] = field.getSourceType().ordinal();
hashComponents[3] = field.getAdditionalFlags();

int hashCompIndex = 2;
int hashCompIndex = 4;
for (int i = 0; i < field.getNumDefs(); i++, hashCompIndex++)
{
hashComponents[hashCompIndex] = getTypeHash(field.getDef(i));
Expand Down Expand Up @@ -685,7 +710,7 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege

int childTypeHash = getJsonTypeDefinition(field.getDef(0), typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = "ty" + (childTypeIndex + 1);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);
typeDef.put("child", childTypeName);
break;
}
Expand Down Expand Up @@ -727,22 +752,26 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege

int childTypeHash = getJsonTypeDefinition(childField, typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = "ty" + (childTypeIndex + 1);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);
int childTypeID = getTypeID(childField);

JSONObject childJson = new JSONObject();
childJson.put("name", childField.getFieldName());
childJson.put("type", childTypeName);
if (childTypeID > 0)

int flags = childTypeID | childField.getAdditionalFlags();
if (flags > 0)
{
int flags = childTypeID | childField.getAdditionalFlags();
childJson.put("flags", flags);
}

if (childField.getFieldType() == FieldType.DATASET)
{
char delim = 0x0001;
childJson.put("xpath", childField.getFieldName() + delim + "Row");
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Row");
}
else if (childField.getFieldType() == FieldType.SET)
{
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Item");
}

fields.put(childJson);
Expand All @@ -758,6 +787,17 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
}
}

if (field.isNonStandardInt())
{
FieldDef nonKeyedField = new FieldDef(field);
nonKeyedField.setSourceType(HpccSrcType.LITTLE_ENDIAN);

int childTypeHash = getJsonTypeDefinition(nonKeyedField, typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);
typeDef.put("child", childTypeName);
}

int newTypeIndex = typeDefinitions.size();
typeDefinitions.add(typeDef);
typeDefinitionMap.put(typeHash, newTypeIndex);
Expand Down Expand Up @@ -935,10 +975,6 @@ private static FieldDef parseJsonTypeDefinition(JSONObject jsonTypeDefinitions,
{
FieldDef fd = new FieldDef("", fieldType, fieldType.description(), length,
isFixedLength(typeID), isUnsigned(typeID), getSourceType(typeID), new FieldDef[0]);
if ((typeID & TYPE_ID_MASK) == type_keyedint)
{
fd.setIsBiased(true);
}
return fd;
}
}
Expand Down
23 changes: 23 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,24 @@ public class TestFieldDefinitions
+ "\"type\": \"ty17\",\r\n \"xpath\": \"childdataset\\u0001Row\",\r\n \"flags\": 1044\r\n },\r\n {\r\n \"name\": \"int2set\",\r\n "
+ "\"type\": \"ty19\",\r\n \"xpath\": \"int2set\\u0001Item\",\r\n \"flags\": 1045\r\n }\r\n ]\r\n}";

private static final String allTypesIndexRecordDefinitionStr = "{\r\n \"ty1\": {\r\n \"fieldType\": 1,\r\n \"length\": 8\r\n },\r\n \"ty2\":"
+ " {\r\n \"fieldType\": 10,\r\n \"length\": 8,\r\n \"child\": \"ty1\"\r\n },\r\n \"ty3\": {\r\n \"fieldType\": 257,\r\n \"length\":"
+ " 8\r\n },\r\n \"ty4\": {\r\n \"fieldType\": 266,\r\n \"length\": 8,\r\n \"child\": \"ty3\"\r\n },\r\n \"ty5\": {\r\n \"fieldType\": "
+ "1,\r\n \"length\": 4\r\n },\r\n \"ty6\": {\r\n \"fieldType\": 10,\r\n \"length\": 4,\r\n \"child\": \"ty5\"\r\n },\r\n \"ty7\": {\r\n "
+ " \"fieldType\": 257,\r\n \"length\": 4\r\n },\r\n \"ty8\": {\r\n \"fieldType\": 266,\r\n \"length\": 4,\r\n \"child\": \"ty7\"\r\n },\r\n"
+ " \"ty9\": {\r\n \"fieldType\": 1,\r\n \"length\": 2\r\n },\r\n \"ty10\": {\r\n \"fieldType\": 10,\r\n \"length\": 2,\r\n \"child\": "
+ "\"ty9\"\r\n },\r\n \"ty11\": {\r\n \"fieldType\": 257,\r\n \"length\": 2\r\n },\r\n \"ty12\": {\r\n \"fieldType\": 266,\r\n \"length\": "
+ "2,\r\n \"child\": \"ty11\"\r\n },\r\n \"ty13\": {\r\n \"fieldType\": 259,\r\n \"length\": 524304\r\n },\r\n \"ty14\": {\r\n \"fieldType\":"
+ " 4,\r\n \"length\": 8\r\n },\r\n \"ty15\": {\r\n \"fieldType\": 0,\r\n \"length\": 1\r\n },\r\n \"ty16\": {\r\n \"fieldType\": 285,\r\n "
+ "\"vinit\": 2,\r\n \"length\": 8\r\n },\r\n \"fieldType\": 13,\r\n \"length\": 61,\r\n \"fields\": [\r\n {\r\n \"name\": \"int8\",\r\n \"type\": \"ty2\",\r\n"
+ " \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint8\",\r\n \"type\": \"ty4\",\r\n \"flags\": 266\r\n },\r\n {\r\n \"name\": \"int4\","
+ "\r\n \"type\": \"ty6\",\r\n \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint4\",\r\n \"type\": \"ty8\",\r\n \"flags\": 266\r\n },\r\n"
+ " {\r\n \"name\": \"int2\",\r\n \"type\": \"ty10\",\r\n \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint2\",\r\n \"type\": \"ty12\",\r\n"
+ " \"flags\": 266\r\n },\r\n {\r\n \"name\": \"udec16\",\r\n \"type\": \"ty13\",\r\n \"flags\": 259\r\n },\r\n {\r\n \"name\": \"fixstr8\",\r\n"
+ " \"type\": \"ty14\",\r\n \"flags\": 4\r\n },\r\n {\r\n \"name\": \"recptr\",\r\n \"type\": \"ty4\",\r\n \"flags\": 266\r\n },\r\n {\r\n"
+ " \"name\": \"isactive\",\r\n \"type\": \"ty15\",\r\n \"flags\": 65536\r\n },\r\n {\r\n \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n"
+ " \"flags\": 65821\r\n }\r\n ]\r\n}";

/**
* Gets the complex record definition json.
*
Expand All @@ -56,6 +74,11 @@ public static String getComplexRecordDefinitionJson()
return complexRecordDefinitionStr;
}

public static String getAllTypesIndexRecordDefinitionJson()
{
return allTypesIndexRecordDefinitionStr;
}

/**
* Gets the complex record definition.
*
Expand Down
42 changes: 23 additions & 19 deletions commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,30 +38,34 @@ public void setup()
@Test
public void testJsonRecordParsing() throws Exception
{
String jsonRecDefStr = TestFieldDefinitions.getComplexRecordDefinitionJson();
try
String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };
for (String recordDefStr : recordDefStrs)
{
JSONObject jsonRecDef = new JSONObject(jsonRecDefStr);
FieldDef recDef = RecordDefinitionTranslator.parseJsonRecordDefinition(jsonRecDef);
try
{
JSONObject expectedRefDef = new JSONObject(recordDefStr);

JSONObject tJsonRecDef = RecordDefinitionTranslator.toJsonRecord(recDef);
FieldDef recDef = RecordDefinitionTranslator.parseJsonRecordDefinition(expectedRefDef);
JSONObject actualRecDef = RecordDefinitionTranslator.toJsonRecord(recDef);

if (tJsonRecDef.equals(jsonRecDef))
// Use similar instead of equals because the order of fields in the JSON object may differ
if (!actualRecDef.similar(expectedRefDef))
{
System.out.println("Expected Record Def: " + expectedRefDef + "\n");
System.out.println("Actual Record Def: " + actualRecDef + "\n");
Assert.fail("Translated JSON record definition differs from original");
}
}
catch (UnparsableContentException e)
{
System.out.println(jsonRecDef + "\n");
System.out.println(tJsonRecDef + "\n");
Assert.fail("Translated JSON record definition differs from original");
Assert.fail("Encountered invalid record definition: '" + recordDefStr + "'");
}
catch (Exception e)
{
System.out.println(e.getMessage());
Assert.fail("Exception occurred: '" + e.getMessage() + "'");
}
}
catch (UnparsableContentException e)
{
Assert.fail("Encountered invalid record definition: '" + jsonRecDefStr + "'");
}
catch (Exception e)
{
System.out.println(e.getMessage());
Assert.fail("Exception occurred: '" + e.getMessage() + "'");
}

}
Expand Down
Loading

0 comments on commit a5bc1c2

Please sign in to comment.