Skip to content

Commit

Permalink
SNOW-1553678 structured types support (#874)
Browse files Browse the repository at this point in the history
  • Loading branch information
@sfc-gh-pmotacki
sfc-gh-pmotacki authored Oct 1, 2024
1 parent 852017f commit b5e084f
Show file tree
Hide file tree
Showing 11 changed files with 1,362 additions and 68 deletions.
5 changes: 5 additions & 0 deletions index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -824,6 +824,11 @@ declare module 'snowflake-sdk' {
*/
isArray(): boolean;

/**
* Returns true if this column is type MAP.
*/
isMap(): boolean;

/**
* Returns the value of this column in a row.
*/
Expand Down
249 changes: 240 additions & 9 deletions lib/connection/result/column.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,11 @@ const Logger = require('../../logger');
const SfTimestamp = require('./sf_timestamp');
const DataTypes = require('./data_types');
const SqlTypes = require('./data_types').SqlTypes;
const dateTimeFormatConverter = require('./datetime_format_converter');
const bigInt = require('big-integer');
const moment = require('moment');
const momentTimezone = require('moment-timezone');
const util = require('../../util');

/**
* Creates a new Column.
Expand All @@ -28,6 +32,7 @@ function Column(options, index, statementParameters, resultVersion) {
const scale = options.scale;
const type = options.type;
const precision = options.precision;
const fieldsMetadata = options.fields;

/**
* Returns the name of this column.
Expand Down Expand Up @@ -104,9 +109,10 @@ function Column(options, index, statementParameters, resultVersion) {
this.isTimestampLtz = createFnIsColumnOfType(type, SqlTypes.isTimestampLtz, SqlTypes);
this.isTimestampNtz = createFnIsColumnOfType(type, SqlTypes.isTimestampNtz, SqlTypes);
this.isTimestampTz = createFnIsColumnOfType(type, SqlTypes.isTimestampTz, SqlTypes);
this.isVariant = createFnIsColumnOfType(type, SqlTypes.isVariant, SqlTypes);
this.isObject = createFnIsColumnOfType(type, SqlTypes.isObject, SqlTypes);
this.isArray = createFnIsColumnOfType(type, SqlTypes.isArray, SqlTypes);
this.isVariant = createFnIsColumnOfType(type, (type) => SqlTypes.isVariant(type, fieldsMetadata), SqlTypes);
this.isObject = createFnIsColumnOfType(type, (type) => SqlTypes.isObject(type, fieldsMetadata), SqlTypes);
this.isArray = createFnIsColumnOfType(type, (type) => SqlTypes.isArray(type, fieldsMetadata), SqlTypes);
this.isMap = createFnIsColumnOfType(type, (type) => SqlTypes.isMap(type, fieldsMetadata), SqlTypes);

let convert;
let toString;
Expand Down Expand Up @@ -167,7 +173,16 @@ function Column(options, index, statementParameters, resultVersion) {
format = statementParameters['BINARY_OUTPUT_FORMAT'];
} else if (this.isVariant()) {
convert = convertRawVariant;
toString = toStringFromVariant;
toString = toStringFromRawValue;
} else if (this.isObject()) {
convert = convertRawStructuredType(convertJsonObject);
toString = toStringFromRawValue;
} else if (this.isArray()) {
convert = convertRawStructuredType(convertJsonArray);
toString = toStringFromRawValue;
} else if (this.isMap()) {
convert = convertRawStructuredType(convertJsonMap);
toString = toStringFromRawValue;
} else {
// column is of type string, so leave value as is
convert = noop;
Expand All @@ -183,7 +198,8 @@ function Column(options, index, statementParameters, resultVersion) {
toString: toString,
format: format,
resultVersion: resultVersion,
statementParameters: statementParameters
statementParameters: statementParameters,
fieldsMetadata: fieldsMetadata
};

/**
Expand Down Expand Up @@ -268,15 +284,230 @@ function convertRawBigInt(rawColumnValue) {
function convertRawBoolean(rawColumnValue) {
let ret;

if ((rawColumnValue === '1') || (rawColumnValue === 'TRUE')) {
if (rawColumnValue === true || (rawColumnValue === '1') || (rawColumnValue.toUpperCase() === 'TRUE')) {
ret = true;
} else if ((rawColumnValue === '0') || (rawColumnValue === 'FALSE')) {
} else if (rawColumnValue === false || (rawColumnValue === '0') || (rawColumnValue.toUpperCase() === 'FALSE')) {
ret = false;
} else {
throw new Error(`Value could not be converted to boolean: ${rawColumnValue}`);
}

return ret;
}

/**
* Converts a raw column value of structured type object to javascript Object
*
* @param {Object} json
* @param {Object} context
*
* @returns {Object}
*/
function convertJsonObject(json, context) {
if (context.fieldsMetadata){
context.fieldsMetadata = context.fieldsMetadata.reduce(function (map, obj) {
map[obj.name] = obj;
return map;
}, {});

const result = {};
Object.keys(json).forEach(function (key) {
const fieldMetadata = context.fieldsMetadata[key];
result[key] = mapStructuredTypeValue(json[key], context, fieldMetadata);
});
return result;
} else {
return json;
}
}

/**
* Converts a raw column value of structured type array to javascript Object
*
* @param {Object} json
* @param {Object} context
*
* @returns {Object}
*/
function convertJsonArray(json, context) {
if (context.fieldsMetadata) {
const result = [];
json.forEach(function (value) {
result.push(mapStructuredTypeValue(value, context, context.fieldsMetadata[0]));
});
return result;
} else {
return json;
}
}

/**
* Converts a raw column value of structured type map to javascript Object
*
* @param {Object} json
* @param {Object} context
*
* @returns {Object}
*/
function convertJsonMap(json, context) {
if (Array.isArray(context.fieldsMetadata) && context.fieldsMetadata.length === 2) {
const result = new Map;
const keyMetadata = context.fieldsMetadata[0];
const valueMetadata = context.fieldsMetadata[1];
Object.keys(json).forEach(function (key) {
const convertedKey = mapStructuredTypeValue(key, context, keyMetadata);
const convertedValue = mapStructuredTypeValue(json[key], context, valueMetadata);
result.set(convertedKey, convertedValue);
});
return result;
} else {
return json;
}
}

/**
* Converts a raw column value of structured type OBJECT to javascript Object
*
* @param {String} rawColumnValue
* @param {Object} column
* @param {Object} context
*
* @returns {Object}
*/
const convertRawStructuredType = (convertJsonFn) => (rawColumnValue, column, context) => {
if (Util.string.isNotNullOrEmpty(rawColumnValue)) {
try {
const json = JSON.parse(rawColumnValue);
return convertJsonFn(json, context);
} catch (jsonParseError) {
Logger.getInstance().debug('Column %s raw value cannot be parsed as JSON: %s ', column.name, jsonParseError.message);
throw new Error(util.format('Column [%s] raw value cannot be parsed as JSON: %s ', column.name, jsonParseError.message));
}
} else {
throw new Error(util.format('Column %s raw value is null or empty ', column.name));
}
};

function mapStructuredTypeValue(columnValue, context, metadataField) {
const formatLtz = context.statementParameters['TIMESTAMP_LTZ_OUTPUT_FORMAT'] ?? context.statementParameters['TIMESTAMP_OUTPUT_FORMAT'];
const formatTz = context.statementParameters['TIMESTAMP_TZ_OUTPUT_FORMAT'] ?? context.statementParameters['TIMESTAMP_OUTPUT_FORMAT'];
const formatNtz = context.statementParameters['TIMESTAMP_NTZ_OUTPUT_FORMAT'];
let value;
switch (metadataField.type) {
case 'text':
value = columnValue;
break;
case 'real':
value = toValueFromNumber(convertRawNumber(columnValue));
break;
case 'fixed':
value = toValueFromNumber(convertRawNumber(columnValue));
break;
case 'boolean':
value = convertRawBoolean(columnValue);
break;
case 'timestamp_ltz':
value = convertTimestampTzString(columnValue, formatLtz, context.statementParameters['TIMEZONE'], metadataField.scale).toSfDate();
break;
case 'timestamp_ntz':
value = convertTimestampNtzString(columnValue, formatNtz, moment.tz.zone('UTC'), metadataField.scale).toSfDate();
break;
case 'timestamp_tz':
value = convertTimestampTzString(columnValue, formatTz, context.statementParameters['TIMEZONE'], metadataField.scale).toSfDate();
break;
case 'date': {
context.format = context.statementParameters['DATE_OUTPUT_FORMAT'];
value = convertDateString(columnValue, context.format );
break;
}
case 'time':
context.format = context.statementParameters['TIME_OUTPUT_FORMAT'];
value = convertTimeString(columnValue, context.format, moment.tz.zone('UTC'), metadataField.scale).toSfTime();
break;
case 'binary':
context.format = context.statementParameters['BINARY_OUTPUT_FORMAT'];
value = convertRawBinary(columnValue, this, context).toJSON().data;
break;
case 'object': {
const internalContext = {
convert: convertRawStructuredType(convertJsonObject),
toValue: noop,
toString: toString,
format: toStringFromRawValue,
resultVersion: context.resultVersion,
statementParameters: context.statementParameters,
fieldsMetadata: metadataField.fields
};
value = convertJsonObject(columnValue, internalContext);
break;
}
case 'array': {
const internalArrayContext = {
convert: convertRawStructuredType(convertJsonArray),
toValue: noop,
toString: toString,
format: toStringFromRawValue,
resultVersion: context.resultVersion,
statementParameters: context.statementParameters,
fieldsMetadata: metadataField.fields
};
value = convertJsonArray(columnValue, internalArrayContext);
break;
}
case 'map': {
const internalMapContext = {
convert: convertRawStructuredType(convertJsonMap),
toValue: noop,
toString: toString,
format: toStringFromRawValue,
resultVersion: context.resultVersion,
statementParameters: context.statementParameters,
fieldsMetadata: metadataField.fields
};
value = convertJsonMap(columnValue, internalMapContext);
break;
}
default:
Logger.getInstance().info(`Column type not supported: ${context.fieldsMetadata.type}`);
throw new Error(`Column type not supported: ${context.fieldsMetadata.type}`);
}
return value;
}

const convertTimestampTzString = function (stringValue, formatSql, timezone, scale) {
const formatMoment = dateTimeFormatConverter.convertSnowflakeFormatToMomentFormat(formatSql, scale);
const epochSeconds = momentTimezone(stringValue, formatMoment).unix();
return new SfTimestamp(epochSeconds, 0, scale, timezone, formatSql);
};

const convertTimestampNtzString = function (stringValue, formatSql, timezone, scale) {
const formatMoment = dateTimeFormatConverter.convertSnowflakeFormatToMomentFormat(formatSql, scale);
const epochSeconds = momentTimezone.utc(stringValue, formatMoment).unix();
return new SfTimestamp(epochSeconds, 0, scale, timezone, formatSql);
};

const convertDateString = function (stringValue, formatSql) {
const formatMoment = dateTimeFormatConverter.convertSnowflakeFormatToMomentFormat(formatSql, 0);
const epochSeconds = momentTimezone.utc(stringValue, formatMoment).unix();
const date = new SfTimestamp(
epochSeconds, // convert to seconds
0, // no nano seconds
0, // no scale required
'UTC', // use utc as the timezone
context.format);
date._valueAsString = stringValue;
return date.toSfDate();
};


const convertTimeString = function (stringValue, formatSql, timezone, scale) {
const formatMoment = dateTimeFormatConverter.convertSnowflakeFormatToMomentFormat(formatSql, scale);
const moment = momentTimezone(stringValue, formatMoment);
const epochSeconds = moment.hours() * 3600 + moment.minutes() * 60 + moment.seconds();
const time = new SfTimestamp(epochSeconds, 0, scale, timezone, formatSql);
time._valueAsString = stringValue;
return time;
};

/**
* Converts a raw column value of type Date to a Snowflake Date.
*
Expand Down Expand Up @@ -627,7 +858,7 @@ function toStringFromTimestamp(columnValue) {
*
* @returns {String}
*/
function toStringFromVariant(columnValue) {
function toStringFromRawValue(columnValue) {
return (columnValue !== null) ? JSON.stringify(columnValue) : DataTypes.getNullValue();
}

Expand Down
39 changes: 28 additions & 11 deletions lib/connection/result/data_types.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ const sqlTypes =
TIMESTAMP_TZ: 'timestamp_tz',
VARIANT: 'variant',
OBJECT: 'object',
ARRAY: 'array'
ARRAY: 'array',
MAP: 'map'
},

/**
Expand Down Expand Up @@ -143,10 +144,11 @@ const sqlTypes =
*
* @returns {Boolean}
*/
isVariant: function (sqlType) {
isVariant: function (sqlType, fieldsMetadata) {
return (sqlType === this.values.VARIANT) ||
(sqlType === this.values.OBJECT) ||
(sqlType === this.values.ARRAY);
(sqlType === this.values.OBJECT && fieldsMetadata == null) ||
(sqlType === this.values.ARRAY && fieldsMetadata == null) ||
(sqlType === this.values.MAP && fieldsMetadata == null);
},

/**
Expand All @@ -156,8 +158,8 @@ const sqlTypes =
*
* @returns {Boolean}
*/
isObject: function (sqlType) {
return (sqlType === this.values.OBJECT);
isObject: function (sqlType, fieldsMetadata) {
return (sqlType === this.values.OBJECT && fieldsMetadata != null);
},

/**
Expand All @@ -167,8 +169,19 @@ const sqlTypes =
*
* @returns {Boolean}
*/
isArray: function (sqlType) {
return (sqlType === this.values.ARRAY);
isArray: function (sqlType, fieldsMetadata) {
return (sqlType === this.values.ARRAY && fieldsMetadata != null);
},

/**
* Determines if a column's SQL type is Map.
*
* @param {Object} sqlType
*
* @returns {Boolean}
*/
isMap: function (sqlType, fieldsMetadata) {
return (sqlType === this.values.MAP && fieldsMetadata != null);
}
};

Expand All @@ -181,7 +194,10 @@ const nativeTypes =
NUMBER: 'NUMBER',
DATE: 'DATE',
JSON: 'JSON',
BUFFER: 'BUFFER'
BUFFER: 'BUFFER',
OBJECT: 'OBJECT',
ARRAY: 'ARRAY',
MAP: 'MAP'
},

/**
Expand Down Expand Up @@ -246,8 +262,9 @@ MAP_SQL_TO_NATIVE[sqlTypeValues.TIMESTAMP_LTZ] = nativeTypeValues.DATE;
MAP_SQL_TO_NATIVE[sqlTypeValues.TIMESTAMP_NTZ] = nativeTypeValues.DATE;
MAP_SQL_TO_NATIVE[sqlTypeValues.TIMESTAMP_TZ] = nativeTypeValues.DATE;
MAP_SQL_TO_NATIVE[sqlTypeValues.VARIANT] = nativeTypeValues.JSON;
MAP_SQL_TO_NATIVE[sqlTypeValues.OBJECT] = nativeTypeValues.JSON;
MAP_SQL_TO_NATIVE[sqlTypeValues.ARRAY] = nativeTypeValues.JSON;
MAP_SQL_TO_NATIVE[sqlTypeValues.OBJECT] = nativeTypeValues.OBJECT;
MAP_SQL_TO_NATIVE[sqlTypeValues.ARRAY] = nativeTypeValues.ARRAY;
MAP_SQL_TO_NATIVE[sqlTypeValues.MAP] = nativeTypeValues.MAP;

exports.SqlTypes = sqlTypes;
exports.NativeTypes = nativeTypes;
Expand Down
Loading

0 comments on commit b5e084f

Please sign in to comment.