diff --git a/lib/connection/result/column.js b/lib/connection/result/column.js
index 034a67016..2434bc499 100644
--- a/lib/connection/result/column.js
+++ b/lib/connection/result/column.js
@@ -545,29 +545,23 @@ function convertRawTimestampHelper(
*/
function convertRawVariant(rawColumnValue, column, context)
{
- var ret;
+ let ret;
// if the input is a non-empty string, convert it to a json object
- if (Util.string.isNotNullOrEmpty(rawColumnValue))
- {
- try
- {
+ if (Util.string.isNotNullOrEmpty(rawColumnValue)) {
+ try {
ret = GlobalConfig.jsonColumnVariantParser(rawColumnValue);
}
- catch (jsonParseError)
- {
- try
- {
+ catch (jsonParseError) {
+ try {
ret = GlobalConfig.xmlColumnVariantParser(rawColumnValue);
}
- catch (xmlParseError)
- {
+ catch (xmlParseError) {
Logger.getInstance().debug("Variant cannot be parsed neither as JSON: %s nor as XML: %s", jsonParseError.message, xmlParseError.message);
throw new Errors.VariantParseError(jsonParseError, xmlParseError);
}
}
}
-
return ret;
}
diff --git a/lib/core.js b/lib/core.js
index 92399403c..9dbe66559 100644
--- a/lib/core.js
+++ b/lib/core.js
@@ -207,14 +207,17 @@ function Core(options)
GlobalConfig.setJsonColumnVariantParser(jsonColumnVariantParser);
}
- let xmlColumnVariantParser = options.xmlColumnVariantParser;
- if (Util.exists(xmlColumnVariantParser))
- {
+ const xmlColumnVariantParser = options.xmlColumnVariantParser;
+ const xmlParserConfig = options.xmlParserConfig;
+ if (Util.exists(xmlColumnVariantParser)) {
Errors.checkArgumentValid(Util.isFunction(xmlColumnVariantParser),
ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_XML_PARSER);
GlobalConfig.setXmlColumnVariantParser(xmlColumnVariantParser);
}
+ else if (Util.exists(xmlParserConfig)) {
+ GlobalConfig.createXmlColumnVariantParserWithParameters(xmlParserConfig);
+ }
}
};
diff --git a/lib/global_config.js b/lib/global_config.js
index 7784ee7a0..a413eb51c 100644
--- a/lib/global_config.js
+++ b/lib/global_config.js
@@ -180,41 +180,75 @@ exports.jsonColumnVariantParser = rawColumnValue => vm.runInContext("(" + rawCol
*
* @param {function: (rawColumnValue: string) => any} value
*/
-exports.setJsonColumnVariantParser = function (value)
-{
+exports.setJsonColumnVariantParser = function (value){
// validate input
Errors.assertInternal(Util.isFunction(value));
exports.jsonColumnVariantParser = value;
};
-// The default XML parser
-exports.xmlColumnVariantParser = rawColumnValue =>
-{
- // check if raw string is in XML format
- // ensure each tag is enclosed and all attributes and elements are valid
- // XMLValidator.validate returns true if valid, returns an error if invalid
- var validateResult = XMLValidator.validate(rawColumnValue);
- if (validateResult === true)
- {
- // use XML parser
- return new XMLParser().parse(rawColumnValue);
- }
- else
- {
- throw new Error(validateResult.err.msg);
- }
+/**
+ * As a default we set parameters values identical like in fast-xml-parser lib defaults
+ * thus preserving backward compatibility if customer doesn't set custom configuration
+ */
+const defaultXmlParserConfiguration = {
+ ignoreAttributes: true,
+ alwaysCreateTextNode: false,
+ attributeNamePrefix: '@',
+ attributesGroupName: null
};
+// The default XML parser
+exports.xmlColumnVariantParser = getXmlColumnVariantParser(defaultXmlParserConfiguration);
+
/**
* Updates the value of the 'xmlColumnVariantParser' parameter.
+ * Return custom XmlParser configuration or default if not exists.
*
* @param {function: (rawColumnValue: string) => any} value
*/
-exports.setXmlColumnVariantParser = function (value)
-{
+exports.setXmlColumnVariantParser = function (value){
// validate input
Errors.assertInternal(Util.isFunction(value));
exports.xmlColumnVariantParser = value;
};
+/**
+ * Create and update the 'xmlColumnVariantParser' parameter using custom parser configuration.
+ *
+ * @param {function: (rawColumnValue: string) => any} params
+ */
+exports.createXmlColumnVariantParserWithParameters = function (params){
+ exports.xmlColumnVariantParser = getXmlColumnVariantParser(params);
+};
+
+/**
+ * Create XMlParser with custom configuration.
+ * Parametrs that you can override:
+ * ignoreAttributes - default true,
+ * attributeNamePrefix - default '@',
+ * attributesGroupName - default null,
+ * alwaysCreateTextNode - default false
+ *
+ * @param {object} config
+ */
+function getXmlColumnVariantParser(config) {
+ const parserConfiguration = {
+ ignoreAttributes: config && config.ignoreAttributes !== undefined && config.ignoreAttributes !== null ? config.ignoreAttributes : defaultXmlParserConfiguration.ignoreAttributes,
+ attributeNamePrefix: config && config.attributeNamePrefix !== undefined && config.attributeNamePrefix !== null ? config.attributeNamePrefix : defaultXmlParserConfiguration.attributeNamePrefix,
+ attributesGroupName: config && config.attributesGroupName !== undefined && config.attributesGroupName !== null ? config.attributesGroupName : defaultXmlParserConfiguration.attributesGroupName,
+ alwaysCreateTextNode: config && config.alwaysCreateTextNode !== undefined && config.alwaysCreateTextNode !== null ? config.alwaysCreateTextNode : defaultXmlParserConfiguration.alwaysCreateTextNode,
+ };
+ return rawColumnValue => {
+ // check if raw string is in XML format
+ // ensure each tag is enclosed and all attributes and elements are valid
+ // XMLValidator.validate returns true if valid, returns an error if invalid
+ const validateResult = XMLValidator.validate(rawColumnValue);
+ if (validateResult === true) {
+ // use XML parser
+ return new XMLParser(parserConfiguration).parse(rawColumnValue);
+ } else {
+ throw new Error(validateResult.err.msg);
+ }
+ };
+}
diff --git a/test/integration/testExecute.js b/test/integration/testExecute.js
index 300a68dd0..92bd5396e 100644
--- a/test/integration/testExecute.js
+++ b/test/integration/testExecute.js
@@ -8,6 +8,8 @@ var connOption = require('./connectionOptions').valid;
var testUtil = require('./testUtil');
var fs = require('fs');
var tmp = require('tmp');
+const globalConfig = require("../../lib/global_config");
+
describe('Execute test', function ()
{
@@ -163,166 +165,89 @@ describe('Execute test', function ()
});
});
-describe('Execute test - variant', function ()
-{
+describe('Execute test - variant', function () {
this.timeout(100000);
- var connection;
+ let connection;
const DATABASE_NAME = connOption.database;
const SCHEMA_NAME = connOption.schema;
- const TEST_VARIANT_TABLE = "TEST_VARIANT_TABLE";
- const TEST_VARIANT_STAGE = "TEST_VARIANT_STAGE";
- const TEST_VARIANT_FORMAT = "TEST_VARIANT_FORMAT";
- const TEST_COL = "COL";
- const TEST_HEADER = "ROOT";
- const TEST_XML_VAL = 123;
- const TEST_JSON_VAL = "<123>";
+ const TEST_VARIANT_TABLE = 'TEST_VARIANT_TABLE';
+ const TEST_VARIANT_STAGE = 'TEST_VARIANT_STAGE';
+ const TEST_VARIANT_FORMAT = 'TEST_VARIANT_FORMAT';
+ const TEST_COL = 'COL';
+ const TEST_HEADER = 'ROOT';
const createTableVariant = `create or replace table ${TEST_VARIANT_TABLE}(${TEST_COL} variant)`;
+ const truncateTableVariant = `truncate table ${TEST_VARIANT_TABLE}`;
const createStageVariant = `CREATE OR REPLACE STAGE ${TEST_VARIANT_STAGE} FILE_FORMAT = ${TEST_VARIANT_FORMAT}`;
const copyIntoVariant = `COPY INTO ${TEST_VARIANT_TABLE} FROM @${DATABASE_NAME}.${SCHEMA_NAME}.${TEST_VARIANT_STAGE}`;
- const selectVariant = `select ${TEST_COL} from ${TEST_VARIANT_TABLE}`;
+ const selectVariant = `select ${TEST_COL}
+ from ${TEST_VARIANT_TABLE}`;
const dropStageVariant = `drop table if exists ${TEST_VARIANT_STAGE}`;
const dropTableVariant = `drop table if exists ${TEST_VARIANT_TABLE}`;
- before(function (done)
- {
+ before(function (done) {
connection = testUtil.createConnection();
async.series([
- function (callback)
- {
+ function (callback) {
testUtil.connect(connection, callback);
}],
done
);
});
-
- after(function (done)
- {
- async.series([
- function (callback)
- {
- testUtil.destroyConnection(connection, callback);
- }],
- done
- );
- });
-
- var testCases =
- [
- {
- name: 'raw xml',
- type: 'XML',
- fileExtension: '.xml',
- },
- {
- name: 'raw json',
- type: 'JSON',
- fileExtension: '.json',
- }
- ];
-
- var createItCallback = function (testCase)
- {
- return function (done)
- {
+ var createItCallback = function (testCase, rowAsserts) {
+ return function (done) {
{
- var createFileFormatVariant = `CREATE OR REPLACE FILE FORMAT ${TEST_VARIANT_FORMAT} TYPE = ${testCase.type}`;
-
- var sampleData;
- if (testCase.type == 'XML')
- {
- sampleData = `<${TEST_HEADER}>${TEST_XML_VAL}${TEST_HEADER}>`;
- }
- else if (testCase.type == 'JSON')
- {
- sampleData = `{${TEST_HEADER}: \"${TEST_JSON_VAL}\"}`;
- }
+ globalConfig.createXmlColumnVariantParserWithParameters({ignoreAttributes: testCase.ignoreAttributes,
+ attributeNamePrefix: testCase.attributeNamePrefix});
- var sampleTempFile = tmp.fileSync({ postfix: testCase.fileExtension });
- fs.writeFileSync(sampleTempFile.name, sampleData);
+ let sampleTempFile = tmp.fileSync({postfix: testCase.fileExtension});
+ fs.writeFileSync(sampleTempFile.name, testCase.sampleData);
var putVariant = `PUT file://${sampleTempFile.name} @${DATABASE_NAME}.${SCHEMA_NAME}.${TEST_VARIANT_STAGE}`;
// Windows user contains a '~' in the path which causes an error
- if (process.platform == "win32")
- {
+ if (process.platform == "win32") {
var fileName = sampleTempFile.name.substring(sampleTempFile.name.lastIndexOf('\\'));
putVariant = `PUT file://${process.env.USERPROFILE}\\AppData\\Local\\Temp\\${fileName} @${DATABASE_NAME}.${SCHEMA_NAME}.${TEST_VARIANT_STAGE}`;
}
async.series(
[
- function (callback)
- {
- // Create variant table
- testUtil.executeCmd(connection, createTableVariant, callback);
- },
- function (callback)
- {
- // Create variant file format
- testUtil.executeCmd(connection, createFileFormatVariant, callback);
- },
- function (callback)
- {
- // Create stage with variant file format
- testUtil.executeCmd(connection, createStageVariant, callback);
- },
- function (callback)
- {
+ function (callback) {
// Upload sample file
testUtil.executeCmd(connection, putVariant, callback);
},
- function (callback)
- {
+ function (callback) {
// Load sample file
testUtil.executeCmd(connection, copyIntoVariant, callback);
},
- function (callback)
- {
+ function (callback) {
// Select values from table
connection.execute({
sqlText: selectVariant,
- complete: function (err, stmt, rows)
- {
+ complete: function (err, stmt, rows) {
var stream = stmt.streamRows();
- stream.on('error', function (err)
- {
+ stream.on('error', function (err) {
done(err);
});
- stream.on('data', function (row)
- {
+ // stream.on('data', rowAsserts);
+
+ stream.on('data', function (row) {
// Check the column, header, and value is correct
- if (testCase.type == 'XML')
- {
- assert.strictEqual(row[TEST_COL][TEST_HEADER], TEST_XML_VAL);
- }
- else if (testCase.type == 'JSON')
- {
- assert.strictEqual(row[TEST_COL][TEST_HEADER], TEST_JSON_VAL);
- }
+ rowAsserts(testCase, row);
});
- stream.on('end', function ()
- {
+
+
+ stream.on('end', function () {
callback();
});
}
});
},
- function (callback)
- {
- // Drop stage
- testUtil.executeCmd(connection, dropStageVariant, callback);
- },
- function (callback)
- {
- // Drop table
- testUtil.executeCmd(connection, dropTableVariant, callback);
- },
- function (callback)
- {
+ function (callback) {
// Delete temp files
fs.closeSync(sampleTempFile.fd);
fs.unlinkSync(sampleTempFile.name);
@@ -330,13 +255,159 @@ describe('Execute test - variant', function ()
}],
done
);
- };
+ }
+ ;
};
};
- for (var index = 0; index < testCases.length; index++)
- {
- var testCase = testCases[index];
- it(testCase.name, createItCallback(testCase));
- }
+ before(async () => {
+ connection = testUtil.createConnection();
+ await testUtil.connectAsync(connection);
+ await testUtil.executeCmdAsync(connection, createTableVariant);
+ });
+
+ after(async () => {
+ // Drop table
+ await testUtil.executeCmdAsync(connection, dropTableVariant);
+ });
+
+ describe('Variant XML', function () {
+ const TEST_ATTRIBUTE_NAME = "attr";
+ const TEST_ATTRIBUTE_VALUE = "attrValue";
+ const ELEMENT_VALUE_FIELD = "#text";
+ const TEST_XML_VAL = 123;
+
+ before(async () => {
+ let createFileFormatVariant = `CREATE OR REPLACE FILE FORMAT ${TEST_VARIANT_FORMAT} TYPE = XML`;
+ await testUtil.executeCmdAsync(connection, createFileFormatVariant);
+ });
+
+ beforeEach(async () => {
+ await testUtil.executeCmdAsync(connection, createStageVariant);
+ });
+
+ afterEach(async () => {
+ await testUtil.executeCmdAsync(connection, truncateTableVariant);
+ await testUtil.executeCmdAsync(connection, dropStageVariant);
+ });
+
+ const testCases =
+ [
+ {
+ name: 'xml_single_element',
+ type: 'XML',
+ fileExtension: '.xml',
+ sampleData: `<${TEST_HEADER}>${TEST_XML_VAL}${TEST_HEADER}>`
+ },
+ {
+ name: 'xml_single_element_with_attribute',
+ type: 'XML',
+ fileExtension: '.xml',
+ sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}>${TEST_XML_VAL}${TEST_HEADER}>`,
+ ignoreAttributes: false
+ },
+ {
+ name: 'xml_with_attribute_and_custom_parser',
+ type: 'XML',
+ fileExtension: '.xml',
+ sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}>${TEST_XML_VAL}${TEST_HEADER}>`,
+ ignoreAttributes: false,
+ attributeNamePrefix: '##'
+ },
+ {
+ name: 'xml_skip_attributes',
+ type: 'XML',
+ fileExtension: '.xml',
+ sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}>${TEST_XML_VAL}${TEST_HEADER}>`,
+ attributeNamePrefix: '##'
+ },
+ {
+ name: 'xml_attribute_only',
+ type: 'XML',
+ fileExtension: '.xml',
+ sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}>${TEST_HEADER}>`,
+ ignoreAttributes: false,
+ attributeNamePrefix: '##'
+
+ },
+ {
+ name: 'xml_attribute_empty_value',
+ type: 'XML',
+ fileExtension: '.xml',
+ sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}/>`,
+ ignoreAttributes: false,
+ attributeNamePrefix: '##'
+ }
+ ];
+
+ const rowAsserts = (testCase, row) => {
+ // Check the column, header, and value is correct
+ const attributeWithPrefix = testCase.attributeNamePrefix + TEST_ATTRIBUTE_NAME;
+
+ if (testCase.name === 'xml_single_element') {
+ assert.strictEqual(row[TEST_COL][TEST_HEADER], TEST_XML_VAL);
+ } else if (testCase.name === 'xml_single_element_with_attribute') {
+ assert.strictEqual(row[TEST_COL][TEST_HEADER][ELEMENT_VALUE_FIELD], TEST_XML_VAL);
+ } else if (testCase.name === 'xml_with_attribute_and_custom_parser') {
+ assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][ELEMENT_VALUE_FIELD], TEST_XML_VAL);
+ assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][attributeWithPrefix], TEST_ATTRIBUTE_VALUE);
+ } else if (testCase.name === 'xml_skip_attributes') {
+ assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER], TEST_XML_VAL);
+ assert.equal(row[TEST_COL]['node'][TEST_HEADER][testCase.attributeNamePrefix + TEST_ATTRIBUTE_NAME], undefined);
+ } else if (testCase.name === 'xml_attribute_only') {
+ assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][ELEMENT_VALUE_FIELD], undefined);
+ assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][testCase.attributeNamePrefix + TEST_ATTRIBUTE_NAME], TEST_ATTRIBUTE_VALUE);
+ } else if (testCase.name === 'xml_attribute_empty_value') {
+ assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][ELEMENT_VALUE_FIELD], undefined);
+ assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][testCase.attributeNamePrefix + TEST_ATTRIBUTE_NAME], TEST_ATTRIBUTE_VALUE);
+ }
+ };
+
+ for (let index = 0; index < testCases.length; index++) {
+ const testCase = testCases[index];
+ it(testCase.name, createItCallback(testCase, rowAsserts));
+ }
+ });
+
+ describe('Variant JSON', function () {
+ const TEST_JSON_VAL = '<123>';
+
+ before(async () => {
+ const createFileFormatVariant = `CREATE OR REPLACE FILE FORMAT ${TEST_VARIANT_FORMAT} TYPE = JSON`;
+ await testUtil.executeCmdAsync(connection, createFileFormatVariant);
+ });
+ //
+ beforeEach(async () => {
+ // Drop stage
+ await testUtil.executeCmdAsync(connection, createStageVariant);
+
+ });
+ afterEach(async () => {
+ // Drop stage
+ await testUtil.executeCmdAsync(connection, truncateTableVariant);
+ await testUtil.executeCmdAsync(connection, dropStageVariant);
+ });
+
+ const testCases =
+ [
+ {
+ name: 'raw_json',
+ type: 'JSON',
+ fileExtension: '.json',
+ sampleData: `{${TEST_HEADER}: \"${TEST_JSON_VAL}\"}`
+ }
+ ];
+
+ const rowAsserts = (testCase, row) => {
+ // Check the column, header, and value is correct
+ if (testCase.name === 'raw_json') {
+ assert.strictEqual(row[TEST_COL][TEST_HEADER], TEST_JSON_VAL);
+ }
+ };
+
+ for (let index = 0; index < testCases.length; index++) {
+ const testCase = testCases[index];
+ it(testCase.name, createItCallback(testCase, rowAsserts));
+ }
+ });
});