diff --git a/lib/connection/result/column.js b/lib/connection/result/column.js index 034a67016..2434bc499 100644 --- a/lib/connection/result/column.js +++ b/lib/connection/result/column.js @@ -545,29 +545,23 @@ function convertRawTimestampHelper( */ function convertRawVariant(rawColumnValue, column, context) { - var ret; + let ret; // if the input is a non-empty string, convert it to a json object - if (Util.string.isNotNullOrEmpty(rawColumnValue)) - { - try - { + if (Util.string.isNotNullOrEmpty(rawColumnValue)) { + try { ret = GlobalConfig.jsonColumnVariantParser(rawColumnValue); } - catch (jsonParseError) - { - try - { + catch (jsonParseError) { + try { ret = GlobalConfig.xmlColumnVariantParser(rawColumnValue); } - catch (xmlParseError) - { + catch (xmlParseError) { Logger.getInstance().debug("Variant cannot be parsed neither as JSON: %s nor as XML: %s", jsonParseError.message, xmlParseError.message); throw new Errors.VariantParseError(jsonParseError, xmlParseError); } } } - return ret; } diff --git a/lib/core.js b/lib/core.js index 92399403c..9dbe66559 100644 --- a/lib/core.js +++ b/lib/core.js @@ -207,14 +207,17 @@ function Core(options) GlobalConfig.setJsonColumnVariantParser(jsonColumnVariantParser); } - let xmlColumnVariantParser = options.xmlColumnVariantParser; - if (Util.exists(xmlColumnVariantParser)) - { + const xmlColumnVariantParser = options.xmlColumnVariantParser; + const xmlParserConfig = options.xmlParserConfig; + if (Util.exists(xmlColumnVariantParser)) { Errors.checkArgumentValid(Util.isFunction(xmlColumnVariantParser), ErrorCodes.ERR_GLOBAL_CONFIGURE_INVALID_XML_PARSER); GlobalConfig.setXmlColumnVariantParser(xmlColumnVariantParser); } + else if (Util.exists(xmlParserConfig)) { + GlobalConfig.createXmlColumnVariantParserWithParameters(xmlParserConfig); + } } }; diff --git a/lib/global_config.js b/lib/global_config.js index 7784ee7a0..a413eb51c 100644 --- a/lib/global_config.js +++ b/lib/global_config.js @@ -180,41 +180,75 @@ exports.jsonColumnVariantParser = rawColumnValue => vm.runInContext("(" + rawCol * * @param {function: (rawColumnValue: string) => any} value */ -exports.setJsonColumnVariantParser = function (value) -{ +exports.setJsonColumnVariantParser = function (value){ // validate input Errors.assertInternal(Util.isFunction(value)); exports.jsonColumnVariantParser = value; }; -// The default XML parser -exports.xmlColumnVariantParser = rawColumnValue => -{ - // check if raw string is in XML format - // ensure each tag is enclosed and all attributes and elements are valid - // XMLValidator.validate returns true if valid, returns an error if invalid - var validateResult = XMLValidator.validate(rawColumnValue); - if (validateResult === true) - { - // use XML parser - return new XMLParser().parse(rawColumnValue); - } - else - { - throw new Error(validateResult.err.msg); - } +/** + * As a default we set parameters values identical like in fast-xml-parser lib defaults + * thus preserving backward compatibility if customer doesn't set custom configuration + */ +const defaultXmlParserConfiguration = { + ignoreAttributes: true, + alwaysCreateTextNode: false, + attributeNamePrefix: '@', + attributesGroupName: null }; +// The default XML parser +exports.xmlColumnVariantParser = getXmlColumnVariantParser(defaultXmlParserConfiguration); + /** * Updates the value of the 'xmlColumnVariantParser' parameter. + * Return custom XmlParser configuration or default if not exists. * * @param {function: (rawColumnValue: string) => any} value */ -exports.setXmlColumnVariantParser = function (value) -{ +exports.setXmlColumnVariantParser = function (value){ // validate input Errors.assertInternal(Util.isFunction(value)); exports.xmlColumnVariantParser = value; }; +/** + * Create and update the 'xmlColumnVariantParser' parameter using custom parser configuration. + * + * @param {function: (rawColumnValue: string) => any} params + */ +exports.createXmlColumnVariantParserWithParameters = function (params){ + exports.xmlColumnVariantParser = getXmlColumnVariantParser(params); +}; + +/** + * Create XMlParser with custom configuration. + * Parametrs that you can override: + * ignoreAttributes - default true, + * attributeNamePrefix - default '@', + * attributesGroupName - default null, + * alwaysCreateTextNode - default false + * + * @param {object} config + */ +function getXmlColumnVariantParser(config) { + const parserConfiguration = { + ignoreAttributes: config && config.ignoreAttributes !== undefined && config.ignoreAttributes !== null ? config.ignoreAttributes : defaultXmlParserConfiguration.ignoreAttributes, + attributeNamePrefix: config && config.attributeNamePrefix !== undefined && config.attributeNamePrefix !== null ? config.attributeNamePrefix : defaultXmlParserConfiguration.attributeNamePrefix, + attributesGroupName: config && config.attributesGroupName !== undefined && config.attributesGroupName !== null ? config.attributesGroupName : defaultXmlParserConfiguration.attributesGroupName, + alwaysCreateTextNode: config && config.alwaysCreateTextNode !== undefined && config.alwaysCreateTextNode !== null ? config.alwaysCreateTextNode : defaultXmlParserConfiguration.alwaysCreateTextNode, + }; + return rawColumnValue => { + // check if raw string is in XML format + // ensure each tag is enclosed and all attributes and elements are valid + // XMLValidator.validate returns true if valid, returns an error if invalid + const validateResult = XMLValidator.validate(rawColumnValue); + if (validateResult === true) { + // use XML parser + return new XMLParser(parserConfiguration).parse(rawColumnValue); + } else { + throw new Error(validateResult.err.msg); + } + }; +} diff --git a/test/integration/testExecute.js b/test/integration/testExecute.js index 300a68dd0..92bd5396e 100644 --- a/test/integration/testExecute.js +++ b/test/integration/testExecute.js @@ -8,6 +8,8 @@ var connOption = require('./connectionOptions').valid; var testUtil = require('./testUtil'); var fs = require('fs'); var tmp = require('tmp'); +const globalConfig = require("../../lib/global_config"); + describe('Execute test', function () { @@ -163,166 +165,89 @@ describe('Execute test', function () }); }); -describe('Execute test - variant', function () -{ +describe('Execute test - variant', function () { this.timeout(100000); - var connection; + let connection; const DATABASE_NAME = connOption.database; const SCHEMA_NAME = connOption.schema; - const TEST_VARIANT_TABLE = "TEST_VARIANT_TABLE"; - const TEST_VARIANT_STAGE = "TEST_VARIANT_STAGE"; - const TEST_VARIANT_FORMAT = "TEST_VARIANT_FORMAT"; - const TEST_COL = "COL"; - const TEST_HEADER = "ROOT"; - const TEST_XML_VAL = 123; - const TEST_JSON_VAL = "<123>"; + const TEST_VARIANT_TABLE = 'TEST_VARIANT_TABLE'; + const TEST_VARIANT_STAGE = 'TEST_VARIANT_STAGE'; + const TEST_VARIANT_FORMAT = 'TEST_VARIANT_FORMAT'; + const TEST_COL = 'COL'; + const TEST_HEADER = 'ROOT'; const createTableVariant = `create or replace table ${TEST_VARIANT_TABLE}(${TEST_COL} variant)`; + const truncateTableVariant = `truncate table ${TEST_VARIANT_TABLE}`; const createStageVariant = `CREATE OR REPLACE STAGE ${TEST_VARIANT_STAGE} FILE_FORMAT = ${TEST_VARIANT_FORMAT}`; const copyIntoVariant = `COPY INTO ${TEST_VARIANT_TABLE} FROM @${DATABASE_NAME}.${SCHEMA_NAME}.${TEST_VARIANT_STAGE}`; - const selectVariant = `select ${TEST_COL} from ${TEST_VARIANT_TABLE}`; + const selectVariant = `select ${TEST_COL} + from ${TEST_VARIANT_TABLE}`; const dropStageVariant = `drop table if exists ${TEST_VARIANT_STAGE}`; const dropTableVariant = `drop table if exists ${TEST_VARIANT_TABLE}`; - before(function (done) - { + before(function (done) { connection = testUtil.createConnection(); async.series([ - function (callback) - { + function (callback) { testUtil.connect(connection, callback); }], done ); }); - - after(function (done) - { - async.series([ - function (callback) - { - testUtil.destroyConnection(connection, callback); - }], - done - ); - }); - - var testCases = - [ - { - name: 'raw xml', - type: 'XML', - fileExtension: '.xml', - }, - { - name: 'raw json', - type: 'JSON', - fileExtension: '.json', - } - ]; - - var createItCallback = function (testCase) - { - return function (done) - { + var createItCallback = function (testCase, rowAsserts) { + return function (done) { { - var createFileFormatVariant = `CREATE OR REPLACE FILE FORMAT ${TEST_VARIANT_FORMAT} TYPE = ${testCase.type}`; - - var sampleData; - if (testCase.type == 'XML') - { - sampleData = `<${TEST_HEADER}>${TEST_XML_VAL}`; - } - else if (testCase.type == 'JSON') - { - sampleData = `{${TEST_HEADER}: \"${TEST_JSON_VAL}\"}`; - } + globalConfig.createXmlColumnVariantParserWithParameters({ignoreAttributes: testCase.ignoreAttributes, + attributeNamePrefix: testCase.attributeNamePrefix}); - var sampleTempFile = tmp.fileSync({ postfix: testCase.fileExtension }); - fs.writeFileSync(sampleTempFile.name, sampleData); + let sampleTempFile = tmp.fileSync({postfix: testCase.fileExtension}); + fs.writeFileSync(sampleTempFile.name, testCase.sampleData); var putVariant = `PUT file://${sampleTempFile.name} @${DATABASE_NAME}.${SCHEMA_NAME}.${TEST_VARIANT_STAGE}`; // Windows user contains a '~' in the path which causes an error - if (process.platform == "win32") - { + if (process.platform == "win32") { var fileName = sampleTempFile.name.substring(sampleTempFile.name.lastIndexOf('\\')); putVariant = `PUT file://${process.env.USERPROFILE}\\AppData\\Local\\Temp\\${fileName} @${DATABASE_NAME}.${SCHEMA_NAME}.${TEST_VARIANT_STAGE}`; } async.series( [ - function (callback) - { - // Create variant table - testUtil.executeCmd(connection, createTableVariant, callback); - }, - function (callback) - { - // Create variant file format - testUtil.executeCmd(connection, createFileFormatVariant, callback); - }, - function (callback) - { - // Create stage with variant file format - testUtil.executeCmd(connection, createStageVariant, callback); - }, - function (callback) - { + function (callback) { // Upload sample file testUtil.executeCmd(connection, putVariant, callback); }, - function (callback) - { + function (callback) { // Load sample file testUtil.executeCmd(connection, copyIntoVariant, callback); }, - function (callback) - { + function (callback) { // Select values from table connection.execute({ sqlText: selectVariant, - complete: function (err, stmt, rows) - { + complete: function (err, stmt, rows) { var stream = stmt.streamRows(); - stream.on('error', function (err) - { + stream.on('error', function (err) { done(err); }); - stream.on('data', function (row) - { + // stream.on('data', rowAsserts); + + stream.on('data', function (row) { // Check the column, header, and value is correct - if (testCase.type == 'XML') - { - assert.strictEqual(row[TEST_COL][TEST_HEADER], TEST_XML_VAL); - } - else if (testCase.type == 'JSON') - { - assert.strictEqual(row[TEST_COL][TEST_HEADER], TEST_JSON_VAL); - } + rowAsserts(testCase, row); }); - stream.on('end', function () - { + + + stream.on('end', function () { callback(); }); } }); }, - function (callback) - { - // Drop stage - testUtil.executeCmd(connection, dropStageVariant, callback); - }, - function (callback) - { - // Drop table - testUtil.executeCmd(connection, dropTableVariant, callback); - }, - function (callback) - { + function (callback) { // Delete temp files fs.closeSync(sampleTempFile.fd); fs.unlinkSync(sampleTempFile.name); @@ -330,13 +255,159 @@ describe('Execute test - variant', function () }], done ); - }; + } + ; }; }; - for (var index = 0; index < testCases.length; index++) - { - var testCase = testCases[index]; - it(testCase.name, createItCallback(testCase)); - } + before(async () => { + connection = testUtil.createConnection(); + await testUtil.connectAsync(connection); + await testUtil.executeCmdAsync(connection, createTableVariant); + }); + + after(async () => { + // Drop table + await testUtil.executeCmdAsync(connection, dropTableVariant); + }); + + describe('Variant XML', function () { + const TEST_ATTRIBUTE_NAME = "attr"; + const TEST_ATTRIBUTE_VALUE = "attrValue"; + const ELEMENT_VALUE_FIELD = "#text"; + const TEST_XML_VAL = 123; + + before(async () => { + let createFileFormatVariant = `CREATE OR REPLACE FILE FORMAT ${TEST_VARIANT_FORMAT} TYPE = XML`; + await testUtil.executeCmdAsync(connection, createFileFormatVariant); + }); + + beforeEach(async () => { + await testUtil.executeCmdAsync(connection, createStageVariant); + }); + + afterEach(async () => { + await testUtil.executeCmdAsync(connection, truncateTableVariant); + await testUtil.executeCmdAsync(connection, dropStageVariant); + }); + + const testCases = + [ + { + name: 'xml_single_element', + type: 'XML', + fileExtension: '.xml', + sampleData: `<${TEST_HEADER}>${TEST_XML_VAL}` + }, + { + name: 'xml_single_element_with_attribute', + type: 'XML', + fileExtension: '.xml', + sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}>${TEST_XML_VAL}`, + ignoreAttributes: false + }, + { + name: 'xml_with_attribute_and_custom_parser', + type: 'XML', + fileExtension: '.xml', + sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}>${TEST_XML_VAL}`, + ignoreAttributes: false, + attributeNamePrefix: '##' + }, + { + name: 'xml_skip_attributes', + type: 'XML', + fileExtension: '.xml', + sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}>${TEST_XML_VAL}`, + attributeNamePrefix: '##' + }, + { + name: 'xml_attribute_only', + type: 'XML', + fileExtension: '.xml', + sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}>`, + ignoreAttributes: false, + attributeNamePrefix: '##' + + }, + { + name: 'xml_attribute_empty_value', + type: 'XML', + fileExtension: '.xml', + sampleData: `<${TEST_HEADER} ${TEST_ATTRIBUTE_NAME}=${TEST_ATTRIBUTE_VALUE}/>`, + ignoreAttributes: false, + attributeNamePrefix: '##' + } + ]; + + const rowAsserts = (testCase, row) => { + // Check the column, header, and value is correct + const attributeWithPrefix = testCase.attributeNamePrefix + TEST_ATTRIBUTE_NAME; + + if (testCase.name === 'xml_single_element') { + assert.strictEqual(row[TEST_COL][TEST_HEADER], TEST_XML_VAL); + } else if (testCase.name === 'xml_single_element_with_attribute') { + assert.strictEqual(row[TEST_COL][TEST_HEADER][ELEMENT_VALUE_FIELD], TEST_XML_VAL); + } else if (testCase.name === 'xml_with_attribute_and_custom_parser') { + assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][ELEMENT_VALUE_FIELD], TEST_XML_VAL); + assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][attributeWithPrefix], TEST_ATTRIBUTE_VALUE); + } else if (testCase.name === 'xml_skip_attributes') { + assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER], TEST_XML_VAL); + assert.equal(row[TEST_COL]['node'][TEST_HEADER][testCase.attributeNamePrefix + TEST_ATTRIBUTE_NAME], undefined); + } else if (testCase.name === 'xml_attribute_only') { + assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][ELEMENT_VALUE_FIELD], undefined); + assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][testCase.attributeNamePrefix + TEST_ATTRIBUTE_NAME], TEST_ATTRIBUTE_VALUE); + } else if (testCase.name === 'xml_attribute_empty_value') { + assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][ELEMENT_VALUE_FIELD], undefined); + assert.strictEqual(row[TEST_COL]['node'][TEST_HEADER][testCase.attributeNamePrefix + TEST_ATTRIBUTE_NAME], TEST_ATTRIBUTE_VALUE); + } + }; + + for (let index = 0; index < testCases.length; index++) { + const testCase = testCases[index]; + it(testCase.name, createItCallback(testCase, rowAsserts)); + } + }); + + describe('Variant JSON', function () { + const TEST_JSON_VAL = '<123>'; + + before(async () => { + const createFileFormatVariant = `CREATE OR REPLACE FILE FORMAT ${TEST_VARIANT_FORMAT} TYPE = JSON`; + await testUtil.executeCmdAsync(connection, createFileFormatVariant); + }); + // + beforeEach(async () => { + // Drop stage + await testUtil.executeCmdAsync(connection, createStageVariant); + + }); + afterEach(async () => { + // Drop stage + await testUtil.executeCmdAsync(connection, truncateTableVariant); + await testUtil.executeCmdAsync(connection, dropStageVariant); + }); + + const testCases = + [ + { + name: 'raw_json', + type: 'JSON', + fileExtension: '.json', + sampleData: `{${TEST_HEADER}: \"${TEST_JSON_VAL}\"}` + } + ]; + + const rowAsserts = (testCase, row) => { + // Check the column, header, and value is correct + if (testCase.name === 'raw_json') { + assert.strictEqual(row[TEST_COL][TEST_HEADER], TEST_JSON_VAL); + } + }; + + for (let index = 0; index < testCases.length; index++) { + const testCase = testCases[index]; + it(testCase.name, createItCallback(testCase, rowAsserts)); + } + }); });