From 2a3f30132a5084fc9823a6f30ba5206e129dfc93 Mon Sep 17 00:00:00 2001 From: Teetoow Date: Tue, 25 Aug 2020 14:00:56 +0200 Subject: [PATCH 01/29] Adapt this Makefile to FreeBSD - Add commands used to create or delete users under FreeBSD - Add specific sed FreeBSD options - Stop using the "-t" options to cp which is not supported under FreeBSD and useless in this case (It's a new GNU cp option added in order to use cp with xargs. If xargs is not used this option is useless since the output directory have to be specified as the last argument (by default)). --- Makefile | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 40bf93efc..c105f17e5 100644 --- a/Makefile +++ b/Makefile @@ -34,11 +34,25 @@ else PLATFORM := linux SHARED_EXT := .so* LIB_PREFIX := lib + SED_FLAGS := + USER_ADD := useradd -m -d /var/www/onlyoffice -r -U onlyoffice + USER_DEL := userdel onlyoffice + endif + ifeq ($(UNAME_S),FreeBSD) + PLATFORM := linux + SHARED_EXT := .so* + LIB_PREFIX := lib + SED_FLAGS := \"\" + USER_ADD := pw user add -m -d /var/www/onlyoffice -n onlyoffice + USER_DEL := pw user del -n onlyoffice endif UNAME_M := $(shell uname -m) ifeq ($(UNAME_M),x86_64) ARCHITECTURE := 64 endif + ifeq ($(UNAME_M),amd64) + ARCHITECTURE := 64 + endif ifneq ($(filter %86,$(UNAME_M)),) ARCHITECTURE := 32 endif @@ -86,66 +100,66 @@ DEBUG = $(BRANDING_DIR)/debug.js all: $(SPELLCHECKER_DICTIONARIES) $(TOOLS) $(SCHEMA) $(CORE_FONTS) $(LICENSE) $(WELCOME) $(INFO) build-date build-date: $(GRUNT_FILES) - sed "s|\(const buildVersion = \).*|\1'${PRODUCT_VERSION}';|" -i $(COMMON_DEFINES_JS) - sed "s|\(const buildNumber = \).*|\1${BUILD_NUMBER};|" -i $(COMMON_DEFINES_JS) - sed "s|\(const buildDate = \).*|\1'$$(date +%F)';|" -i $(LICENSE_JS) + sed -e "s|\(const buildVersion = \).*|\1'${PRODUCT_VERSION}';|" -i$(SED_FLAGS) $(COMMON_DEFINES_JS) + sed -e "s|\(const buildNumber = \).*|\1${BUILD_NUMBER};|" -i$(SED_FLAGS) $(COMMON_DEFINES_JS) + sed -e "s|\(const buildDate = \).*|\1'$$(date +%F)';|" -i$(SED_FLAGS) $(LICENSE_JS) test -e $(DEBUG) && \ cp $(DEBUG) $(OUTPUT)/Common/sources || true $(SPELLCHECKER_DICTIONARIES): $(GRUNT_FILES) mkdir -p $(SPELLCHECKER_DICTIONARIES) && \ - cp -r -t $(SPELLCHECKER_DICTIONARIES) $(SPELLCHECKER_DICTIONARY_FILES) + cp -r $(SPELLCHECKER_DICTIONARY_FILES) $(SPELLCHECKER_DICTIONARIES) $(SCHEMA): mkdir -p $(SCHEMA) && \ - cp -r -t $(SCHEMA) $(SCHEMA_FILES) + cp -r $(SCHEMA_FILES) $(SCHEMA) $(TOOLS): mkdir -p $(TOOLS) && \ - cp -r -t $(TOOLS) $(TOOLS_FILES) + cp -r $(TOOLS_FILES) $(TOOLS) $(LICENSE): mkdir -p $(OUTPUT) && \ - cp -r -t $(OUTPUT) $(LICENSE_FILES) + cp -r $(LICENSE_FILES) $(OUTPUT) $(GRUNT_FILES): cd $(@D) && \ npm install && \ $(GRUNT_ENV) $(GRUNT) $(GRUNT_FLAGS) mkdir -p $(OUTPUT) - cp -r -t $(OUTPUT) ./build/server/* + cp -r ./build/server/* $(OUTPUT) echo "Done" > $@ $(WELCOME): mkdir -p $(WELCOME) && \ - cp -r -t $(WELCOME) $(WELCOME_FILES) + cp -r $(WELCOME_FILES) $(WELCOME) $(INFO): mkdir -p $(INFO) && \ - cp -r -t $(INFO) $(INFO_FILES) + cp -r $(INFO_FILES) $(INFO) $(CORE_FONTS): mkdir -p $(CORE_FONTS) && \ - cp -r -t $(CORE_FONTS) $(CORE_FONTS_FILES) + cp -r $(CORE_FONTS_FILES) $(CORE_FONTS) clean: rm -rf $(GRUNT_FILES) install: mkdir -pv ${DESTDIR}/var/www/onlyoffice - if ! id -u onlyoffice > /dev/null 2>&1; then useradd -m -d /var/www/onlyoffice -r -U onlyoffice; fi + if ! id -u onlyoffice > /dev/null 2>&1; then $(USER_ADD); fi mkdir -p ${DESTDIR}${DOCUMENT_ROOT}/fonts mkdir -p ${DESTDIR}/var/log/onlyoffice/documentserver mkdir -p ${DESTDIR}/var/lib/onlyoffice/documentserver/App_Data - cp -fr -t ${DESTDIR}${DOCUMENT_ROOT} build/* ../web-apps/deploy/* + cp -fr build/* ../web-apps/deploy/* ${DESTDIR}${DOCUMENT_ROOT} mkdir -p ${DESTDIR}/etc/onlyoffice/documentserver mv ${DESTDIR}${DOCUMENT_ROOT}/server/Common/config/* ${DESTDIR}/etc/onlyoffice/documentserver - chown onlyoffice:onlyoffice -R ${DESTDIR}/var/www/onlyoffice - chown onlyoffice:onlyoffice -R ${DESTDIR}/var/log/onlyoffice - chown onlyoffice:onlyoffice -R ${DESTDIR}/var/lib/onlyoffice + chown -R onlyoffice:onlyoffice ${DESTDIR}/var/www/onlyoffice + chown -R onlyoffice:onlyoffice ${DESTDIR}/var/log/onlyoffice + chown -R onlyoffice:onlyoffice ${DESTDIR}/var/lib/onlyoffice # Make symlinks for shared libs find \ @@ -153,6 +167,7 @@ install: -maxdepth 1 \ -name *$(SHARED_EXT) \ -exec sh -c 'ln -sf {} ${DESTDIR}/lib/$$(basename {})' \; + ldconfig sudo -u onlyoffice "${DESTDIR}${DOCUMENT_ROOT}/server/tools/allfontsgen"\ --input="${DESTDIR}${DOCUMENT_ROOT}/core-fonts"\ @@ -169,7 +184,7 @@ install: --output="${DESTDIR}${DOCUMENT_ROOT}/sdkjs/common/Images" uninstall: - userdel onlyoffice + $(USER_DEL) # Unlink installed shared libs find /lib -type l | while IFS= read -r lnk; do if (readlink "$$lnk" | grep -q '^${DOCUMENT_ROOT}/server/FileConverter/bin/'); then rm "$$lnk"; fi; done From 1afab27bdb4eb180cf0314b446b7d4beb52d24ac Mon Sep 17 00:00:00 2001 From: Alexander Trofimov Date: Mon, 21 Dec 2020 11:25:22 +0300 Subject: [PATCH 02/29] [spellchecker] Update (#235) --- SpellChecker/npm-shrinkwrap.json | 4 ++-- SpellChecker/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/SpellChecker/npm-shrinkwrap.json b/SpellChecker/npm-shrinkwrap.json index 7a2a9bd05..99512abc0 100644 --- a/SpellChecker/npm-shrinkwrap.json +++ b/SpellChecker/npm-shrinkwrap.json @@ -276,8 +276,8 @@ "integrity": "sha512-+D4s2HCnxPd5PjjI0STKwncjXTUKKqm74MDMz9OPXavjsGmjkvwgLtA5yoxJUdmpj52+2u+RrXgPipahKczMKg==" }, "nodehun": { - "version": "git+https://git@github.com/ONLYOFFICE/nodehun.git#f3deabd5d0bf746b2fe477a14eee41e50e87bb4f", - "from": "git+https://git@github.com/ONLYOFFICE/nodehun.git#f3deabd5d0bf746b2fe477a14eee41e50e87bb4f", + "version": "git+https://git@github.com/ONLYOFFICE/nodehun.git#2411a56828c7d58214c61781b4a5c63d18adba99", + "from": "git+https://git@github.com/ONLYOFFICE/nodehun.git#2411a56828c7d58214c61781b4a5c63d18adba99", "requires": { "node-addon-api": "*" } diff --git a/SpellChecker/package.json b/SpellChecker/package.json index baa40d56b..7c3053991 100644 --- a/SpellChecker/package.json +++ b/SpellChecker/package.json @@ -8,7 +8,7 @@ "co": "^4.6.0", "config": "^2.0.1", "express": "^4.16.4", - "nodehun": "git+https://git@github.com/ONLYOFFICE/nodehun.git#f3deabd5d0bf746b2fe477a14eee41e50e87bb4f", + "nodehun": "git+https://git@github.com/ONLYOFFICE/nodehun.git#2411a56828c7d58214c61781b4a5c63d18adba99", "sockjs": "^0.3.19" }, "pkg": { From aef6f35f79e68da6b46611435759b8c9a1092126 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 11 Mar 2021 12:36:47 +0300 Subject: [PATCH 03/29] [bug] Fix bug with anonymous; for 74b77d1e9c87b31ffc1afdf27d3ee13c94d8ca3f (#243) --- DocService/sources/DocsCoServer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index a50990094..7854870e2 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -1884,7 +1884,7 @@ exports.install = function(server, callbackFunction) { if (null != user.lastname) { dataUser.lastname = user.lastname; } - if (null != user.name) { + if (user.name) { dataUser.username = user.name; } } From 12e40569dd0b6fd3cb5957425bae14966660f513 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 15 Mar 2021 11:08:50 +0300 Subject: [PATCH 04/29] [bug] Fix changedocinfo command in mode without token. (#244) --- DocService/sources/DocsCoServer.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 7854870e2..bdee92696 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -450,7 +450,7 @@ function removePresence(conn) { } let changeConnectionInfo = co.wrap(function*(conn, cmd) { - if (conn.canChangeName && conn.user) { + if (!conn.denyChangeName && conn.user) { yield* publish({type: commonDefines.c_oPublishType.changeConnecitonInfo, docId: conn.docId, useridoriginal: conn.user.idOriginal, cmd: cmd}); return true; } @@ -476,7 +476,7 @@ function fillJwtByConnection(conn) { edit.ds_view = conn.user.view; edit.ds_isCloseCoAuthoring = conn.isCloseCoAuthoring; edit.ds_isEnterCorrectPassword = conn.isEnterCorrectPassword; - edit.ds_canChangeName = conn.canChangeName; + edit.ds_denyChangeName = conn.denyChangeName; var options = {algorithm: cfgTokenSessionAlgorithm, expiresIn: cfgTokenSessionExpires / 1000}; var secret = utils.getSecretByElem(cfgSecretSession); @@ -1865,7 +1865,7 @@ exports.install = function(server, callbackFunction) { data.isCloseCoAuthoring = edit.ds_isCloseCoAuthoring; } data.isEnterCorrectPassword = edit.ds_isEnterCorrectPassword; - data.canChangeName = edit.ds_canChangeName; + data.denyChangeName = edit.ds_denyChangeName; if (edit.user) { var dataUser = data.user; var user = edit.user; @@ -1888,8 +1888,8 @@ exports.install = function(server, callbackFunction) { dataUser.username = user.name; } } - if (!(edit.user && edit.user.name)) { - data.canChangeName = true; + if (edit.user && edit.user.name) { + data.denyChangeName = true; } } @@ -1991,7 +1991,7 @@ exports.install = function(server, callbackFunction) { }; conn.isCloseCoAuthoring = data.isCloseCoAuthoring; conn.isEnterCorrectPassword = data.isEnterCorrectPassword; - conn.canChangeName = data.canChangeName; + conn.denyChangeName = data.denyChangeName; conn.editorType = data['editorType']; if (data.sessionTimeConnect) { conn.sessionTimeConnect = data.sessionTimeConnect; @@ -2952,7 +2952,7 @@ exports.install = function(server, callbackFunction) { participants = getParticipants(data.docId); for (i = 0; i < participants.length; ++i) { participant = participants[i]; - if (participant.canChangeName && participant.user.idOriginal === data.useridoriginal) { + if (!participant.denyChangeName && participant.user.idOriginal === data.useridoriginal) { hasChanges = true; logger.debug('changeConnectionInfo: docId = %s, userId = %s', data.docId, data.useridoriginal); participant.user.username = cmd.getUserName(); From 06134b39cec027ee952bf7d794b82128134cb811 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 18 Mar 2021 19:19:29 +0300 Subject: [PATCH 05/29] [bug] Fix bug 49147 (#245) --- FileConverter/sources/converter.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 24a77b39c..ff96c5ff9 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -424,8 +424,7 @@ function* processChanges(tempDirs, cmd, authorProps) { } changesAuthor = change.user_id_original; changesIndex = utils.getIndexFromUserId(change.user_id, change.user_id_original); - authorProps.lastModifiedBy = change.user_name; - authorProps.modified = change.change_date.toISOString().slice(0, 19) + 'Z'; + let strDate = baseConnector.getDateTime(change.change_date); changesHistory.changes.push({'created': strDate, 'user': {'id': changesAuthor, 'name': change.user_name}}); yield* streamWrite(streamObj, '['); @@ -435,6 +434,10 @@ function* processChanges(tempDirs, cmd, authorProps) { yield* streamWrite(streamObj, change.change_data); streamObj.isNoChangesInFile = false; } + if (changes.length > 0) { + authorProps.lastModifiedBy = changes[changes.length - 1].user_name; + authorProps.modified = changes[changes.length - 1].change_date.toISOString().slice(0, 19) + 'Z'; + } if (changes.length === curIndexEnd - curIndexStart) { curIndexStart += cfgMaxRequestChanges; curIndexEnd = Math.min(curIndexStart + cfgMaxRequestChanges, forceSaveIndex); From c7e7818a3ea0778cee68a443169b4897032b846d Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 2 Apr 2021 09:57:10 +0300 Subject: [PATCH 06/29] [feature] Add m_sTitle param to converter --- FileConverter/sources/converter.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index ff96c5ff9..a4548338d 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -91,6 +91,7 @@ function TaskQueueDataConvert(task) { this.key = cmd.savekey ? cmd.savekey : cmd.id; this.fileFrom = null; this.fileTo = null; + this.title = cmd.getTitle(); if(constants.AVS_OFFICESTUDIO_FILE_OTHER_PDFA !== cmd.outputformat){ this.formatTo = cmd.outputformat; } else { @@ -127,6 +128,7 @@ TaskQueueDataConvert.prototype = { xml += this.serializeXmlProp('m_sKey', this.key); xml += this.serializeXmlProp('m_sFileFrom', this.fileFrom); xml += this.serializeXmlProp('m_sFileTo', this.fileTo); + xml += this.serializeXmlProp('m_sTitle', this.title); xml += this.serializeXmlProp('m_nFormatTo', this.formatTo); xml += this.serializeXmlProp('m_bIsPDFA', this.isPDFA); xml += this.serializeXmlProp('m_nCsvTxtEncoding', this.csvTxtEncoding); From 9e12841752ac58dc21f0cb24d7c3be1c3f72ed0f Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 30 Mar 2021 12:09:04 +0300 Subject: [PATCH 07/29] [bug] Fix vulnerability by adding isbuilder parameter to downloadAs request --- Common/sources/commondefines.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 2a38911c5..a57f35341 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -37,6 +37,7 @@ const constants = require('./constants'); function InputCommand(data, copyExplicit) { //must be set explicitly to prevent vulnerability(downloadAs(with url) creates request to integrator with authorization) this['withAuthorization'] = undefined;//bool + this['isbuilder'] = undefined;//bool if (data) { this['c'] = data['c']; this['id'] = data['id']; @@ -91,11 +92,11 @@ function InputCommand(data, copyExplicit) { this['rediskey'] = data['rediskey']; this['nobase64'] = data['nobase64']; this['forgotten'] = data['forgotten']; - this['isbuilder'] = data['isbuilder']; this['status_info_in'] = data['status_info_in']; this['attempt'] = data['attempt']; if (copyExplicit) { this['withAuthorization'] = data['withAuthorization']; + this['isbuilder'] = data['isbuilder']; } } else { this['c'] = undefined;//string command @@ -143,7 +144,6 @@ function InputCommand(data, copyExplicit) { this['rediskey'] = undefined; this['nobase64'] = true; this['forgotten'] = undefined; - this['isbuilder'] = undefined; this['status_info_in'] = undefined; this['attempt'] = undefined; } From 85d14731b2ffe0e8c792898e9951e0e31ef95eed Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 2 Apr 2021 15:48:19 +0300 Subject: [PATCH 08/29] [bug] Fix vulnerability with 'insert image from url' and 'compare document from url' --- Common/config/default.json | 2 ++ Common/config/development-linux.json | 4 +++ Common/config/development-mac.json | 4 +++ Common/config/development-windows.json | 4 +++ Common/npm-shrinkwrap.json | 15 ++++++++++ Common/package.json | 1 + Common/sources/utils.js | 39 ++++++++++++++++++++++++-- FileConverter/sources/converter.js | 4 +-- 8 files changed, 69 insertions(+), 4 deletions(-) diff --git a/Common/config/default.json b/Common/config/default.json index 605e2a9e7..b0d427333 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -157,6 +157,8 @@ "useforrequest": false, "errorcode": 403 }, + "request-filtering-agent" : { + }, "secret": { "browser": {"string": "secret", "file": "", "tenants": {}}, "inbox": {"string": "secret", "file": "", "tenants": {}}, diff --git a/Common/config/development-linux.json b/Common/config/development-linux.json index 93ed63917..7243ac7cb 100644 --- a/Common/config/development-linux.json +++ b/Common/config/development-linux.json @@ -32,6 +32,10 @@ "utils": { "utils_common_fontdir": "/usr/share/fonts" }, + "request-filtering-agent" : { + "allowPrivateIPAddress": true, + "allowMetaIPAddress": true + }, "sockjs": { "sockjs_url": "/web-apps/vendor/sockjs/sockjs.min.js" } diff --git a/Common/config/development-mac.json b/Common/config/development-mac.json index 20a78dd23..8da2ab528 100644 --- a/Common/config/development-mac.json +++ b/Common/config/development-mac.json @@ -38,6 +38,10 @@ "dbUser": "root", "dbPass": "onlyoffice" }, + "request-filtering-agent" : { + "allowPrivateIPAddress": true, + "allowMetaIPAddress": true + }, "editor": { "spellcheckerUrl": "http://127.0.0.1:8080" }, diff --git a/Common/config/development-windows.json b/Common/config/development-windows.json index 7c724c227..2712b6917 100644 --- a/Common/config/development-windows.json +++ b/Common/config/development-windows.json @@ -38,6 +38,10 @@ "dbUser": "root", "dbPass": "onlyoffice" }, + "request-filtering-agent" : { + "allowPrivateIPAddress": true, + "allowMetaIPAddress": true + }, "editor": { "spellcheckerUrl": "http://127.0.0.1:8080" }, diff --git a/Common/npm-shrinkwrap.json b/Common/npm-shrinkwrap.json index 3a0274a80..270d857d1 100644 --- a/Common/npm-shrinkwrap.json +++ b/Common/npm-shrinkwrap.json @@ -642,6 +642,21 @@ } } }, + "request-filtering-agent": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/request-filtering-agent/-/request-filtering-agent-1.0.5.tgz", + "integrity": "sha512-zkO5wg1VhoPqz5FIIXLY+iEveiX+p6A/i++SS4ZXd0UDms6VtHLLYLw9hQnIbMsKbYwHgzhNeIis7jMncoG9ag==", + "requires": { + "ipaddr.js": "^1.9.1" + }, + "dependencies": { + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + } + } + }, "rfdc": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.1.2.tgz", diff --git a/Common/package.json b/Common/package.json index ae461ed8c..ba743d3ae 100644 --- a/Common/package.json +++ b/Common/package.json @@ -22,6 +22,7 @@ "node-cache": "^4.2.1", "node-statsd": "^0.1.1", "request": "^2.88.0", + "request-filtering-agent": "^1.0.5", "rhea": "^0.3.9", "uri-js": "^4.2.2" } diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 41adeb2b8..74d96bf7c 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -57,6 +57,7 @@ const constants = require('./constants'); const logger = require('./logger'); const forwarded = require('forwarded'); const mime = require('mime'); +const { RequestFilteringHttpAgent, RequestFilteringHttpsAgent } = require("request-filtering-agent"); var configIpFilter = config.get('services.CoAuthoring.ipfilter'); var cfgIpFilterRules = configIpFilter.get('rules'); @@ -75,6 +76,7 @@ var cfgRequestDefaults = config.get('services.CoAuthoring.requestDefaults'); const cfgTokenOutboxInBody = config.get('services.CoAuthoring.token.outbox.inBody'); const cfgTokenEnableRequestOutbox = config.get('services.CoAuthoring.token.enable.request.outbox'); const cfgTokenOutboxUrlExclusionRegex = config.get('services.CoAuthoring.token.outbox.urlExclusionRegex'); +const cfgRequesFilteringAgent = config.get('services.CoAuthoring.request-filtering-agent'); var ANDROID_SAFE_FILENAME = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ._-+,@£$€!½§~\'=()[]{}0123456789'; @@ -96,6 +98,10 @@ var g_oIpFilterRules = function() { }(); const pemfileCache = new NodeCache({stdTTL: ms(cfgExpPemStdTtl) / 1000, checkperiod: ms(cfgExpPemCheckPeriod) / 1000, errorOnMissing: false, useClones: true}); +function getRequestFilterAgent(url, options) { + return url.startsWith("https") ? new RequestFilteringHttpsAgent(options) : new RequestFilteringHttpAgent(options); +} + exports.CONVERTION_TIMEOUT = 1.5 * (cfgVisibilityTimeout + cfgQueueRetentionPeriod) * 1000; exports.addSeconds = function(date, sec) { @@ -247,13 +253,39 @@ function raiseError(ro, code, msg) { ro.emit('error', error); } function downloadUrlPromise(uri, optTimeout, optLimit, opt_Authorization) { + //todo replace deprecated request module + let filterPrivate = opt_Authorization ? false : true; + const maxRedirects = (undefined !== cfgRequestDefaults.maxRedirects) ? cfgRequestDefaults.maxRedirects : 10; + const followRedirect = (undefined !== cfgRequestDefaults.followRedirect) ? cfgRequestDefaults.followRedirect : true; + var redirectsFollowed = 0; + let doRequest = function(curUrl) { + return downloadUrlPromiseWithoutRedirect(curUrl, optTimeout, optLimit, opt_Authorization, filterPrivate) + .catch(function(err) { + let response = err.response; + if (response && response.statusCode >= 300 && response.statusCode < 400 && response.caseless.has('location')) { + let newUrl = response.caseless.get('location'); + if (followRedirect && redirectsFollowed < maxRedirects) { + redirectsFollowed++; + logger.debug('downloadUrlPromise %d redirect: %s', redirectsFollowed, newUrl); + return doRequest(newUrl); + } + } + throw err; + }); + }; + return doRequest(uri); +} +function downloadUrlPromiseWithoutRedirect(uri, optTimeout, optLimit, opt_Authorization, opt_filterPrivate) { return new Promise(function (resolve, reject) { //IRI to URI uri = URI.serialize(URI.parse(uri)); var urlParsed = url.parse(uri); //if you expect binary data, you should set encoding: null let connectionAndInactivity = optTimeout && optTimeout.connectionAndInactivity && ms(optTimeout.connectionAndInactivity); - var options = {uri: urlParsed, encoding: null, timeout: connectionAndInactivity}; + var options = {uri: urlParsed, encoding: null, timeout: connectionAndInactivity, followRedirect: false}; + if (opt_filterPrivate) { + options.agent = getRequestFilterAgent(uri, cfgRequesFilteringAgent); + } if (opt_Authorization) { options.headers = {}; options.headers[cfgTokenOutboxHeader] = cfgTokenOutboxPrefix + opt_Authorization; @@ -277,7 +309,10 @@ function downloadUrlPromise(uri, optTimeout, optLimit, opt_Authorization) { } resolve(body); } else { - reject(new Error('Error response: statusCode:' + response.statusCode + ' ;body:\r\n' + body)); + let error = new Error('Error response: statusCode:' + response.statusCode + ' ;body:\r\n' + body); + error.statusCode = response.statusCode; + error.response = response; + reject(error); } } }).on('response', function(response) { diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 8b3f00f05..18f8cff54 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -707,8 +707,8 @@ function* ExecuteTask(task) { curDate = new Date(); } if (tempDirs) { - deleteFolderRecursive(tempDirs.temp); - logger.debug('deleteFolderRecursive (id=%s)', dataConvert.key); + //deleteFolderRecursive(tempDirs.temp); + logger.debug('deleteFolderRecursive %s(id=%s)', tempDirs.temp, dataConvert.key); if(clientStatsD) { clientStatsD.timing('conv.deleteFolderRecursive', new Date() - curDate); curDate = new Date(); From 6776d6fa0841f82156e84ecde985d26227f0c246 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sat, 3 Apr 2021 07:57:43 +0300 Subject: [PATCH 09/29] [bug] Fix unwanted changes in 253679565cce6a8327f88915bcddbf5fe6f86fdc --- FileConverter/sources/converter.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 18f8cff54..8b3f00f05 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -707,8 +707,8 @@ function* ExecuteTask(task) { curDate = new Date(); } if (tempDirs) { - //deleteFolderRecursive(tempDirs.temp); - logger.debug('deleteFolderRecursive %s(id=%s)', tempDirs.temp, dataConvert.key); + deleteFolderRecursive(tempDirs.temp); + logger.debug('deleteFolderRecursive (id=%s)', dataConvert.key); if(clientStatsD) { clientStatsD.timing('conv.deleteFolderRecursive', new Date() - curDate); curDate = new Date(); From caebdd6a0aae3f9bf2175fea7af5974f387a4a34 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 2 Apr 2021 20:10:33 +0300 Subject: [PATCH 10/29] [bug] For bug 49002 --- Common/config/default.json | 1 + Common/config/development-linux.json | 3 ++- Common/config/development-mac.json | 1 + Common/config/development-windows.json | 1 + FileConverter/sources/converter.js | 4 ++++ 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Common/config/default.json b/Common/config/default.json index 15e472b4d..e180ced1b 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -254,6 +254,7 @@ "x2tPath": "null", "docbuilderPath": "null", "docbuilderAllFontsPath": "null", + "docbuilderCoreFontsPath": "", "args": "", "spawnOptions": {}, "errorfiles": "", diff --git a/Common/config/development-linux.json b/Common/config/development-linux.json index ef2eac3bc..ffd501f92 100644 --- a/Common/config/development-linux.json +++ b/Common/config/development-linux.json @@ -51,7 +51,8 @@ "presentationThemesDir": "../../../sdkjs/slide/themes", "x2tPath": "../../FileConverter/bin/x2t", "docbuilderPath": "../../FileConverter/bin/docbuilder", - "docbuilderAllFontsPath": "../../App_Data/docbuilder/AllFonts.js" + "docbuilderAllFontsPath": "../../App_Data/docbuilder/AllFonts.js", + "docbuilderCoreFontsPath": "../../../core-fonts" } }, "SpellChecker": { diff --git a/Common/config/development-mac.json b/Common/config/development-mac.json index 9f1826cb5..2bdab09df 100644 --- a/Common/config/development-mac.json +++ b/Common/config/development-mac.json @@ -58,6 +58,7 @@ "x2tPath": "../../FileConverter/bin/x2t", "docbuilderPath": "../../FileConverter/Bin/docbuilder", "docbuilderAllFontsPath": "../../App_Data/docbuilder/AllFonts.js", + "docbuilderCoreFontsPath": "../../../core-fonts", "errorfiles": "error" } }, diff --git a/Common/config/development-windows.json b/Common/config/development-windows.json index 8af1f3340..2c15af5af 100644 --- a/Common/config/development-windows.json +++ b/Common/config/development-windows.json @@ -58,6 +58,7 @@ "x2tPath": "../../FileConverter/Bin/x2t.exe", "docbuilderPath": "../../FileConverter/Bin/docbuilder.exe", "docbuilderAllFontsPath": "../../App_Data/docbuilder/AllFonts.js", + "docbuilderCoreFontsPath": "../../../core-fonts", "errorfiles": "error" } }, diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index a4548338d..837caddb3 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -60,6 +60,7 @@ var cfgPresentationThemesDir = configConverter.get('presentationThemesDir'); var cfgX2tPath = configConverter.get('x2tPath'); var cfgDocbuilderPath = configConverter.get('docbuilderPath'); var cfgDocbuilderAllFontsPath = configConverter.get('docbuilderAllFontsPath'); +var cfgDocbuilderCoreFontsPath = configConverter.get('docbuilderCoreFontsPath'); var cfgArgs = configConverter.get('args'); var cfgSpawnOptions = configConverter.get('spawnOptions'); if (cfgSpawnOptions.env) { @@ -683,6 +684,9 @@ function* ExecuteTask(task) { fs.mkdirSync(path.join(tempDirs.result, 'output')); processPath = cfgDocbuilderPath; childArgs.push('--all-fonts-path=' + cfgDocbuilderAllFontsPath); + if (cfgDocbuilderCoreFontsPath) { + childArgs.push('--fonts-dir=' + cfgDocbuilderCoreFontsPath); + } childArgs.push('--save-use-only-names=' + tempDirs.result + '/output'); childArgs.push(dataConvert.fileFrom); } From 9b06651d905242e947ded7aa38237f9bf0865659 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 7 Apr 2021 11:24:14 +0300 Subject: [PATCH 11/29] [bug] Fix bug 49516 with redirect --- Common/sources/utils.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 74d96bf7c..15bb019f8 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -263,11 +263,15 @@ function downloadUrlPromise(uri, optTimeout, optLimit, opt_Authorization) { .catch(function(err) { let response = err.response; if (response && response.statusCode >= 300 && response.statusCode < 400 && response.caseless.has('location')) { - let newUrl = response.caseless.get('location'); + let redirectTo = response.caseless.get('location'); if (followRedirect && redirectsFollowed < maxRedirects) { + if (!/^https?:/.test(redirectTo) && err.request) { + redirectTo = url.resolve(err.request.uri.href, redirectTo) + } + + logger.debug('downloadUrlPromise redirectsFollowed:%d redirectTo: %s', redirectsFollowed, redirectTo); redirectsFollowed++; - logger.debug('downloadUrlPromise %d redirect: %s', redirectsFollowed, newUrl); - return doRequest(newUrl); + return doRequest(redirectTo); } } throw err; @@ -311,6 +315,7 @@ function downloadUrlPromiseWithoutRedirect(uri, optTimeout, optLimit, opt_Author } else { let error = new Error('Error response: statusCode:' + response.statusCode + ' ;body:\r\n' + body); error.statusCode = response.statusCode; + error.request = ro; error.response = response; reject(error); } From 9cb1188bd7c453966d343c1ec0d6f2e3ca087de5 Mon Sep 17 00:00:00 2001 From: Roman Demidov Date: Wed, 7 Apr 2021 13:34:20 +0300 Subject: [PATCH 12/29] Add docbuilderCoreFontsPath parameter definition to production (#250) * Add docbuilderCoreFontsPath parameter definition to production * Add changes --- Common/config/production-linux.json | 3 ++- Common/config/production-windows.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Common/config/production-linux.json b/Common/config/production-linux.json index 07585e2a0..0e2a99705 100644 --- a/Common/config/production-linux.json +++ b/Common/config/production-linux.json @@ -60,7 +60,8 @@ "presentationThemesDir": "/var/www/onlyoffice/documentserver/sdkjs/slide/themes", "x2tPath": "/var/www/onlyoffice/documentserver/server/FileConverter/bin/x2t", "docbuilderPath": "/var/www/onlyoffice/documentserver/server/FileConverter/bin/docbuilder", - "docbuilderAllFontsPath": "/var/lib/onlyoffice/documentserver/App_Data/docbuilder/AllFonts.js" + "docbuilderAllFontsPath": "/var/lib/onlyoffice/documentserver/App_Data/docbuilder/AllFonts.js", + "docbuilderCoreFontsPath": "/var/www/onlyoffice/documentserver/core-fonts" } }, "FileStorage": { diff --git a/Common/config/production-windows.json b/Common/config/production-windows.json index 8d1caecb4..22fd43e77 100644 --- a/Common/config/production-windows.json +++ b/Common/config/production-windows.json @@ -60,7 +60,8 @@ "presentationThemesDir": "../../sdkjs/slide/themes", "x2tPath": "../FileConverter/bin/x2t.exe", "docbuilderPath": "../FileConverter/bin/docbuilder.exe", - "docbuilderAllFontsPath": "../App_Data/docbuilder/AllFonts.js" + "docbuilderAllFontsPath": "../App_Data/docbuilder/AllFonts.js", + "docbuilderCoreFontsPath": "../../core-fonts" } }, "SpellChecker": { From cd93ba1d9fe02ba00cb1de4b2282c71a8f39811d Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 14 Apr 2021 11:52:05 +0300 Subject: [PATCH 13/29] [config] For bug 49634 --- Common/config/default.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Common/config/default.json b/Common/config/default.json index b0d427333..59f58a837 100644 --- a/Common/config/default.json +++ b/Common/config/default.json @@ -158,6 +158,8 @@ "errorcode": 403 }, "request-filtering-agent" : { + "allowPrivateIPAddress": true, + "allowMetaIPAddress": true }, "secret": { "browser": {"string": "secret", "file": "", "tenants": {}}, From 98ae4f72235fa3fd8ac686681a29d338709da411 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 27 Apr 2021 18:27:02 +0300 Subject: [PATCH 14/29] [bug] For bug 49716 --- Common/sources/commondefines.js | 8 ++++++++ DocService/sources/canvasservice.js | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 3f704833c..817788a6e 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -88,6 +88,7 @@ function InputCommand(data, copyExplicit) { this['inline'] = data['inline']; this['password'] = data['password']; this['savepassword'] = data['savepassword']; + this['withoutPassword'] = data['withoutPassword']; this['outputurls'] = data['outputurls']; this['closeonerror'] = data['closeonerror']; this['serverVersion'] = data['serverVersion']; @@ -143,6 +144,7 @@ function InputCommand(data, copyExplicit) { this['inline'] = undefined;//content disposition this['password'] = undefined; this['savepassword'] = undefined; + this['withoutPassword'] = undefined; this['outputurls'] = undefined; this['closeonerror'] = undefined; this['serverVersion'] = undefined; @@ -377,6 +379,12 @@ InputCommand.prototype = { setSavePassword: function(data) { this['savepassword'] = data; }, + getWithoutPassword: function() { + return this['withoutPassword']; + }, + setWithoutPassword: function(data) { + this['withoutPassword'] = data; + }, setOutputUrls: function(data) { this['outputurls'] = data; }, diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index d665c80ca..a2b609553 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1172,7 +1172,7 @@ exports.downloadAs = function(req, res) { } var selectRes = yield taskResult.select(docId); var row = selectRes.length > 0 ? selectRes[0] : null; - if (row && row.password) { + if (row && row.password && !cmd.getWithoutPassword()) { cmd.setSavePassword(row.password); } cmd.setData(req.body); From ed425743b5169902cae41d00289094d82a635bc5 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 30 Apr 2021 20:05:53 +0300 Subject: [PATCH 15/29] [bug] Fix bug 49946 --- DocService/sources/DocsCoServer.js | 40 ++++++++++++----- DocService/sources/baseConnector.js | 56 ++++++++++++++++++++++++ DocService/sources/canvasservice.js | 40 ++++++++++++----- DocService/sources/gc.js | 4 +- DocService/sources/mySqlBaseConnector.js | 2 +- DocService/sources/taskresult.js | 9 +++- FileConverter/sources/converter.js | 6 ++- 7 files changed, 127 insertions(+), 30 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index bdee92696..a4fab7825 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -707,6 +707,18 @@ function* getChangesIndex(docId) { } return res; } + +const hasChanges = co.wrap(function*(docId) { + let puckerIndex = yield* getChangesIndex(docId); + if (0 === puckerIndex) { + let selectRes = yield taskResult.select(docId); + if (selectRes.length > 0 && selectRes[0].password) { + return sqlBase.DocumentPassword.prototype.hasPasswordChanges(docId, selectRes[0].password); + } + return false; + } + return true; +}); function* setForceSave(docId, forceSave, cmd, success) { let forceSaveType = forceSave.getType(); if (commonDefines.c_oAscForceSaveTypes.Form !== forceSaveType) { @@ -781,6 +793,16 @@ let startForceSave = co.wrap(function*(docId, type, opt_userdata, opt_userId, op logger.debug('startForceSave end:docId = %s', docId); return res; }); +let resetForceSaveAfterChanges = co.wrap(function*(docId, newChangesLastTime, puckerIndex, baseUrl) { + //last save + if (newChangesLastTime) { + yield editorData.setForceSave(docId, newChangesLastTime, puckerIndex, baseUrl); + if (cfgForceSaveEnable) { + let expireAt = newChangesLastTime + cfgForceSaveInterval; + yield editorData.addForceSaveTimerNX(docId, expireAt); + } + } +}); function* startRPC(conn, responseKey, data) { let docId = conn.docId; logger.debug('startRPC start responseKey:%s , %j:docId = %s', responseKey, data, docId); @@ -856,8 +878,8 @@ function* sendStatusDocument(docId, bChangeBase, opt_userAction, opt_userIndex, var status = c_oAscServerStatus.Editing; var participants = yield* getOriginalParticipantsId(docId); if (0 === participants.length) { - var puckerIndex = yield* getChangesIndex(docId); - if (!(puckerIndex > 0) || opt_forceClose) { + let bHasChanges = yield hasChanges(docId); + if (!bHasChanges || opt_forceClose) { status = c_oAscServerStatus.Closed; } } @@ -1180,11 +1202,12 @@ exports.hasEditors = hasEditors; exports.getEditorsCountPromise = co.wrap(getEditorsCount); exports.getCallback = getCallback; exports.getIsShutdown = getIsShutdown; -exports.getChangesIndexPromise = co.wrap(getChangesIndex); +exports.hasChanges = hasChanges; exports.cleanDocumentOnExitPromise = co.wrap(cleanDocumentOnExit); exports.cleanDocumentOnExitNoChangesPromise = co.wrap(cleanDocumentOnExitNoChanges); exports.setForceSave = setForceSave; exports.startForceSave = startForceSave; +exports.resetForceSaveAfterChanges = resetForceSaveAfterChanges; exports.checkJwt = checkJwt; exports.getRequestParams = getRequestParams; exports.checkJwtHeader = checkJwtHeader; @@ -1399,8 +1422,7 @@ exports.install = function(server, callbackFunction) { // Только если редактируем if (false === isView) { bHasEditors = yield* hasEditors(docId, hvals); - var puckerIndex = yield* getChangesIndex(docId); - bHasChanges = puckerIndex > 0; + bHasChanges = yield hasChanges(docId); let needSendStatus = true; if (conn.encrypted) { @@ -2539,13 +2561,7 @@ exports.install = function(server, callbackFunction) { // Автоматически снимаем lock сами и посылаем индекс для сохранения yield* unSaveLock(conn, changesIndex, newChangesLastTime); //last save - if (newChangesLastTime) { - yield editorData.setForceSave(docId, newChangesLastTime, puckerIndex, utils.getBaseUrlByConnection(conn)); - if (cfgForceSaveEnable) { - let expireAt = newChangesLastTime + cfgForceSaveInterval; - yield editorData.addForceSaveTimerNX(docId, expireAt); - } - } + yield resetForceSaveAfterChanges(docId, newChangesLastTime, puckerIndex, utils.getBaseUrlByConnection(conn)); } else { let changesToSend = arrNewDocumentChanges; if(changesToSend.length > cfgPubSubMaxChanges) { diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js index 95c857fa6..967a7925a 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/baseConnector.js @@ -314,3 +314,59 @@ UserCallback.prototype.getCallbackByUserIndex = function(docId, callbacksStr, op return callbackUrl; }; exports.UserCallback = UserCallback; + +function DocumentPassword() { + this.password = undefined; + this.userId = undefined; + this.userIndex = undefined; +} +DocumentPassword.prototype.fromString = function(passwordStr){ + var parsed = JSON.parse(passwordStr); + this.fromValues(parsed.password, parsed.userId, parsed.userIndex); +}; +DocumentPassword.prototype.fromValues = function(password, userId, userIndex){ + if(null !== password){ + this.password = password; + } + if(null !== userId){ + this.userId = userId; + } + if(null !== userIndex){ + this.userIndex = userIndex; + } +}; +DocumentPassword.prototype.delimiter = String.fromCharCode(5); +DocumentPassword.prototype.toSQLInsert = function(){ + return this.delimiter + JSON.stringify(this); +}; +DocumentPassword.prototype.isInitial = function(){ + return !this.userId; +}; +DocumentPassword.prototype.getDocPassword = function(docId, docPasswordStr) { + let res = {initial: undefined, current: undefined, userId: undefined, userIndex: undefined}; + if (docPasswordStr) { + logger.debug("getDocPassword: docId = %s passwords = %s", docId, docPasswordStr); + let passwords = docPasswordStr.split(UserCallback.prototype.delimiter); + + for (let i = 1; i < passwords.length; ++i) { + let password = new DocumentPassword(); + password.fromString(passwords[i]); + if (password.isInitial()) { + res.initial = password.password; + } + res.current = password.password; + res.userId = password.userId; + res.userIndex = password.userIndex; + } + } + return res; +}; +DocumentPassword.prototype.getCurPassword = function(docId, docPasswordStr) { + let docPassword = this.getDocPassword(docId, docPasswordStr); + return docPassword.current; +}; +DocumentPassword.prototype.hasPasswordChanges = function(docId, docPasswordStr) { + let docPassword = this.getDocPassword(docId, docPasswordStr); + return docPassword.initial !== docPassword.current; +}; +exports.DocumentPassword = DocumentPassword; diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index a2b609553..8ae797243 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -136,7 +136,7 @@ function* getOutputData(cmd, outputData, key, optConn, optAdditionalOutput, opt_ let row = selectRes[0]; status = row.status; statusInfo = row.status_info; - password = row.password; + password = sqlBase.DocumentPassword.prototype.getCurPassword(key, row.password); creationDate = row.created_at && row.created_at.getTime(); } switch (status) { @@ -419,7 +419,7 @@ function* commandReopen(conn, cmd, outputData) { let selectRes = yield taskResult.select(cmd.getDocId()); if (selectRes.length > 0) { let row = selectRes[0]; - if (row.password) { + if (sqlBase.DocumentPassword.prototype.getCurPassword(cmd.getDocId(), row.password)) { logger.debug('commandReopen has password: docId = %s', cmd.getDocId()); yield* commandOpenFillOutput(conn, cmd, outputData, false); docsCoServer.modifyConnectionForPassword(conn, constants.FILE_STATUS_OK === outputData.getStatus()); @@ -502,8 +502,13 @@ function* commandSfctByCmd(cmd, opt_priority, opt_expiration, opt_queue) { yield* addRandomKeyTaskCmd(cmd); var selectRes = yield taskResult.select(cmd.getDocId()); var row = selectRes.length > 0 ? selectRes[0] : null; - if (row && row.password) { - cmd.setSavePassword(row.password); + let docPassword = row && sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), row.password); + if (docPassword.current) { + cmd.setSavePassword(docPassword.current); + if (docPassword.userId) { + cmd.setUserId(docPassword.userId); + cmd.setUserIndex(docPassword.userIndex); + } } var queueData = getSaveTask(cmd); queueData.setFromChanges(true); @@ -699,7 +704,7 @@ function* commandSetPassword(conn, cmd, outputData) { if (selectRes.length > 0) { let row = selectRes[0]; hasPasswordCol = undefined !== row.password; - if (taskResult.FileStatus.Ok === row.status && row.password) { + if (taskResult.FileStatus.Ok === row.status && sqlBase.DocumentPassword.prototype.getCurPassword(cmd.getDocId(), row.password)) { hasDocumentPassword = true; } } @@ -712,6 +717,10 @@ function* commandSetPassword(conn, cmd, outputData) { var task = new taskResult.TaskResultData(); task.key = cmd.getDocId(); task.password = cmd.getPassword() || ""; + if (conn.user) { + task.innerUserId = conn.user.idOriginal; + task.innerUserIndex = utils.getIndexFromUserId(conn.user.id, conn.user.idOriginal); + } var upsertRes = yield taskResult.updateIf(task, updateMask); if (upsertRes.affectedRows > 0) { @@ -719,6 +728,9 @@ function* commandSetPassword(conn, cmd, outputData) { if (!conn.isEnterCorrectPassword) { docsCoServer.modifyConnectionForPassword(conn, true); } + let newChangesLastDate = new Date(); + newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding + yield docsCoServer.resetForceSaveAfterChanges(cmd.getDocId(), newChangesLastDate.getTime(), 0x7FFFFFFF, utils.getBaseUrlByConnection(conn)); } else { logger.debug('commandSetPassword sql update error: docId = %s', cmd.getDocId()); outputData.setStatus('err'); @@ -815,7 +827,7 @@ function* commandSfcCallback(cmd, isSfcm, isEncrypted) { } else { isError = true; } - if (getRes) { + if (getRes && userLastChangeId) { logger.debug('Callback commandSfcCallback: docId = %s callback = %s', docId, getRes.server.href); var outputSfc = new commonDefines.OutputSfcData(); outputSfc.setKey(docId); @@ -954,7 +966,7 @@ function* commandSfcCallback(cmd, isSfcm, isEncrypted) { } } } else { - logger.warn('Empty Callback commandSfcCallback: docId = %s', docId); + logger.warn('Empty Callback or userLastChangeId=%s commandSfcCallback: docId = %s', userLastChangeId, docId); storeForgotten = true; } if (undefined !== updateIfTask && !isSfcm) { @@ -1172,8 +1184,9 @@ exports.downloadAs = function(req, res) { } var selectRes = yield taskResult.select(docId); var row = selectRes.length > 0 ? selectRes[0] : null; - if (row && row.password && !cmd.getWithoutPassword()) { - cmd.setSavePassword(row.password); + let password = row && sqlBase.DocumentPassword.prototype.getCurPassword(cmd.getDocId(), row.password); + if (password && !cmd.getWithoutPassword()) { + cmd.setSavePassword(password); } cmd.setData(req.body); var outputData = new OutputData(cmd.getCommand()); @@ -1289,8 +1302,13 @@ exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt cmd.setStatusInfoIn(statusInfo); cmd.setUserActionId(opt_userId); cmd.setUserActionIndex(opt_userIndex); - if (row.password) { - cmd.setSavePassword(row.password); + let docPassword = row && sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), row.password); + if (docPassword.current) { + cmd.setSavePassword(docPassword.current); + if (docPassword.userId) { + cmd.setUserId(docPassword.userId); + cmd.setUserIndex(docPassword.userIndex); + } } yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js index c9c983751..f8ca3fe4d 100644 --- a/DocService/sources/gc.js +++ b/DocService/sources/gc.js @@ -102,8 +102,8 @@ var checkDocumentExpire = function() { for (var i = 0; i < expiredKeys.length; ++i) { var docId = expiredKeys[i]; if (docId) { - var puckerIndex = yield docsCoServer.getChangesIndexPromise(docId); - if (puckerIndex > 0) { + var hasChanges = yield docsCoServer.hasChanges(docId); + if (hasChanges) { yield docsCoServer.createSaveTimerPromise(docId, null, null, queue, true); startSaveCount++; } else { diff --git a/DocService/sources/mySqlBaseConnector.js b/DocService/sources/mySqlBaseConnector.js index 1c04fd7d1..69c594bb2 100644 --- a/DocService/sources/mySqlBaseConnector.js +++ b/DocService/sources/mySqlBaseConnector.js @@ -78,7 +78,7 @@ let addSqlParam = function (val, values) { }; exports.addSqlParameter = addSqlParam; let concatParams = function (val1, val2) { - return `CONCAT(${val1}, ${val2})`; + return `CONCAT(COALESCE(${val1}, ''), COALESCE(${val2}, ''))`; }; exports.concatParams = concatParams; diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index 9bf15d1f7..7146c45dc 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -66,6 +66,9 @@ function TaskResultData() { this.callback = null; this.baseurl = null; this.password = null; + + this.innerUserId = null;//not a DB field + this.innerUserIndex = null;//not a DB field } TaskResultData.prototype.completeDefaults = function() { if (!this.key) { @@ -152,8 +155,10 @@ function toUpdateArray(task, updateTime, isMask, values) { res.push(`baseurl=${sqlParam}`); } if (null != task.password) { - let sqlParam = addSqlParam(task.password, values); - res.push(`password=${sqlParam}`); + var documentPassword = new sqlBase.DocumentPassword(); + documentPassword.fromValues(task.password, task.innerUserId, task.innerUserIndex); + let sqlParam = addSqlParam(documentPassword.toSQLInsert(), values); + res.push(`password=${concatParams('password', sqlParam)}`); } return res; } diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index 837caddb3..ca71c5cad 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -457,8 +457,10 @@ function* processChanges(tempDirs, cmd, authorProps) { changesAuthor = forceSave.getAuthorUserId(); changesIndex = forceSave.getAuthorUserIndex(); } - cmd.setUserId(changesAuthor); - cmd.setUserIndex(changesIndex); + if (null != changesAuthor && null != changesIndex) { + cmd.setUserId(changesAuthor); + cmd.setUserIndex(changesIndex); + } fs.writeFileSync(path.join(tempDirs.result, 'changesHistory.json'), JSON.stringify(changesHistory), 'utf8'); return res; } From e1a309a622e60e890db64d776437e118a0a93ca6 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 4 May 2021 15:09:46 +0300 Subject: [PATCH 16/29] [bug] Fix bug 49714 --- Common/sources/commondefines.js | 8 ++++++ DocService/sources/DocsCoServer.js | 15 +++++++--- DocService/sources/baseConnector.js | 22 ++++++-------- DocService/sources/canvasservice.js | 21 +++++++------- DocService/sources/converterservice.js | 5 ++-- DocService/sources/editorDataMemory.js | 4 +-- DocService/sources/taskresult.js | 40 ++++++++++++++++++++------ FileConverter/sources/converter.js | 27 +++++++++++++---- 8 files changed, 95 insertions(+), 47 deletions(-) diff --git a/Common/sources/commondefines.js b/Common/sources/commondefines.js index 817788a6e..3622e3583 100644 --- a/Common/sources/commondefines.js +++ b/Common/sources/commondefines.js @@ -37,6 +37,7 @@ const constants = require('./constants'); function InputCommand(data, copyExplicit) { //must be set explicitly to prevent vulnerability(downloadAs(with url) creates request to integrator with authorization) this['withAuthorization'] = undefined;//bool + this['externalChangeInfo'] = undefined;//zero DB changes case: set password, undo all changes if (data) { this['c'] = data['c']; this['id'] = data['id']; @@ -100,6 +101,7 @@ function InputCommand(data, copyExplicit) { this['attempt'] = data['attempt']; if (copyExplicit) { this['withAuthorization'] = data['withAuthorization']; + this['externalChangeInfo'] = data['externalChangeInfo']; } } else { this['c'] = undefined;//string command @@ -438,6 +440,12 @@ InputCommand.prototype = { }, setWithAuthorization: function(data) { this['withAuthorization'] = data; + }, + getExternalChangeInfo: function() { + return this['externalChangeInfo']; + }, + setExternalChangeInfo: function(data) { + this['externalChangeInfo'] = data; } }; diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index a4fab7825..f62f11ca0 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -709,6 +709,7 @@ function* getChangesIndex(docId) { } const hasChanges = co.wrap(function*(docId) { + //todo check editorData.getForceSave in case of "undo all changes" let puckerIndex = yield* getChangesIndex(docId); if (0 === puckerIndex) { let selectRes = yield taskResult.select(docId); @@ -779,7 +780,7 @@ let startForceSave = co.wrap(function*(docId, type, opt_userdata, opt_userId, op priority = constants.QUEUE_PRIORITY_LOW; } //start new convert - let status = yield* converterService.convertFromChanges(docId, baseUrl, forceSave, opt_userdata, + let status = yield* converterService.convertFromChanges(docId, baseUrl, forceSave, startedForceSave.changeInfo, opt_userdata, opt_userConnectionId, opt_responseKey, priority, expiration, opt_queue); if (constants.NO_ERROR === status.err) { res.time = forceSave.getTime(); @@ -793,10 +794,13 @@ let startForceSave = co.wrap(function*(docId, type, opt_userdata, opt_userId, op logger.debug('startForceSave end:docId = %s', docId); return res; }); -let resetForceSaveAfterChanges = co.wrap(function*(docId, newChangesLastTime, puckerIndex, baseUrl) { +function getExternalChangeInfo(user, date) { + return {user_id: user.id, user_id_original: user.idOriginal, user_name: user.username, change_date: date}; +} +let resetForceSaveAfterChanges = co.wrap(function*(docId, newChangesLastTime, puckerIndex, baseUrl, changeInfo) { //last save if (newChangesLastTime) { - yield editorData.setForceSave(docId, newChangesLastTime, puckerIndex, baseUrl); + yield editorData.setForceSave(docId, newChangesLastTime, puckerIndex, baseUrl, changeInfo); if (cfgForceSaveEnable) { let expireAt = newChangesLastTime + cfgForceSaveInterval; yield editorData.addForceSaveTimerNX(docId, expireAt); @@ -1026,6 +1030,7 @@ function* cleanDocumentOnExit(docId, deleteChanges) { yield editorData.cleanDocumentOnExit(docId); //remove changes if (deleteChanges) { + yield taskResult.restoreInitialPassword(docId); sqlBase.deleteChanges(docId, null); //delete forgotten after successful send on callbackUrl yield storage.deletePath(cfgForgottenFiles + '/' + docId); @@ -1208,6 +1213,7 @@ exports.cleanDocumentOnExitNoChangesPromise = co.wrap(cleanDocumentOnExitNoChang exports.setForceSave = setForceSave; exports.startForceSave = startForceSave; exports.resetForceSaveAfterChanges = resetForceSaveAfterChanges; +exports.getExternalChangeInfo = getExternalChangeInfo; exports.checkJwt = checkJwt; exports.getRequestParams = getRequestParams; exports.checkJwtHeader = checkJwtHeader; @@ -2561,7 +2567,8 @@ exports.install = function(server, callbackFunction) { // Автоматически снимаем lock сами и посылаем индекс для сохранения yield* unSaveLock(conn, changesIndex, newChangesLastTime); //last save - yield resetForceSaveAfterChanges(docId, newChangesLastTime, puckerIndex, utils.getBaseUrlByConnection(conn)); + let changeInfo = getExternalChangeInfo(conn.user, newChangesLastTime); + yield resetForceSaveAfterChanges(docId, newChangesLastTime, puckerIndex, utils.getBaseUrlByConnection(conn), changeInfo); } else { let changesToSend = arrNewDocumentChanges; if(changesToSend.length > cfgPubSubMaxChanges) { diff --git a/DocService/sources/baseConnector.js b/DocService/sources/baseConnector.js index 967a7925a..eafc0b6e6 100644 --- a/DocService/sources/baseConnector.js +++ b/DocService/sources/baseConnector.js @@ -317,22 +317,18 @@ exports.UserCallback = UserCallback; function DocumentPassword() { this.password = undefined; - this.userId = undefined; - this.userIndex = undefined; + this.change = undefined; } DocumentPassword.prototype.fromString = function(passwordStr){ var parsed = JSON.parse(passwordStr); - this.fromValues(parsed.password, parsed.userId, parsed.userIndex); + this.fromValues(parsed.password, parsed.change); }; -DocumentPassword.prototype.fromValues = function(password, userId, userIndex){ +DocumentPassword.prototype.fromValues = function(password, change){ if(null !== password){ this.password = password; } - if(null !== userId){ - this.userId = userId; - } - if(null !== userIndex){ - this.userIndex = userIndex; + if(null !== change) { + this.change = change; } }; DocumentPassword.prototype.delimiter = String.fromCharCode(5); @@ -340,10 +336,10 @@ DocumentPassword.prototype.toSQLInsert = function(){ return this.delimiter + JSON.stringify(this); }; DocumentPassword.prototype.isInitial = function(){ - return !this.userId; + return !this.change; }; DocumentPassword.prototype.getDocPassword = function(docId, docPasswordStr) { - let res = {initial: undefined, current: undefined, userId: undefined, userIndex: undefined}; + let res = {initial: undefined, current: undefined, change: undefined}; if (docPasswordStr) { logger.debug("getDocPassword: docId = %s passwords = %s", docId, docPasswordStr); let passwords = docPasswordStr.split(UserCallback.prototype.delimiter); @@ -353,10 +349,10 @@ DocumentPassword.prototype.getDocPassword = function(docId, docPasswordStr) { password.fromString(passwords[i]); if (password.isInitial()) { res.initial = password.password; + } else { + res.change = password.change; } res.current = password.password; - res.userId = password.userId; - res.userIndex = password.userIndex; } } return res; diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 8ae797243..9fd5f6afd 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -505,9 +505,8 @@ function* commandSfctByCmd(cmd, opt_priority, opt_expiration, opt_queue) { let docPassword = row && sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), row.password); if (docPassword.current) { cmd.setSavePassword(docPassword.current); - if (docPassword.userId) { - cmd.setUserId(docPassword.userId); - cmd.setUserIndex(docPassword.userIndex); + if (docPassword.change) { + cmd.setExternalChangeInfo(docPassword.change); } } var queueData = getSaveTask(cmd); @@ -714,12 +713,15 @@ function* commandSetPassword(conn, cmd, outputData) { updateMask.key = cmd.getDocId(); updateMask.status = taskResult.FileStatus.Ok; + let newChangesLastDate = new Date(); + newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding + var task = new taskResult.TaskResultData(); task.key = cmd.getDocId(); task.password = cmd.getPassword() || ""; + let changeInfo = null; if (conn.user) { - task.innerUserId = conn.user.idOriginal; - task.innerUserIndex = utils.getIndexFromUserId(conn.user.id, conn.user.idOriginal); + changeInfo = task.innerPasswordChange = docsCoServer.getExternalChangeInfo(conn.user, newChangesLastDate.getTime()); } var upsertRes = yield taskResult.updateIf(task, updateMask); @@ -728,9 +730,7 @@ function* commandSetPassword(conn, cmd, outputData) { if (!conn.isEnterCorrectPassword) { docsCoServer.modifyConnectionForPassword(conn, true); } - let newChangesLastDate = new Date(); - newChangesLastDate.setMilliseconds(0);//remove milliseconds avoid issues with MySQL datetime rounding - yield docsCoServer.resetForceSaveAfterChanges(cmd.getDocId(), newChangesLastDate.getTime(), 0x7FFFFFFF, utils.getBaseUrlByConnection(conn)); + yield docsCoServer.resetForceSaveAfterChanges(cmd.getDocId(), newChangesLastDate.getTime(), 0, utils.getBaseUrlByConnection(conn), changeInfo); } else { logger.debug('commandSetPassword sql update error: docId = %s', cmd.getDocId()); outputData.setStatus('err'); @@ -1305,9 +1305,8 @@ exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt let docPassword = row && sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), row.password); if (docPassword.current) { cmd.setSavePassword(docPassword.current); - if (docPassword.userId) { - cmd.setUserId(docPassword.userId); - cmd.setUserIndex(docPassword.userIndex); + if (docPassword.change) { + cmd.setExternalChangeInfo(docPassword.change); } } yield* addRandomKeyTaskCmd(cmd); diff --git a/DocService/sources/converterservice.js b/DocService/sources/converterservice.js index fef043a24..a539d1676 100644 --- a/DocService/sources/converterservice.js +++ b/DocService/sources/converterservice.js @@ -148,8 +148,8 @@ function* convertByCmd(cmd, async, baseUrl, opt_fileTo, opt_taskExist, opt_prior return status; } -function* convertFromChanges(docId, baseUrl, forceSave, opt_userdata, opt_userConnectionId, opt_responseKey, opt_priority, - opt_expiration, opt_queue, opt_redisKey) { +function* convertFromChanges(docId, baseUrl, forceSave, externalChangeInfo, opt_userdata, opt_userConnectionId, + opt_responseKey, opt_priority, opt_expiration, opt_queue, opt_redisKey) { var cmd = new commonDefines.InputCommand(); cmd.setCommand('sfcm'); cmd.setDocId(docId); @@ -158,6 +158,7 @@ function* convertFromChanges(docId, baseUrl, forceSave, opt_userdata, opt_userCo cmd.setCodepage(commonDefines.c_oAscCodePageUtf8); cmd.setDelimiter(commonDefines.c_oAscCsvDelimiter.Comma); cmd.setForceSave(forceSave); + cmd.setExternalChangeInfo(externalChangeInfo); if (opt_userdata) { cmd.setUserData(opt_userdata); } diff --git a/DocService/sources/editorDataMemory.js b/DocService/sources/editorDataMemory.js index 22b560339..6bd29434e 100644 --- a/DocService/sources/editorDataMemory.js +++ b/DocService/sources/editorDataMemory.js @@ -162,9 +162,9 @@ EditorData.prototype.getdelSaved = function(docId) { data.saved = undefined; return Promise.resolve(res); }; -EditorData.prototype.setForceSave = function(docId, time, index, baseUrl) { +EditorData.prototype.setForceSave = function(docId, time, index, baseUrl, changeInfo) { let data = this._getDocumentData(docId); - data.forceSave = {time: time, index: index, baseUrl: baseUrl, started: false, ended: false}; + data.forceSave = {time: time, index: index, baseUrl: baseUrl, changeInfo: changeInfo, started: false, ended: false}; return Promise.resolve(); }; EditorData.prototype.getForceSave = function(docId) { diff --git a/DocService/sources/taskresult.js b/DocService/sources/taskresult.js index 7146c45dc..708da59a1 100644 --- a/DocService/sources/taskresult.js +++ b/DocService/sources/taskresult.js @@ -67,8 +67,7 @@ function TaskResultData() { this.baseurl = null; this.password = null; - this.innerUserId = null;//not a DB field - this.innerUserIndex = null;//not a DB field + this.innerPasswordChange = null;//not a DB field } TaskResultData.prototype.completeDefaults = function() { if (!this.key) { @@ -118,7 +117,7 @@ function select(docId) { }, undefined, undefined, values); }); } -function toUpdateArray(task, updateTime, isMask, values) { +function toUpdateArray(task, updateTime, isMask, values, setPassword) { var res = []; if (null != task.status) { let sqlParam = addSqlParam(task.status, values); @@ -154,19 +153,22 @@ function toUpdateArray(task, updateTime, isMask, values) { let sqlParam = addSqlParam(task.baseurl, values); res.push(`baseurl=${sqlParam}`); } - if (null != task.password) { + if (setPassword) { + let sqlParam = addSqlParam(task.password, values); + res.push(`password=${sqlParam}`); + } else if (null != task.password || setPassword) { var documentPassword = new sqlBase.DocumentPassword(); - documentPassword.fromValues(task.password, task.innerUserId, task.innerUserIndex); + documentPassword.fromValues(task.password, task.innerPasswordChange); let sqlParam = addSqlParam(documentPassword.toSQLInsert(), values); res.push(`password=${concatParams('password', sqlParam)}`); } return res; } -function update(task) { +function update(task, setPassword) { return new Promise(function(resolve, reject) { let values = []; - let updateElems = toUpdateArray(task, true, false, values); + let updateElems = toUpdateArray(task, true, false, values, setPassword); let sqlSet = updateElems.join(', '); let sqlParam = addSqlParam(task.key, values); let sqlCommand = `UPDATE task_result SET ${sqlSet} WHERE id=${sqlParam};`; @@ -183,8 +185,8 @@ function update(task) { function updateIf(task, mask) { return new Promise(function(resolve, reject) { let values = []; - let commandArg = toUpdateArray(task, true, false, values); - let commandArgMask = toUpdateArray(mask, false, true, values); + let commandArg = toUpdateArray(task, true, false, values, false); + let commandArgMask = toUpdateArray(mask, false, true, values, false); commandArgMask.push('id=' + addSqlParam(mask.key, values)); let sqlSet = commandArg.join(', '); let sqlWhere = commandArgMask.join(' AND '); @@ -198,6 +200,25 @@ function updateIf(task, mask) { }, undefined, undefined, values); }); } +function restoreInitialPassword(docId) { + return select(docId).then(function(selectRes) { + if (selectRes.length > 0) { + var row = selectRes[0]; + let docPassword = sqlBase.DocumentPassword.prototype.getDocPassword(docId, row.password); + var updateTask = new TaskResultData(); + updateTask.key = docId; + if (docPassword.initial) { + var documentPassword = new sqlBase.DocumentPassword(); + documentPassword.fromValues(docPassword.initial); + updateTask.password = documentPassword.toSQLInsert(); + return update(updateTask, true); + } else if (docPassword.current) { + updateTask.password = null; + return update(updateTask, true); + } + } + }); +} function addRandomKey(task, opt_prefix, opt_size) { return new Promise(function(resolve, reject) { @@ -291,6 +312,7 @@ exports.upsert = upsert; exports.select = select; exports.update = update; exports.updateIf = updateIf; +exports.restoreInitialPassword = restoreInitialPassword; exports.addRandomKeyTask = addRandomKeyTask; exports.remove = remove; exports.getExpired = getExpired; diff --git a/FileConverter/sources/converter.js b/FileConverter/sources/converter.js index ca71c5cad..8ac3df229 100644 --- a/FileConverter/sources/converter.js +++ b/FileConverter/sources/converter.js @@ -407,11 +407,28 @@ function* processChanges(tempDirs, cmd, authorProps) { forceSaveTime = forceSave.getTime(); forceSaveIndex = forceSave.getIndex(); } + let extChangeInfo = cmd.getExternalChangeInfo(); + let extChanges; + if (extChangeInfo) { + extChanges = [{ + id: cmd.getDocId(), change_id: 0, change_data: "", user_id: extChangeInfo.user_id, + user_id_original: extChangeInfo.user_id_original, user_name: extChangeInfo.user_name, + change_date: new Date(extChangeInfo.change_date) + }]; + } + let streamObj = yield* streamCreate(cmd.getDocId(), changesDir, indexFile++, {highWaterMark: cfgStreamWriterBufferSize}); let curIndexStart = 0; let curIndexEnd = Math.min(curIndexStart + cfgMaxRequestChanges, forceSaveIndex); - while (curIndexStart < curIndexEnd) { - let changes = yield baseConnector.getChangesPromise(cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime); + while (curIndexStart < curIndexEnd || extChanges) { + let changes = []; + if (curIndexStart < curIndexEnd) { + changes = yield baseConnector.getChangesPromise(cmd.getDocId(), curIndexStart, curIndexEnd, forceSaveTime); + } + if (0 === changes.length && extChanges) { + changes = extChanges; + } + extChanges = undefined; for (let i = 0; i < changes.length; ++i) { let change = changes[i]; if (change.change_data.startsWith('ENCRYPTED;')) { @@ -457,10 +474,8 @@ function* processChanges(tempDirs, cmd, authorProps) { changesAuthor = forceSave.getAuthorUserId(); changesIndex = forceSave.getAuthorUserIndex(); } - if (null != changesAuthor && null != changesIndex) { - cmd.setUserId(changesAuthor); - cmd.setUserIndex(changesIndex); - } + cmd.setUserId(changesAuthor); + cmd.setUserIndex(changesIndex); fs.writeFileSync(path.join(tempDirs.result, 'changesHistory.json'), JSON.stringify(changesHistory), 'utf8'); return res; } From 6c78acf883931b8c9e7b89f7194a93c65e8bd188 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 4 May 2021 15:45:24 +0300 Subject: [PATCH 17/29] [bug] For 57657dcaad6b0b60ffb051fe9e047e3adea670c0 (concat null in postgres) --- DocService/sources/postgreSqlBaseConnector.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocService/sources/postgreSqlBaseConnector.js b/DocService/sources/postgreSqlBaseConnector.js index 0c54b5d0d..204af6690 100644 --- a/DocService/sources/postgreSqlBaseConnector.js +++ b/DocService/sources/postgreSqlBaseConnector.js @@ -96,7 +96,7 @@ let addSqlParam = function (val, values) { }; exports.addSqlParameter = addSqlParam; let concatParams = function (val1, val2) { - return `${val1} || ${val2}`; + return `COALESCE(${val1}, '') || COALESCE(${val2}, '')`; }; exports.concatParams = concatParams; var isSupportOnConflict = true; From b3a1dc7fbeb1c597233f368066d3c66d6e8bc00b Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 12 May 2021 15:09:36 +0300 Subject: [PATCH 18/29] [bug] Move from cron to setTimeout to avoid "uncaughtException: Something went wrong. cron reached maximum iterations" --- DocService/sources/DocsCoServer.js | 16 +++++---------- DocService/sources/gc.js | 31 ++++++++++++++---------------- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index f62f11ca0..688f3c780 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -78,7 +78,6 @@ const url = require('url'); const os = require('os'); const cluster = require('cluster'); const crypto = require('crypto'); -const cron = require('cron'); const co = require('co'); const jwt = require('jsonwebtoken'); const jwa = require('jwa'); @@ -165,6 +164,7 @@ let pubsub; let queue; let licenseInfo = {type: constants.LICENSE_RESULT.Error, light: false, branding: false, customization: false, plugins: false}; let shutdownFlag = false; +let expDocumentsStep = gc.getCronStep(cfgExpDocumentsCron); const MIN_SAVE_EXPIRATION = 60000; const FORCE_SAVE_EXPIRATION = Math.min(Math.max(cfgForceSaveInterval, MIN_SAVE_EXPIRATION), @@ -3011,15 +3011,13 @@ exports.install = function(server, callbackFunction) { yield editorData.setEditorConnections(countEdit, countView, now, PRECISION); } function expireDoc() { - var cronJob = this; return co(function* () { try { var countEditByShard = 0; var countViewByShard = 0; logger.debug('expireDoc connections.length = %d', connections.length); var nowMs = new Date().getTime(); - var nextMs = cronJob.nextDate(); - var maxMs = Math.max(nowMs + cfgExpSessionCloseCommand, nextMs); + var maxMs = nowMs + Math.max(cfgExpSessionCloseCommand, expDocumentsStep); for (var i = 0; i < connections.length; ++i) { var conn = connections[i]; if (cfgExpSessionAbsolute > 0) { @@ -3068,16 +3066,12 @@ exports.install = function(server, callbackFunction) { } } catch (err) { logger.error('expireDoc error:\r\n%s', err.stack); + } finally { + setTimeout(expireDoc, expDocumentsStep); } }); } - var innerPingJob = function(opt_isStart) { - if (!opt_isStart) { - logger.warn('expireDoc restart'); - } - new cron.CronJob(cfgExpDocumentsCron, expireDoc, innerPingJob, true); - }; - innerPingJob(true); + setTimeout(expireDoc, expDocumentsStep); pubsub = new pubsubService(); pubsub.on('message', pubsubOnMessage); diff --git a/DocService/sources/gc.js b/DocService/sources/gc.js index f8ca3fe4d..bff66a8d3 100644 --- a/DocService/sources/gc.js +++ b/DocService/sources/gc.js @@ -47,7 +47,6 @@ var constants = require('./../../Common/sources/constants'); var commondefines = require('./../../Common/sources/commondefines'); var queueService = require('./../../Common/sources/taskqueueRabbitMQ'); var pubsubService = require('./pubsubRabbitMQ'); -const editorDataStorage = require('./' + configCommon.get('services.CoAuthoring.server.editorDataStorage')); var cfgExpFilesCron = config.get('expire.filesCron'); var cfgExpDocumentsCron = config.get('expire.documentsCron'); @@ -56,6 +55,14 @@ var cfgExpFilesRemovedAtOnce = config.get('expire.filesremovedatonce'); var cfgForceSaveEnable = config.get('autoAssembly.enable'); var cfgForceSaveStep = ms(config.get('autoAssembly.step')); +function getCronStep(cronTime){ + let cronJob = new cron.CronJob(cronTime, function(){}); + let dates = cronJob.nextDates(2); + return dates[1] - dates[0]; +} +let expFilesStep = getCronStep(cfgExpFilesCron); +let expDocumentsStep = getCronStep(cfgExpDocumentsCron); + var checkFileExpire = function() { return co(function* () { try { @@ -83,6 +90,8 @@ var checkFileExpire = function() { logger.debug('checkFileExpire end: removedCount = %d', removedCount); } catch (e) { logger.error('checkFileExpire error:\r\n%s', e.stack); + } finally { + setTimeout(checkFileExpire, expFilesStep); } }); }; @@ -124,6 +133,7 @@ var checkDocumentExpire = function() { logger.error('checkDocumentExpire error:\r\n%s', e.stack); } logger.debug('checkDocumentExpire end: startSaveCount = %d, removedCount = %d', startSaveCount, removedCount); + setTimeout(checkDocumentExpire, expDocumentsStep); } }); }; @@ -172,24 +182,11 @@ let forceSaveTimeout = function() { }); }; -var documentExpireJob = function(opt_isStart) { - if (!opt_isStart) { - logger.warn('checkDocumentExpire restart'); - } - new cron.CronJob(cfgExpDocumentsCron, checkDocumentExpire, documentExpireJob, true); -}; - -var fileExpireJob = function(opt_isStart) { - if (!opt_isStart) { - logger.warn('checkFileExpire restart'); - } - new cron.CronJob(cfgExpFilesCron, checkFileExpire, fileExpireJob, true); -}; - exports.startGC = function() { - documentExpireJob(true); - fileExpireJob(true); + setTimeout(checkDocumentExpire, expDocumentsStep); + setTimeout(checkFileExpire, expFilesStep); if (cfgForceSaveEnable) { setTimeout(forceSaveTimeout, cfgForceSaveStep); } }; +exports.getCronStep = getCronStep; From 27e3f5e184c0f30efe97b7585fdfaefab26dea9a Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Tue, 18 May 2021 16:00:36 +0300 Subject: [PATCH 19/29] [bug] Fix bug 50466 --- Common/sources/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index a80ce1fc1..163ef83ed 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -127,7 +127,7 @@ exports.encodeXml = function(value) { case '\r': return ' '; case '\n': return ' '; case '\t': return ' '; - case '\xA0': return '&#A0;'; + case '\xA0': return ' '; } }); }; From c9f081748f156d9e2477241f265556a6b56ffed3 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 19 May 2021 14:27:53 +0300 Subject: [PATCH 20/29] [bug] Fix bug 50481 --- DocService/sources/canvasservice.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 9fd5f6afd..9558baaca 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -505,9 +505,9 @@ function* commandSfctByCmd(cmd, opt_priority, opt_expiration, opt_queue) { let docPassword = row && sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), row.password); if (docPassword.current) { cmd.setSavePassword(docPassword.current); - if (docPassword.change) { - cmd.setExternalChangeInfo(docPassword.change); - } + } + if (docPassword.change) { + cmd.setExternalChangeInfo(docPassword.change); } var queueData = getSaveTask(cmd); queueData.setFromChanges(true); @@ -1305,9 +1305,9 @@ exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt let docPassword = row && sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), row.password); if (docPassword.current) { cmd.setSavePassword(docPassword.current); - if (docPassword.change) { - cmd.setExternalChangeInfo(docPassword.change); - } + } + if (docPassword.change) { + cmd.setExternalChangeInfo(docPassword.change); } yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); From 4bf6fc88aebe07ee8df62c935150968c01cbec60 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 19 May 2021 16:14:00 +0300 Subject: [PATCH 21/29] [bug] For bug 50481 --- DocService/sources/canvasservice.js | 30 ++++++++++++----------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 9558baaca..1b3b34001 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -255,6 +255,15 @@ function* addRandomKeyTaskCmd(cmd) { var task = yield* taskResult.addRandomKeyTask(cmd.getDocId()); cmd.setSaveKey(task.key); } +function addPasswordToCmd(cmd, docPasswordStr) { + let docPassword = sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), docPasswordStr); + if (docPassword.current) { + cmd.setSavePassword(docPassword.current); + } + if (docPassword.change) { + cmd.setExternalChangeInfo(docPassword.change); + } +} function* saveParts(cmd, filename) { var result = false; var saveType = cmd.getSaveType(); @@ -502,13 +511,7 @@ function* commandSfctByCmd(cmd, opt_priority, opt_expiration, opt_queue) { yield* addRandomKeyTaskCmd(cmd); var selectRes = yield taskResult.select(cmd.getDocId()); var row = selectRes.length > 0 ? selectRes[0] : null; - let docPassword = row && sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), row.password); - if (docPassword.current) { - cmd.setSavePassword(docPassword.current); - } - if (docPassword.change) { - cmd.setExternalChangeInfo(docPassword.change); - } + addPasswordToCmd(cmd, row && row.password); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); let priority = null != opt_priority ? opt_priority : constants.QUEUE_PRIORITY_LOW; @@ -1184,10 +1187,7 @@ exports.downloadAs = function(req, res) { } var selectRes = yield taskResult.select(docId); var row = selectRes.length > 0 ? selectRes[0] : null; - let password = row && sqlBase.DocumentPassword.prototype.getCurPassword(cmd.getDocId(), row.password); - if (password && !cmd.getWithoutPassword()) { - cmd.setSavePassword(password); - } + addPasswordToCmd(cmd, row && row.password); cmd.setData(req.body); var outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { @@ -1302,13 +1302,7 @@ exports.saveFromChanges = function(docId, statusInfo, optFormat, opt_userId, opt cmd.setStatusInfoIn(statusInfo); cmd.setUserActionId(opt_userId); cmd.setUserActionIndex(opt_userIndex); - let docPassword = row && sqlBase.DocumentPassword.prototype.getDocPassword(cmd.getDocId(), row.password); - if (docPassword.current) { - cmd.setSavePassword(docPassword.current); - } - if (docPassword.change) { - cmd.setExternalChangeInfo(docPassword.change); - } + addPasswordToCmd(cmd, row && row.password); yield* addRandomKeyTaskCmd(cmd); var queueData = getSaveTask(cmd); queueData.setFromChanges(true); From 36b1588f9fdddc1a593ac6d58fb429118ae39d9e Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Fri, 7 May 2021 15:51:33 +0300 Subject: [PATCH 22/29] [feature] Add 'license' command to CommandService.ashx --- DocService/sources/DocsCoServer.js | 27 ++++++++++++++++++++++---- DocService/sources/editorDataMemory.js | 5 ++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 688f3c780..69de7dc26 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -3223,12 +3223,25 @@ exports.licenseInfo = function(req, res) { } }); }; +let commandLicense = co.wrap(function*() { + let res = {license: {}, server: null, quota: {}}; + Object.assign(res.license, licenseInfo); + res.server = {buildVersion: commonDefines.buildVersion, buildNumber: commonDefines.buildNumber}; + const nowUTC = getLicenseNowUtc(); + let scores = []; + let execRes = yield editorData.getPresenceUniqueUser(nowUTC, scores); + execRes.forEach(function(currentValue, index){ + res.quota[currentValue] = new Date(scores[index] * 1000); + }); + return res; +}); // Команда с сервера (в частности teamlab) exports.commandFromServer = function (req, res) { return co(function* () { let result = commonDefines.c_oAscServerCommandErrors.NoError; let docId = 'commandFromServer'; let version = undefined; + let outputLicense = undefined; try { let authRes = getRequestParams(docId, req); let params = authRes.params; @@ -3239,7 +3252,7 @@ exports.commandFromServer = function (req, res) { } // Ключ id-документа docId = params.key; - if (commonDefines.c_oAscServerCommandErrors.NoError === result && null == docId && 'version' != params.c) { + if (commonDefines.c_oAscServerCommandErrors.NoError === result && null == docId && 'version' != params.c && 'license' != params.c) { result = commonDefines.c_oAscServerCommandErrors.DocumentIdError; } else if(commonDefines.c_oAscServerCommandErrors.NoError === result) { logger.debug('Start commandFromServer: docId = %s c = %s', docId, params.c); @@ -3287,6 +3300,9 @@ exports.commandFromServer = function (req, res) { case 'version': version = commonDefines.buildVersion + '.' + commonDefines.buildNumber; break; + case 'license': + outputLicense = yield commandLicense(); + break; default: result = commonDefines.c_oAscServerCommandErrors.UnknownCommand; break; @@ -3297,9 +3313,12 @@ exports.commandFromServer = function (req, res) { logger.error('Error commandFromServer: docId = %s\r\n%s', docId, err.stack); } finally { //undefined value are excluded in JSON.stringify - const output = JSON.stringify({'key': docId, 'error': result, 'version': version}); - logger.debug('End commandFromServer: docId = %s %s', docId, output); - const outputBuffer = Buffer.from(output, 'utf8'); + let output = {'key': docId, 'error': result, 'version': version}; + if (outputLicense) { + Object.assign(output, outputLicense); + } + logger.debug('End commandFromServer: docId = %s %j', docId, output); + const outputBuffer = Buffer.from(JSON.stringify(output), 'utf8'); res.setHeader('Content-Type', 'application/json'); res.setHeader('Content-Length', outputBuffer.length); res.send(outputBuffer); diff --git a/DocService/sources/editorDataMemory.js b/DocService/sources/editorDataMemory.js index 6bd29434e..2915c4545 100644 --- a/DocService/sources/editorDataMemory.js +++ b/DocService/sources/editorDataMemory.js @@ -226,12 +226,15 @@ EditorData.prototype.addPresenceUniqueUser = function(userId, expireAt) { this.uniqueUser[userId] = expireAt; return Promise.resolve(); }; -EditorData.prototype.getPresenceUniqueUser = function(nowUTC) { +EditorData.prototype.getPresenceUniqueUser = function(nowUTC, opt_scores) { let res = []; for (let userId in this.uniqueUser) { if (this.uniqueUser.hasOwnProperty(userId)) { if (this.uniqueUser[userId] > nowUTC) { res.push(userId); + if(opt_scores) { + opt_scores.push(this.uniqueUser[userId]); + } } else { delete this.uniqueUser[userId]; } From ae9c548b4822fa9d43bed38f1a82d69ee8b73fff Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Wed, 12 May 2021 09:55:28 +0300 Subject: [PATCH 23/29] [feature] Rework 'license' command params of CommandSevice --- Common/sources/license.js | 45 ++++++++++++++++++++++++++++++ DocService/sources/DocsCoServer.js | 7 ++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/Common/sources/license.js b/Common/sources/license.js index 19f0b61e4..6e43f4dd3 100644 --- a/Common/sources/license.js +++ b/Common/sources/license.js @@ -33,6 +33,7 @@ 'use strict'; const constants = require('./constants'); +const commonDefines = require('./commonDefines'); const buildDate = '6/29/2016'; const oBuildDate = new Date(buildDate); @@ -56,4 +57,48 @@ exports.readLicense = function*() { endDate: null }; }; +exports.convertToFileParams = function(licenseInfo) { + // todo + // { + // user_quota = 0; + // portal_count = 0; + // process = 2; + // ssbranding = false; + // whiteLabel = false; + // } + let license = {}; + license.end_date = licenseInfo.endDate && licenseInfo.endDate.toJSON(); + license.trial = constants.LICENSE_MODE.Trial === licenseInfo.mode; + license.developer = constants.LICENSE_MODE.Developer === licenseInfo.mode; + switch (licenseInfo.mode) { + case constants.LICENSE_MODE.Developer: + license.mode = 'developer'; + break; + case constants.LICENSE_MODE.Trial: + license.mode = 'trial'; + break; + default: + license.mode = ''; + break; + } + license.light = licenseInfo.light; + license.branding = licenseInfo.branding; + license.customization = licenseInfo.customization; + license.plugins = licenseInfo.plugins; + license.connections = licenseInfo.connections; + license.users_count = licenseInfo.usersCount; + license.users_expire = licenseInfo.usersExpire / constants.LICENSE_EXPIRE_USERS_ONE_DAY; + return license; +}; +exports.convertToServerParams = function(licenseInfo) { + let license = {}; + license.workersCount = licenseInfo.count; + license.resultType = licenseInfo.type; + license.packageType = licenseInfo.packageType; + license.buildDate = licenseInfo.buildDate && licenseInfo.buildDate.toJSON(); + license.buildVersion = commonDefines.buildVersion; + license.buildNumber = commonDefines.buildNumber; + return license; +}; + exports.packageType = constants.PACKAGE_TYPE_OS; diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 69de7dc26..1b40dab2f 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -90,6 +90,7 @@ const constants = require('./../../Common/sources/constants'); const utils = require('./../../Common/sources/utils'); const commonDefines = require('./../../Common/sources/commondefines'); const statsDClient = require('./../../Common/sources/statsdclient'); +const license = require('./../../Common/sources/license'); const configCommon = require('config'); const config = configCommon.get('services.CoAuthoring'); const sqlBase = require('./baseConnector'); @@ -3224,9 +3225,7 @@ exports.licenseInfo = function(req, res) { }); }; let commandLicense = co.wrap(function*() { - let res = {license: {}, server: null, quota: {}}; - Object.assign(res.license, licenseInfo); - res.server = {buildVersion: commonDefines.buildVersion, buildNumber: commonDefines.buildNumber}; + let res = {license: license.convertToFileParams(licenseInfo), server: license.convertToServerParams(licenseInfo), quota: {}}; const nowUTC = getLicenseNowUtc(); let scores = []; let execRes = yield editorData.getPresenceUniqueUser(nowUTC, scores); @@ -3252,7 +3251,7 @@ exports.commandFromServer = function (req, res) { } // Ключ id-документа docId = params.key; - if (commonDefines.c_oAscServerCommandErrors.NoError === result && null == docId && 'version' != params.c && 'license' != params.c) { + if (commonDefines.c_oAscServerCommandErrors.NoError === result && null == docId && 'version' !== params.c && 'license' !== params.c) { result = commonDefines.c_oAscServerCommandErrors.DocumentIdError; } else if(commonDefines.c_oAscServerCommandErrors.NoError === result) { logger.debug('Start commandFromServer: docId = %s c = %s', docId, params.c); From 864752b1636f4188b13eb7fec7669dfe8bb92f66 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 13 May 2021 13:40:47 +0300 Subject: [PATCH 24/29] [bug] Move convertToFileParams, convertToServerParams from replaceable license.js # Conflicts: # Common/sources/utils.js --- Common/sources/license.js | 44 ----------------------------- Common/sources/utils.js | 45 ++++++++++++++++++++++++++++++ DocService/sources/DocsCoServer.js | 2 +- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/Common/sources/license.js b/Common/sources/license.js index 6e43f4dd3..ea257bdcf 100644 --- a/Common/sources/license.js +++ b/Common/sources/license.js @@ -33,7 +33,6 @@ 'use strict'; const constants = require('./constants'); -const commonDefines = require('./commonDefines'); const buildDate = '6/29/2016'; const oBuildDate = new Date(buildDate); @@ -57,48 +56,5 @@ exports.readLicense = function*() { endDate: null }; }; -exports.convertToFileParams = function(licenseInfo) { - // todo - // { - // user_quota = 0; - // portal_count = 0; - // process = 2; - // ssbranding = false; - // whiteLabel = false; - // } - let license = {}; - license.end_date = licenseInfo.endDate && licenseInfo.endDate.toJSON(); - license.trial = constants.LICENSE_MODE.Trial === licenseInfo.mode; - license.developer = constants.LICENSE_MODE.Developer === licenseInfo.mode; - switch (licenseInfo.mode) { - case constants.LICENSE_MODE.Developer: - license.mode = 'developer'; - break; - case constants.LICENSE_MODE.Trial: - license.mode = 'trial'; - break; - default: - license.mode = ''; - break; - } - license.light = licenseInfo.light; - license.branding = licenseInfo.branding; - license.customization = licenseInfo.customization; - license.plugins = licenseInfo.plugins; - license.connections = licenseInfo.connections; - license.users_count = licenseInfo.usersCount; - license.users_expire = licenseInfo.usersExpire / constants.LICENSE_EXPIRE_USERS_ONE_DAY; - return license; -}; -exports.convertToServerParams = function(licenseInfo) { - let license = {}; - license.workersCount = licenseInfo.count; - license.resultType = licenseInfo.type; - license.packageType = licenseInfo.packageType; - license.buildDate = licenseInfo.buildDate && licenseInfo.buildDate.toJSON(); - license.buildVersion = commonDefines.buildVersion; - license.buildNumber = commonDefines.buildNumber; - return license; -}; exports.packageType = constants.PACKAGE_TYPE_OS; diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 163ef83ed..9dd489f83 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -54,6 +54,7 @@ const jwt = require('jsonwebtoken'); const NodeCache = require( "node-cache" ); const ms = require('ms'); const constants = require('./constants'); +const commonDefines = require('./commondefines'); const logger = require('./logger'); const forwarded = require('forwarded'); const mime = require('mime'); @@ -868,3 +869,47 @@ exports.decryptPassword = co.wrap(function* (password) { const { data: decrypted } = yield openpgp.decrypt(params); return decrypted; }); + +exports.convertLicenseInfoToFileParams = function(licenseInfo) { + // todo + // { + // user_quota = 0; + // portal_count = 0; + // process = 2; + // ssbranding = false; + // whiteLabel = false; + // } + let license = {}; + license.end_date = licenseInfo.endDate && licenseInfo.endDate.toJSON(); + license.trial = constants.LICENSE_MODE.Trial === licenseInfo.mode; + license.developer = constants.LICENSE_MODE.Developer === licenseInfo.mode; + switch (licenseInfo.mode) { + case constants.LICENSE_MODE.Developer: + license.mode = 'developer'; + break; + case constants.LICENSE_MODE.Trial: + license.mode = 'trial'; + break; + default: + license.mode = ''; + break; + } + license.light = licenseInfo.light; + license.branding = licenseInfo.branding; + license.customization = licenseInfo.customization; + license.plugins = licenseInfo.plugins; + license.connections = licenseInfo.connections; + license.users_count = licenseInfo.usersCount; + license.users_expire = licenseInfo.usersExpire / constants.LICENSE_EXPIRE_USERS_ONE_DAY; + return license; +}; +exports.convertLicenseInfoToServerParams = function(licenseInfo) { + let license = {}; + license.workersCount = licenseInfo.count; + license.resultType = licenseInfo.type; + license.packageType = licenseInfo.packageType; + license.buildDate = licenseInfo.buildDate && licenseInfo.buildDate.toJSON(); + license.buildVersion = commonDefines.buildVersion; + license.buildNumber = commonDefines.buildNumber; + return license; +}; diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index 1b40dab2f..f8da91c0d 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -3225,7 +3225,7 @@ exports.licenseInfo = function(req, res) { }); }; let commandLicense = co.wrap(function*() { - let res = {license: license.convertToFileParams(licenseInfo), server: license.convertToServerParams(licenseInfo), quota: {}}; + let res = {license: utils.convertLicenseInfoToFileParams(licenseInfo), server: utils.convertLicenseInfoToServerParams(licenseInfo), quota: {}}; const nowUTC = getLicenseNowUtc(); let scores = []; let execRes = yield editorData.getPresenceUniqueUser(nowUTC, scores); From 1772605ca76304da92d4c92a2479b97fef0457d6 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Sun, 16 May 2021 23:36:18 +0300 Subject: [PATCH 25/29] [feature] Rework 'license.quota' command params of CommandSevice --- DocService/sources/DocsCoServer.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/DocService/sources/DocsCoServer.js b/DocService/sources/DocsCoServer.js index f8da91c0d..03655108f 100644 --- a/DocService/sources/DocsCoServer.js +++ b/DocService/sources/DocsCoServer.js @@ -3225,12 +3225,15 @@ exports.licenseInfo = function(req, res) { }); }; let commandLicense = co.wrap(function*() { - let res = {license: utils.convertLicenseInfoToFileParams(licenseInfo), server: utils.convertLicenseInfoToServerParams(licenseInfo), quota: {}}; + let res = { + license: utils.convertLicenseInfoToFileParams(licenseInfo), + server: utils.convertLicenseInfoToServerParams(licenseInfo), quota: {users: []} + }; const nowUTC = getLicenseNowUtc(); let scores = []; let execRes = yield editorData.getPresenceUniqueUser(nowUTC, scores); - execRes.forEach(function(currentValue, index){ - res.quota[currentValue] = new Date(scores[index] * 1000); + execRes.forEach(function(currentValue, index) { + res.quota.users.push({userid: currentValue, expire: new Date(scores[index] * 1000)}); }); return res; }); From 279f6514dd00fe47760a5a207f41d35c9af5f0c5 Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Mon, 24 May 2021 11:53:15 +0300 Subject: [PATCH 26/29] [bug] Fix bug 49716 --- DocService/sources/canvasservice.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DocService/sources/canvasservice.js b/DocService/sources/canvasservice.js index 1b3b34001..887a8e4b5 100644 --- a/DocService/sources/canvasservice.js +++ b/DocService/sources/canvasservice.js @@ -1187,7 +1187,9 @@ exports.downloadAs = function(req, res) { } var selectRes = yield taskResult.select(docId); var row = selectRes.length > 0 ? selectRes[0] : null; - addPasswordToCmd(cmd, row && row.password); + if (!cmd.getWithoutPassword()) { + addPasswordToCmd(cmd, row && row.password); + } cmd.setData(req.body); var outputData = new OutputData(cmd.getCommand()); switch (cmd.getCommand()) { From f0df6e670e987c785a860064d3b5f4a3602c53ce Mon Sep 17 00:00:00 2001 From: Alexander Trofimov Date: Wed, 26 May 2021 17:07:05 +0300 Subject: [PATCH 27/29] [run] Fix (#266) Fix running-document-server task --- Common/config/development-linux.json | 28 +++++++++++----------- Common/config/development-mac.json | 32 +++++++++++++------------- Common/config/development-windows.json | 30 ++++++++++++------------ 3 files changed, 45 insertions(+), 45 deletions(-) diff --git a/Common/config/development-linux.json b/Common/config/development-linux.json index d20dbe5eb..233af649f 100644 --- a/Common/config/development-linux.json +++ b/Common/config/development-linux.json @@ -1,10 +1,10 @@ { "log": { - "filePath": "../../Common/config/log4js/development.json" + "filePath": "../Common/config/log4js/development.json" }, "storage": { "fs": { - "folderPath": "../../App_Data" + "folderPath": "../App_Data" } }, "services": { @@ -13,19 +13,19 @@ "port": 8001, "static_content": { "/fonts": { - "path": "../../../fonts" + "path": "../../fonts" }, "/sdkjs": { - "path": "../../../sdkjs" + "path": "../../sdkjs" }, "/web-apps": { - "path": "../../../web-apps" + "path": "../../web-apps" }, "/sdkjs-plugins": { - "path": "../../../sdkjs-plugins" + "path": "../../sdkjs-plugins" }, "/dictionaries": { - "path": "../../../dictionaries" + "path": "../../dictionaries" }, "/info": { "path": "../branding/info" @@ -45,23 +45,23 @@ } }, "license": { - "license_file": "./../../license.lic", + "license_file": "./../license.lic", "warning_limit_percents": 70, "packageType": 0 }, "FileConverter": { "converter": { "fontDir": "/usr/share/fonts", - "presentationThemesDir": "../../../sdkjs/slide/themes", - "x2tPath": "../../FileConverter/bin/x2t", - "docbuilderPath": "../../FileConverter/bin/docbuilder", - "docbuilderAllFontsPath": "../../App_Data/docbuilder/AllFonts.js", - "docbuilderCoreFontsPath": "../../../core-fonts" + "presentationThemesDir": "../../sdkjs/slide/themes", + "x2tPath": "../FileConverter/bin/x2t", + "docbuilderPath": "../FileConverter/bin/docbuilder", + "docbuilderAllFontsPath": "../App_Data/docbuilder/AllFonts.js", + "docbuilderCoreFontsPath": "../../core-fonts" } }, "SpellChecker": { "server": { - "dictDir": "../../../dictionaries" + "dictDir": "../../dictionaries" } } } diff --git a/Common/config/development-mac.json b/Common/config/development-mac.json index 8ba54d7dc..181734c53 100644 --- a/Common/config/development-mac.json +++ b/Common/config/development-mac.json @@ -1,10 +1,10 @@ { "log": { - "filePath": "../../Common/config/log4js/development.json" + "filePath": "../Common/config/log4js/development.json" }, "storage": { "fs": { - "folderPath": "../../App_Data" + "folderPath": "../App_Data" } }, "services": { @@ -13,22 +13,22 @@ "port": 8001, "static_content": { "/fonts": { - "path": "../../../fonts" + "path": "../../fonts" }, "/sdkjs": { - "path": "../../../sdkjs" + "path": "../../sdkjs" }, "/web-apps": { - "path": "../../../web-apps" + "path": "../../web-apps" }, "/sdkjs-plugins": { - "path": "../../../sdkjs-plugins" + "path": "../../sdkjs-plugins" }, "/dictionaries": { - "path": "../../../dictionaries" + "path": "../../dictionaries" }, "/info": { - "path": "../../branding/info" + "path": "../branding/info" } } }, @@ -46,29 +46,29 @@ "allowMetaIPAddress": true }, "sockjs": { - "sockjs_url": "/office/vendor/sockjs/sockjs.min.js" + "sockjs_url": "/web-apps/vendor/sockjs/sockjs.min.js" } } }, "license": { - "license_file": "./../../license.lic", + "license_file": "./../license.lic", "warning_limit_percents": 70, "packageType": 0 }, "FileConverter": { "converter": { "fontDir": "", - "presentationThemesDir": "../../../OfficeWeb/PowerPoint/themes", - "x2tPath": "../../FileConverter/bin/x2t", - "docbuilderPath": "../../FileConverter/Bin/docbuilder", - "docbuilderAllFontsPath": "../../App_Data/docbuilder/AllFonts.js", - "docbuilderCoreFontsPath": "../../../core-fonts", + "presentationThemesDir": "../../sdkjs/slide/themes", + "x2tPath": "../FileConverter/bin/x2t", + "docbuilderPath": "../FileConverter/Bin/docbuilder", + "docbuilderAllFontsPath": "../App_Data/docbuilder/AllFonts.js", + "docbuilderCoreFontsPath": "../../core-fonts", "errorfiles": "error" } }, "SpellChecker": { "server": { - "dictDir": "../../../dictionaries" + "dictDir": "../../dictionaries" } } } \ No newline at end of file diff --git a/Common/config/development-windows.json b/Common/config/development-windows.json index 2647b67a2..79cad2d12 100644 --- a/Common/config/development-windows.json +++ b/Common/config/development-windows.json @@ -1,10 +1,10 @@ { "log": { - "filePath": "../../Common/config/log4js/development.json" + "filePath": "../Common/config/log4js/development.json" }, "storage": { "fs": { - "folderPath": "../../App_Data" + "folderPath": "../App_Data" } }, "services": { @@ -13,22 +13,22 @@ "port": 8001, "static_content": { "/fonts": { - "path": "../../../fonts" + "path": "../../fonts" }, "/sdkjs": { - "path": "../../../sdkjs" + "path": "../../sdkjs" }, "/web-apps": { - "path": "../../../web-apps" + "path": "../../web-apps" }, "/sdkjs-plugins": { - "path": "../../../sdkjs-plugins" + "path": "../../sdkjs-plugins" }, "/dictionaries": { - "path": "../../../dictionaries" + "path": "../../dictionaries" }, "/info": { - "path": "../../branding/info" + "path": "../branding/info" } } }, @@ -51,24 +51,24 @@ } }, "license": { - "license_file": "./../../license.lic", + "license_file": "./../license.lic", "warning_limit_percents": 70, "packageType": 0 }, "FileConverter": { "converter": { "fontDir": "", - "presentationThemesDir": "../../../sdkjs/slide/themes", - "x2tPath": "../../FileConverter/Bin/x2t.exe", - "docbuilderPath": "../../FileConverter/Bin/docbuilder.exe", - "docbuilderAllFontsPath": "../../App_Data/docbuilder/AllFonts.js", - "docbuilderCoreFontsPath": "../../../core-fonts", + "presentationThemesDir": "../../sdkjs/slide/themes", + "x2tPath": "../FileConverter/Bin/x2t.exe", + "docbuilderPath": "../FileConverter/Bin/docbuilder.exe", + "docbuilderAllFontsPath": "../App_Data/docbuilder/AllFonts.js", + "docbuilderCoreFontsPath": "../../core-fonts", "errorfiles": "error" } }, "SpellChecker": { "server": { - "dictDir": "../../../dictionaries" + "dictDir": "../../dictionaries" } } } \ No newline at end of file From ae1c10ffc60b2af983fcbcc56665287e01387598 Mon Sep 17 00:00:00 2001 From: Alexander Trofimov Date: Mon, 31 May 2021 09:26:20 +0300 Subject: [PATCH 28/29] [run] Port (#267) Change develop port 8001 -> 8000 --- Common/config/development-linux.json | 2 +- Common/config/development-mac.json | 2 +- Common/config/development-windows.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Common/config/development-linux.json b/Common/config/development-linux.json index 233af649f..26de849df 100644 --- a/Common/config/development-linux.json +++ b/Common/config/development-linux.json @@ -10,7 +10,7 @@ "services": { "CoAuthoring": { "server": { - "port": 8001, + "port": 8000, "static_content": { "/fonts": { "path": "../../fonts" diff --git a/Common/config/development-mac.json b/Common/config/development-mac.json index 181734c53..30adaa9c7 100644 --- a/Common/config/development-mac.json +++ b/Common/config/development-mac.json @@ -10,7 +10,7 @@ "services": { "CoAuthoring": { "server": { - "port": 8001, + "port": 8000, "static_content": { "/fonts": { "path": "../../fonts" diff --git a/Common/config/development-windows.json b/Common/config/development-windows.json index 79cad2d12..4c87b9527 100644 --- a/Common/config/development-windows.json +++ b/Common/config/development-windows.json @@ -10,7 +10,7 @@ "services": { "CoAuthoring": { "server": { - "port": 8001, + "port": 8000, "static_content": { "/fonts": { "path": "../../fonts" From ce86c02a1aaef7dbfb69d13ea997837829a6c08a Mon Sep 17 00:00:00 2001 From: Sergey Konovalov Date: Thu, 3 Jun 2021 13:10:34 +0300 Subject: [PATCH 29/29] [bug] Fix bug 50708 --- Common/sources/utils.js | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/Common/sources/utils.js b/Common/sources/utils.js index 9dd489f83..f39533721 100644 --- a/Common/sources/utils.js +++ b/Common/sources/utils.js @@ -881,18 +881,15 @@ exports.convertLicenseInfoToFileParams = function(licenseInfo) { // } let license = {}; license.end_date = licenseInfo.endDate && licenseInfo.endDate.toJSON(); - license.trial = constants.LICENSE_MODE.Trial === licenseInfo.mode; - license.developer = constants.LICENSE_MODE.Developer === licenseInfo.mode; - switch (licenseInfo.mode) { - case constants.LICENSE_MODE.Developer: - license.mode = 'developer'; - break; - case constants.LICENSE_MODE.Trial: - license.mode = 'trial'; - break; - default: - license.mode = ''; - break; + license.timelimited = 0 !== (constants.LICENSE_MODE.Limited & licenseInfo.mode); + license.trial = 0 !== (constants.LICENSE_MODE.Trial & licenseInfo.mode); + license.developer = 0 !== (constants.LICENSE_MODE.Developer & licenseInfo.mode); + if(license.developer) { + license.mode = 'developer'; + } else if(license.trial) { + license.mode = 'trial'; + } else { + license.mode = ''; } license.light = licenseInfo.light; license.branding = licenseInfo.branding;