Skip to content

Commit

Permalink
1. No longer copy built-in resources to the cache path on startup, bu…
Browse files Browse the repository at this point in the history 
…t use them directly.

2. The cache path is only used to store subsequent updated content.
3. Load cache resources and built-in resources at the same time, and give priority to cache resources when ids are repeated.
4. No longer checks resource file integrity at startup.
  • Loading branch information
@zhengyangliu
zhengyangliu committed Jun 15, 2024
1 parent df8f5d2 commit b367b83
Show file tree
Hide file tree
Showing 9 changed files with 73 additions and 169 deletions.
115 changes: 22 additions & 93 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,123 +2,52 @@ const fs = require('fs-extra');
const Emitter = require('events');
const path = require('path');
const compareVersions = require('compare-versions');
const clc = require('cli-color');
const lockFile = require('proper-lockfile');

const ResourceServer = require('./src/server');
const ResourceUpdater = require('./src/updater');
const {checkDirHash} = require('./src/calc-dir-hash');
const {INIT_RESOURCES_STEP} = require('./src/state');
const getConfigHash = require('./src/get-config-hash');
const {
DIRECTORY_NAME,
DEFAULT_USER_DATA_PATH,
DEFAULT_LOCALE,
RECHECK_INTERVAL
DEFAULT_CACHE_RESOURCES_PATH,
DEFAULT_BUILTIN_RESOURCES_PATH,
DEFAULT_LOCALE
} = require('./src/config');

class OpenblockResourceServer extends Emitter{
constructor (userDataPath, initialResourcesPath, locale = DEFAULT_LOCALE) {
constructor (cacheResourcesPath, builtinResourcesPath, locale = DEFAULT_LOCALE) {
super();

if (userDataPath) {
this._userDataPath = path.join(userDataPath, DIRECTORY_NAME);
if (cacheResourcesPath) {
this._cacheResourcesPath = path.join(cacheResourcesPath);
} else {
this._userDataPath = path.join(DEFAULT_USER_DATA_PATH, DIRECTORY_NAME);
this._cacheResourcesPath = path.join(DEFAULT_CACHE_RESOURCES_PATH);
}
this._configPath = path.join(this._userDataPath, 'config.json');

if (initialResourcesPath) {
this._resourcesPath = path.join(initialResourcesPath);
if (builtinResourcesPath) {
this._builtinResourcesPath = path.join(builtinResourcesPath);
} else {
this._resourcesPath = path.join(__dirname, DIRECTORY_NAME);
this._builtinResourcesPath = path.join(DEFAULT_BUILTIN_RESOURCES_PATH);
}

// If 'OpenBlockResources' exists in the upper-level directory, the content in this
// directory will be used first, rather than the content in the software installation path.
// This method is used when customizing by a third-party manufacturer, so as to avoid overwriting
// the content of the third - party manufacturer when updating the software.
const thirdPartyResourcesPath = path.join(this._resourcesPath, '../../OpenBlockResources');
const thirdPartyResourcesPath = path.join(this._builtinResourcesPath, '../../OpenBlockResources');
if (fs.existsSync(thirdPartyResourcesPath)) {
this._resourcesPath = thirdPartyResourcesPath;
this._builtinResourcesPath = thirdPartyResourcesPath;
}

this._locale = locale;

this._latestVersion = null;
this.updater = null;
}

checkResources () {
if (!fs.existsSync(this._configPath)){
return Promise.reject(`Cannot find config file: ${this._configPath}`);
if (fs.existsSync(this._cacheResourcesPath)) {
this._configPath = path.join(this._cacheResourcesPath, 'config.json');
} else {
this._configPath = path.join(this._builtinResourcesPath, 'config.json');
}

const dirHash = getConfigHash(this._configPath);
console.log(this._configPath);

// If no hash value in config file, report a warning but don't stop the process.
if (!dirHash) {
console.warn(clc.yellow(`WARN: no hash value found in ${this._configPath}`));
return Promise.resolve();
}

return checkDirHash(this._userDataPath, dirHash);
}

initializeResources (callback = null) {
if (callback) {
callback({phase: INIT_RESOURCES_STEP.verifying});
}
fs.ensureDirSync(this._userDataPath);

const copyResources = () => {
console.log(`copy ${this._resourcesPath} to ${this._userDataPath}`);
if (callback) {
callback({phase: INIT_RESOURCES_STEP.copying});
}

// copy the initial resources to user data directory
return fs.mkdirs(this._userDataPath)
.then(() => fs.copy(this._resourcesPath, this._userDataPath))
.then(() => {
lockFile.unlockSync(this._userDataPath);
return this.checkResources();
});
};

const waitUntillInitializeFinish = () => {
if (lockFile.checkSync(this._userDataPath)) {
setTimeout(() => {
console.log(clc.yellow(`WARN: A resource initialize process is already running, will recheck proccess state after ${RECHECK_INTERVAL} ms`)); // eslint-disable-line max-len
waitUntillInitializeFinish();
}, RECHECK_INTERVAL);
} else {
this.emit('initialize-finish');
}
};

if (lockFile.checkSync(this._userDataPath)) {
waitUntillInitializeFinish();
return new Promise(resolve => {
this.on('initialize-finish', () => {
resolve();
});
});
}
this._locale = locale;

lockFile.lockSync(this._userDataPath);
return this.checkResources()
.then(() => {
lockFile.unlockSync(this._userDataPath);
})
.catch(e => {
console.log(clc.yellow(`WARN: Check resources failed, try to initialize resources: ${e}`));
if (fs.existsSync(this._userDataPath)){
return fs.rm(this._userDataPath, {recursive: true, force: true})
.then(() => copyResources());
}
return copyResources();
});
this._latestVersion = null;
this.updater = null;
}

checkUpdate (option) {
Expand All @@ -133,7 +62,7 @@ class OpenblockResourceServer extends Emitter{
if (!this.updater) {
this.updater = new ResourceUpdater(
this._locale === 'CN' && config.updater.cn ? config.updater.cn : config.updater.default,
path.dirname(this._userDataPath));
path.dirname(this._cacheResourcesPath));
}

return this.updater.checkUpdate(option)
Expand All @@ -154,7 +83,7 @@ class OpenblockResourceServer extends Emitter{
}

listen (port = null) {
const server = new ResourceServer(this._userDataPath);
const server = new ResourceServer(this._cacheResourcesPath, this._builtinResourcesPath);

server.on('error', e => {
this.emit('error', e);
Expand Down
3 changes: 1 addition & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"name": "openblock-resource",
"version": "0.2.0",
"description": "External resources for OpenBlock projects",
"main": "start.js",
"main": "index.js",
"scripts": {
"fetch": "rimraf external-resources download && node script/download.js --repo=openblockcc/external-resources-v2 --plat=github",
"start": "node ./test/start-server.js",
Expand All @@ -22,7 +22,6 @@
"test:cnUpdate": "node test/cnUpdate.js",
"test:multi-update": "node test/multi-update.js",
"test:multi-server": "node test/multi-server.js",
"test:multi-init": "rimraf ../.openblockData/external-resources && node test/multi-initial.js",
"test:abort-check": "node test/abortCheckUpdate.js",
"test:abort-update": "node test/abortUpdate.js"
},
Expand Down
12 changes: 6 additions & 6 deletions src/config.js
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
const path = require('path');

/**
* The name of reousce directory.
* The path of default cache resource.
* @readonly
*/
const DIRECTORY_NAME = 'external-resources';
const DEFAULT_CACHE_RESOURCES_PATH = path.join(__dirname, '../../.openblockData/external-resources');

/**
* The path of default user data directory.
* The path of default build-in resource.
* @readonly
*/
const DEFAULT_USER_DATA_PATH = path.join(__dirname, '../../.openblockData');
const DEFAULT_BUILTIN_RESOURCES_PATH = path.join(__dirname, '../external-resources');

/**
* The locale of default.
Expand Down Expand Up @@ -50,8 +50,8 @@ const RECHECK_INTERVAL = 1000 * 1;


module.exports = {
DIRECTORY_NAME,
DEFAULT_USER_DATA_PATH,
DEFAULT_CACHE_RESOURCES_PATH,
DEFAULT_BUILTIN_RESOURCES_PATH,
DEFAULT_LOCALE,
DEFAULT_HOST,
DEFAULT_PORT,
Expand Down
12 changes: 8 additions & 4 deletions src/device.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,21 @@ class OpenBlockDevice {
this.type = TYPE;
}

assembleData (userDataPath, formatMessage) {
assembleData (dataPath, formatMessage) {
const devicesThumbnailData = [];

if (!fs.existsSync(dataPath)) {
return devicesThumbnailData;
}

const devices = requireAll({
dirname: `${path.join(userDataPath, this.type)}`,
dirname: `${path.join(dataPath, this.type)}`,
filter: /index.js$/,
recursive: true
});

// eslint-disable-next-line global-require
const deviceList = require(path.join(userDataPath, this.type, 'device.js'));
const deviceList = require(path.join(dataPath, this.type, 'device.js'));

const parseDeviceList = [];
deviceList.forEach(deviceId => {
Expand All @@ -39,7 +43,7 @@ class OpenBlockDevice {
parseDeviceList.forEach(deviceId => {
let matched = false;
Object.entries(devices).forEach(dev => {
const translationsFile = path.join(userDataPath, this.type, dev[0], 'translations.js');
const translationsFile = path.join(dataPath, this.type, dev[0], 'translations.js');
let translations;
if (fs.existsSync(translationsFile)) {
// eslint-disable-next-line global-require
Expand Down
12 changes: 8 additions & 4 deletions src/extension.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,18 @@ class OpenBlockExtension {
this.type = TYPE;
}

assembleData (userDataPath, formatMessage) {
assembleData (dataPath, formatMessage) {
const extensionsThumbnailData = [];

const extPath = path.join(userDataPath, this.type);
if (!fs.existsSync(dataPath)) {
return extensionsThumbnailData;
}

const extPath = path.join(dataPath, this.type);
if (fs.existsSync(extPath)) {
const data = requireAll({dirname: extPath, filter: /index.js$/, recursive: true});
Object.entries(data).forEach(ext => {
const translationsFile = path.join(userDataPath, this.type, ext[0], 'translations.js');
const translationsFile = path.join(dataPath, this.type, ext[0], 'translations.js');
let translations;
if (fs.existsSync(translationsFile)){
// eslint-disable-next-line global-require
Expand Down Expand Up @@ -49,7 +53,7 @@ class OpenBlockExtension {
content.toolbox = path.join(basePath, content.toolbox);
}
if (content.library) {
content.library = path.join(userDataPath, basePath, content.library);
content.library = path.join(dataPath, basePath, content.library);
}
if (content.main) {
content.main = path.join(basePath, content.main);
Expand Down
34 changes: 24 additions & 10 deletions src/server.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
const formatMessage = require('format-message');
const express = require('express');
const Emitter = require('events');
const path = require('path');
const fs = require('fs');
const fetch = require('node-fetch');
const http = require('http');
const clc = require('cli-color');
Expand All @@ -23,12 +21,14 @@ class ResourceServer extends Emitter{

/**
* Construct a OpenBlock resource server object.
* @param {string} userDataPath - the path of user data.
* @param {string} cacheResourcesPath - the path of cache resources.
* @param {string} builtinResourcesPath - the path of builtin resources.
*/
constructor (userDataPath) {
constructor (cacheResourcesPath, builtinResourcesPath) {
super();

this._userDataPath = userDataPath;
this._cacheResourcesPath = cacheResourcesPath;
this._builtinResourcesPath = builtinResourcesPath;
this._host = DEFAULT_HOST;
this._port = DEFAULT_PORT;

Expand All @@ -40,6 +40,19 @@ class ResourceServer extends Emitter{
this.extensionsIndexData = {};
}

// If the id is the same, cached data is used first
mergeData (cacheData, builtinData, attributes) {
const resultMap = new Map();

cacheData.forEach(item => resultMap.set(item[`${attributes}`], item));
builtinData.forEach(item => {
if (!resultMap.has(item[`${attributes}`])) {
resultMap.set(item[`${attributes}`], item);
}
});
return Array.from(resultMap.values());
}

// If the cache is not exist, generate it.
generateCache (locale) {
if (this.deviceIndexData[`${locale}`] &&
Expand All @@ -53,12 +66,12 @@ class ResourceServer extends Emitter{
});

this.deviceIndexData[`${locale}`] =
JSON.stringify(this.devices.assembleData(this._userDataPath,
this._formatMessage));
JSON.stringify(this.mergeData(this.devices.assembleData(this._cacheResourcesPath, this._formatMessage),
this.devices.assembleData(this._builtinResourcesPath, this._formatMessage), 'deviceId'));

this.extensionsIndexData[`${locale}`] =
JSON.stringify(this.extensions.assembleData(this._userDataPath,
this._formatMessage));
JSON.stringify(this.mergeData(this.extensions.assembleData(this._cacheResourcesPath, this._formatMessage),
this.extensions.assembleData(this._builtinResourcesPath, this._formatMessage), 'extensionId'));
}

isSameServer (host, port) {
Expand Down Expand Up @@ -100,7 +113,8 @@ class ResourceServer extends Emitter{
res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept');
next();
});
this._app.use(express.static(`${this._userDataPath}`));
this._app.use(express.static(`${this._cacheResourcesPath}`));
this._app.use(express.static(`${this._builtinResourcesPath}`));

this._app.get('/', (req, res) => {
res.send(SERVER_NAME);
Expand Down
34 changes: 0 additions & 34 deletions test/multi-initial.js
View file

This file was deleted.

Loading

0 comments on commit b367b83

Please sign in to comment.