Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

use boost-bcp to reduce external size #15770

Merged
merged 1 commit into from
Jul 20, 2023
Merged

use boost-bcp to reduce external size #15770

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
293 changes: 288 additions & 5 deletions native/utils/clear-platform.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
'use strict';

// @ts-check

const chalk = require('chalk');
const path = require("path");
const fs = require('fs-extra');
const tar = require('tar');
const zlib = require('zlib');
const axios = require('axios');
const ProgressBar = require('progress');
const os = require('os');
const child_process = require('child_process');
const fetch = require('node-fetch');

// usage: npm run clear-platform
let externalDir = path.join(__dirname, "../external");
Expand All @@ -13,13 +22,22 @@ let winUsefulDirs = ['android', 'cmake', 'emscripten', 'ohos', 'sources', 'win64
let linuxUsefulDirs = ['android', 'cmake', 'emscripten', 'linux', 'ohos', 'sources'];
let allDirs = [];

['log', 'warn', 'error', 'info'].forEach(field => {
const oldFn = console[field];
console[field] = function (...args) {
oldFn.apply(this, [`[clear-platform.js:${field}] `, ...args]);
};
});


main();

function main() {
async function main() {
readDirectory(externalDir, allDirs);
console.log(chalk.green(`==== current exists 3rd-libs directories ====`));
console.log(allDirs);
cleanPlatform(process.platform);
await minimizeBoost();
}

function readDirectory(path, filesList) {
Expand Down Expand Up @@ -52,7 +70,7 @@ function cleanPlatform(platform) {
if (winUselessDirs.length > 0) {
console.log(chalk.magenta(`==== Remove win32 useless 3rd-libs ====`));
console.log(winUselessDirs);
for (let i = 0; i < winUselessDirs.length; i++) {
for (let i = 0; i < winUselessDirs.length; i++) {
let clearDirectory = path.join(externalDir, winUselessDirs[i]);
console.log(` ${chalk.green('Remove directory: ')} ${clearDirectory}`);
ensureRemove(clearDirectory);
Expand All @@ -63,7 +81,7 @@ function cleanPlatform(platform) {
}
else if (platform === 'linux') { // linux
let linuxUselessDirs = getArrDiff(allDirs, linuxUsefulDirs);
if (linuxUselessDirs.length > 0) {
if (linuxUselessDirs.length > 0) {
console.log(chalk.magenta(`==== Remove linux useless 3rd-libs ====`));
console.log(linuxUselessDirs);
for (let i = 0; i < linuxUselessDirs.length; i++) {
Expand Down Expand Up @@ -99,7 +117,272 @@ function ensureRemove(clearDirectory) {
}

function getArrDiff(arr1, arr2) {
return arr1.concat(arr2).filter(function(v, i, arr) {
return arr.indexOf(v) === arr.lastIndexOf(v);
return arr1.concat(arr2).filter(function (v, i, arr) {
return arr.indexOf(v) === arr.lastIndexOf(v);
});
}

function execName(fn, winPostFix = '.exe', linuxPosFix = '') {
return os.platform() === 'win32' ? fn + winPostFix : fn + linuxPosFix;
}

async function runCommand(dir, cmd, ...args) {
console.log(`Exec ${cmd} with [${args.join(', ')}]`);
return new Promise((resolve, reject) => {
const child = child_process.spawn(cmd, args, {
cwd: dir,
shell: true,
});
child.stdout.on('data', (data) => {
console.log(data.toString('utf8'));
});
child.stderr.on('data', (data) => {
console.error(data.toString('utf8'));
});
child.on('error', (data) => {
console.log(`Error:`);
console.error(data.toString('utf8'));
});
child.on('close', (code) => {
if (code == 0) {
return resolve();
}
console.error(`Command ${cmd} exited with code ${code}`)
reject(new Error(`Command ${cmd} exited with code ${code}`));
})
});
}

async function filterFiles(root, tstFn) {
const all = [];
const walk = async (dir) => {
const st = fs.statSync(dir);
if (st.isDirectory()) {
const files = await fs.readdir(dir);
const sub = [];
for (let f of files) {
if (f.startsWith('.')) continue;
const p = path.join(dir, f);
if (tstFn(p, true)) sub.push(walk(p));
}
await Promise.all(sub);
} else {
if (tstFn(dir, false)) all.push(dir);
}
}
await walk(root);
return all;
}

async function batcher(array, batchSize, fn) {
for (let i = 0, l = array.length; i < l; i += batchSize) {
await Promise.all(array.slice(i, i + batchSize).map(fn))
}
}

async function minimizeBoost() {
const BOOST_VERSION = [1, 78, 0]
const BOOST_VERSION_DOTTED = BOOST_VERSION.join('.');
const BOOST_VERSION_UNDERSCORE = BOOST_VERSION.join('_');
const BOOST_ZIP_FILE = `boost_${BOOST_VERSION_UNDERSCORE}.tar.gz`;
const EXTERNAL_SOURCE_DIR = path.join(__dirname, '..', 'external', 'sources');
const BOOST_DOWNLOAD = path.join(EXTERNAL_SOURCE_DIR, 'boost-download');
const BOOST_ORIG_PATH = path.join(EXTERNAL_SOURCE_DIR, 'boost');
const BOOST_DEST = path.join(EXTERNAL_SOURCE_DIR, 'boost-minimized');
const BOOST_DECOMPRESSED_PATH = path.join(BOOST_DOWNLOAD, `boost_${BOOST_VERSION_UNDERSCORE}`);
const BOOST_FILE_PATH = path.join(BOOST_DOWNLOAD, BOOST_ZIP_FILE);
const BOOST_URL_REMOTE = `https://jaist.dl.sourceforge.net/project/boost/boost/${BOOST_VERSION_DOTTED}/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz`
const BOOST_URL_LOCAL = `http://ftp.cocos.org/TestBuilds/Editor-3d/tools/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz`

const BCP_PATH = path.join(BOOST_DECOMPRESSED_PATH, 'dist', 'bin', execName('bcp'));
const ENGINE_ROOT = path.normalize(path.join(__dirname, '../../'));

let headersUsedByEngine = [];

const fnCheckDirectories = async () => {
if (!await fs.pathExists(BOOST_DOWNLOAD)) {
await fs.mkdir(BOOST_DOWNLOAD);
}
if (!await fs.pathExists(BOOST_DEST)) {
await fs.mkdir(BOOST_DEST);
}
}

const fnDownloadWithAxios = async (url, dstUrl) => {
let bar = null;
const dstFile = fs.createWriteStream(dstUrl);
return new Promise((resolve, reject) => {
axios.get(url, { responseType: 'stream' })
.then((res) => {
// console.log(res.headers);
const total = res.headers['Content-Length'] || res.headers['content-length'];
bar = new ProgressBar('Downloading [:bar] :percent :etas', {
complete: '=',
incomplete: ' ',
width: 40,
total: parseInt(total)
});
res.data.pipe(dstFile);
res.data.on('data', (chunk) => { bar.tick(chunk.length); });
res.data.on('end', () => {
console.log(` Download Finished`);
resolve();
});
}).catch(err => reject(err));
});
};

const fnDownloadWithCurl = async (url, dstUrl) => {
await runCommand(BOOST_DOWNLOAD, 'curl', url, '-o', `"${dstUrl}"`)
}

const rm = async (file) => {
const st = fs.statSync(file);
if (st.isDirectory()) {
const files = (await fs.readdir(file)).filter(x => x !== '.' && x !== '..').map(x => path.join(file, x));
await Promise.all(files.map(x => rm(x)));
await fs.rmdir(file)
} else {
await fs.unlink(file);
}
};

const fnDownloadBoost = async () => {
const targetExist = await fs.pathExists(BOOST_FILE_PATH);
if (!targetExist) {
let boostURL;
console.log(`Testing local url: ${BOOST_URL_LOCAL}`);
try {
const FETCH_TIMEOUTOUT = 3000;
const testLocal = await Promise.race( [fetch(BOOST_URL_LOCAL), new Promise((_, r)=> {
setTimeout(()=> r(new Error('Request timed out')), FETCH_TIMEOUTOUT);
})]);
if (testLocal.ok) {
console.log(` Use ${BOOST_URL_LOCAL}!`);
boostURL = BOOST_URL_LOCAL;
} else {
console.log(` Failed on error code, use ${BOOST_URL_REMOTE}!`);
boostURL = BOOST_URL_REMOTE;
}
} catch (e) {
console.error(e);
console.log(` Failed on exception, use ${BOOST_URL_REMOTE}!`);
boostURL = BOOST_URL_REMOTE;
}
console.log(` downloading ${boostURL}\n to ${BOOST_FILE_PATH}`);

await fnDownloadWithAxios(boostURL, BOOST_FILE_PATH);
// await fnDownloadWithCurl(BOOST_URL, BOOST_FILE_PATH);
} else {
console.log(`Skip downloading, file ${BOOST_FILE_PATH} exists.`);
}
};

const fnDecompressBoost = async () => {
if (await fs.pathExists(BOOST_DECOMPRESSED_PATH)) {
console.log(` Folder ${BOOST_DECOMPRESSED_PATH} exists, skip decompressing...`);
return;
}
console.log(`Decompressing file ${BOOST_FILE_PATH}`);
const srcStream = fs.createReadStream(BOOST_FILE_PATH);
const st = fs.statSync(BOOST_FILE_PATH);
let bar;
return new Promise((resolve, reject) => {
bar = new ProgressBar(' Extracting [:bar] :percent :etas', {
complete: '=',
incomplete: ' ',
width: 40,
total: st.size
});
srcStream.on('data', chunk => bar.tick(chunk.length))
.pipe(zlib.createGunzip())
.pipe(tar.extract({ cwd: BOOST_DOWNLOAD }))
.on('entry', (entry) => {
// console.log(` processing ${entry.path}`);
})
.on('error', err => {
console.error(` decompressing error!`);
reject(err);
})
.on('end', () => {
console.log(` decompressing done!`);
resolve();
});
});
};

const fnCompileBoost = async () => {
if (await fs.pathExists(BCP_PATH)) {
console.log(` File ${BCP_PATH} already exists, skip compilation`);
return;
}
const bootstrap = path.join(BOOST_DECOMPRESSED_PATH, execName('bootstrap', '.bat', '.sh'));
const b2 = path.join(BOOST_DECOMPRESSED_PATH, execName('b2'));
await runCommand(BOOST_DECOMPRESSED_PATH, bootstrap);
await runCommand(BOOST_DECOMPRESSED_PATH, b2, 'headers');
await runCommand(BOOST_DECOMPRESSED_PATH, b2, 'tools/bcp');

if (await fs.pathExists(BCP_PATH)) {
console.log(` Compile bcp.exe successed!`);
} else {
console.error(` Failed to compile bcp.exe, file not found!`)
}
};

const fnCollectBoostHeaders = async () => {
console.log(` Analyzing source files...`);
const allSource = await filterFiles(ENGINE_ROOT, (path, isdir) => {
return isdir ? !/native(\\|\/)+external/.test(path) : ['.h', '.hpp', '.cpp', '.mm'].filter(e => path.endsWith(e)).length > 0;
});
console.log(`${allSource.length} files found!`);
const R = /#include\s+[<"]boost\/([^">]+)/
const boostHeaders = {};
await batcher(allSource, 20, async (file) => {
const data = await fs.readFile(file, 'utf8')
const includes = data.split(`\n`).filter(x => x.trim().startsWith("#include")).map(x => x.trim()).filter(x => R.test(x));
includes.forEach(x => {
const m = x.match(R);
if (m) {
boostHeaders[m[1]] = true;
}
})
})
headersUsedByEngine = Object.keys(boostHeaders).sort();
console.log(` headers: ${headersUsedByEngine.join(', ')}`);
};

const fnRunBCP = async () => {
await runCommand(BOOST_DECOMPRESSED_PATH, BCP_PATH,
`--boost="${BOOST_DECOMPRESSED_PATH}"`,
'graph', /* manually add components here */
...headersUsedByEngine,
BOOST_DEST
);

if (await fs.pathExists(BOOST_ORIG_PATH)) {
console.log(`Remove old folder ${BOOST_ORIG_PATH}`);
await rm(BOOST_ORIG_PATH);
}
console.log(`Link to minimized boost folder ${BOOST_DEST}`);
await fs.rename(path.join(BOOST_DEST, 'boost'), BOOST_ORIG_PATH);
if (await fs.pathExists(BOOST_DEST)) {
console.log(`Remove folder ${BOOST_DEST}`);
await rm(BOOST_DEST);
}
};

const fnCleanUp = async () => {
if (await fs.pathExists(BOOST_DOWNLOAD)) {
await rm(BOOST_DOWNLOAD);
}
}


await fnCheckDirectories();
await fnDownloadBoost();
await fnDecompressBoost();
await fnCompileBoost();
await fnCollectBoostHeaders();
await fnRunBCP();
await fnCleanUp(); // skip this step when debugging
}
Loading