Skip to content

Commit

Permalink
use boost-bcp to reduce external size (#15770)
Browse files Browse the repository at this point in the history
  • Loading branch information
@PatriceJiang
PatriceJiang authored Jul 20, 2023
1 parent 41304cf commit 9ce02b5
Show file tree
Hide file tree
Showing 3 changed files with 806 additions and 396 deletions.
293 changes: 288 additions & 5 deletions native/utils/clear-platform.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
'use strict';

// @ts-check

const chalk = require('chalk');
const path = require("path");
const fs = require('fs-extra');
const tar = require('tar');
const zlib = require('zlib');
const axios = require('axios');
const ProgressBar = require('progress');
const os = require('os');
const child_process = require('child_process');
const fetch = require('node-fetch');

// usage: npm run clear-platform
let externalDir = path.join(__dirname, "../external");
Expand All @@ -13,13 +22,22 @@ let winUsefulDirs = ['android', 'cmake', 'emscripten', 'ohos', 'sources', 'win64
let linuxUsefulDirs = ['android', 'cmake', 'emscripten', 'linux', 'ohos', 'sources'];
let allDirs = [];

['log', 'warn', 'error', 'info'].forEach(field => {
const oldFn = console[field];
console[field] = function (...args) {
oldFn.apply(this, [`[clear-platform.js:${field}] `, ...args]);
};
});


main();

function main() {
async function main() {
readDirectory(externalDir, allDirs);
console.log(chalk.green(`==== current exists 3rd-libs directories ====`));
console.log(allDirs);
cleanPlatform(process.platform);
await minimizeBoost();
}

function readDirectory(path, filesList) {
Expand Down Expand Up @@ -52,7 +70,7 @@ function cleanPlatform(platform) {
if (winUselessDirs.length > 0) {
console.log(chalk.magenta(`==== Remove win32 useless 3rd-libs ====`));
console.log(winUselessDirs);
for (let i = 0; i < winUselessDirs.length; i++) {
for (let i = 0; i < winUselessDirs.length; i++) {
let clearDirectory = path.join(externalDir, winUselessDirs[i]);
console.log(` ${chalk.green('Remove directory: ')} ${clearDirectory}`);
ensureRemove(clearDirectory);
Expand All @@ -63,7 +81,7 @@ function cleanPlatform(platform) {
}
else if (platform === 'linux') { // linux
let linuxUselessDirs = getArrDiff(allDirs, linuxUsefulDirs);
if (linuxUselessDirs.length > 0) {
if (linuxUselessDirs.length > 0) {
console.log(chalk.magenta(`==== Remove linux useless 3rd-libs ====`));
console.log(linuxUselessDirs);
for (let i = 0; i < linuxUselessDirs.length; i++) {
Expand Down Expand Up @@ -99,7 +117,272 @@ function ensureRemove(clearDirectory) {
}

function getArrDiff(arr1, arr2) {
return arr1.concat(arr2).filter(function(v, i, arr) {
return arr.indexOf(v) === arr.lastIndexOf(v);
return arr1.concat(arr2).filter(function (v, i, arr) {
return arr.indexOf(v) === arr.lastIndexOf(v);
});
}

function execName(fn, winPostFix = '.exe', linuxPosFix = '') {
return os.platform() === 'win32' ? fn + winPostFix : fn + linuxPosFix;
}

async function runCommand(dir, cmd, ...args) {
console.log(`Exec ${cmd} with [${args.join(', ')}]`);
return new Promise((resolve, reject) => {
const child = child_process.spawn(cmd, args, {
cwd: dir,
shell: true,
});
child.stdout.on('data', (data) => {
console.log(data.toString('utf8'));
});
child.stderr.on('data', (data) => {
console.error(data.toString('utf8'));
});
child.on('error', (data) => {
console.log(`Error:`);
console.error(data.toString('utf8'));
});
child.on('close', (code) => {
if (code == 0) {
return resolve();
}
console.error(`Command ${cmd} exited with code ${code}`)
reject(new Error(`Command ${cmd} exited with code ${code}`));
})
});
}

async function filterFiles(root, tstFn) {
const all = [];
const walk = async (dir) => {
const st = fs.statSync(dir);
if (st.isDirectory()) {
const files = await fs.readdir(dir);
const sub = [];
for (let f of files) {
if (f.startsWith('.')) continue;
const p = path.join(dir, f);
if (tstFn(p, true)) sub.push(walk(p));
}
await Promise.all(sub);
} else {
if (tstFn(dir, false)) all.push(dir);
}
}
await walk(root);
return all;
}

async function batcher(array, batchSize, fn) {
for (let i = 0, l = array.length; i < l; i += batchSize) {
await Promise.all(array.slice(i, i + batchSize).map(fn))
}
}

async function minimizeBoost() {
const BOOST_VERSION = [1, 78, 0]
const BOOST_VERSION_DOTTED = BOOST_VERSION.join('.');
const BOOST_VERSION_UNDERSCORE = BOOST_VERSION.join('_');
const BOOST_ZIP_FILE = `boost_${BOOST_VERSION_UNDERSCORE}.tar.gz`;
const EXTERNAL_SOURCE_DIR = path.join(__dirname, '..', 'external', 'sources');
const BOOST_DOWNLOAD = path.join(EXTERNAL_SOURCE_DIR, 'boost-download');
const BOOST_ORIG_PATH = path.join(EXTERNAL_SOURCE_DIR, 'boost');
const BOOST_DEST = path.join(EXTERNAL_SOURCE_DIR, 'boost-minimized');
const BOOST_DECOMPRESSED_PATH = path.join(BOOST_DOWNLOAD, `boost_${BOOST_VERSION_UNDERSCORE}`);
const BOOST_FILE_PATH = path.join(BOOST_DOWNLOAD, BOOST_ZIP_FILE);
const BOOST_URL_REMOTE = `https://jaist.dl.sourceforge.net/project/boost/boost/${BOOST_VERSION_DOTTED}/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz`
const BOOST_URL_LOCAL = `http://ftp.cocos.org/TestBuilds/Editor-3d/tools/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz`

const BCP_PATH = path.join(BOOST_DECOMPRESSED_PATH, 'dist', 'bin', execName('bcp'));
const ENGINE_ROOT = path.normalize(path.join(__dirname, '../../'));

let headersUsedByEngine = [];

const fnCheckDirectories = async () => {
if (!await fs.pathExists(BOOST_DOWNLOAD)) {
await fs.mkdir(BOOST_DOWNLOAD);
}
if (!await fs.pathExists(BOOST_DEST)) {
await fs.mkdir(BOOST_DEST);
}
}

const fnDownloadWithAxios = async (url, dstUrl) => {
let bar = null;
const dstFile = fs.createWriteStream(dstUrl);
return new Promise((resolve, reject) => {
axios.get(url, { responseType: 'stream' })
.then((res) => {
// console.log(res.headers);
const total = res.headers['Content-Length'] || res.headers['content-length'];
bar = new ProgressBar('Downloading [:bar] :percent :etas', {
complete: '=',
incomplete: ' ',
width: 40,
total: parseInt(total)
});
res.data.pipe(dstFile);
res.data.on('data', (chunk) => { bar.tick(chunk.length); });
res.data.on('end', () => {
console.log(` Download Finished`);
resolve();
});
}).catch(err => reject(err));
});
};

const fnDownloadWithCurl = async (url, dstUrl) => {
await runCommand(BOOST_DOWNLOAD, 'curl', url, '-o', `"${dstUrl}"`)
}

const rm = async (file) => {
const st = fs.statSync(file);
if (st.isDirectory()) {
const files = (await fs.readdir(file)).filter(x => x !== '.' && x !== '..').map(x => path.join(file, x));
await Promise.all(files.map(x => rm(x)));
await fs.rmdir(file)
} else {
await fs.unlink(file);
}
};

const fnDownloadBoost = async () => {
const targetExist = await fs.pathExists(BOOST_FILE_PATH);
if (!targetExist) {
let boostURL;
console.log(`Testing local url: ${BOOST_URL_LOCAL}`);
try {
const FETCH_TIMEOUTOUT = 3000;
const testLocal = await Promise.race( [fetch(BOOST_URL_LOCAL), new Promise((_, r)=> {
setTimeout(()=> r(new Error('Request timed out')), FETCH_TIMEOUTOUT);
})]);
if (testLocal.ok) {
console.log(` Use ${BOOST_URL_LOCAL}!`);
boostURL = BOOST_URL_LOCAL;
} else {
console.log(` Failed on error code, use ${BOOST_URL_REMOTE}!`);
boostURL = BOOST_URL_REMOTE;
}
} catch (e) {
console.error(e);
console.log(` Failed on exception, use ${BOOST_URL_REMOTE}!`);
boostURL = BOOST_URL_REMOTE;
}
console.log(` downloading ${boostURL}\n to ${BOOST_FILE_PATH}`);

await fnDownloadWithAxios(boostURL, BOOST_FILE_PATH);
// await fnDownloadWithCurl(BOOST_URL, BOOST_FILE_PATH);
} else {
console.log(`Skip downloading, file ${BOOST_FILE_PATH} exists.`);
}
};

const fnDecompressBoost = async () => {
if (await fs.pathExists(BOOST_DECOMPRESSED_PATH)) {
console.log(` Folder ${BOOST_DECOMPRESSED_PATH} exists, skip decompressing...`);
return;
}
console.log(`Decompressing file ${BOOST_FILE_PATH}`);
const srcStream = fs.createReadStream(BOOST_FILE_PATH);
const st = fs.statSync(BOOST_FILE_PATH);
let bar;
return new Promise((resolve, reject) => {
bar = new ProgressBar(' Extracting [:bar] :percent :etas', {
complete: '=',
incomplete: ' ',
width: 40,
total: st.size
});
srcStream.on('data', chunk => bar.tick(chunk.length))
.pipe(zlib.createGunzip())
.pipe(tar.extract({ cwd: BOOST_DOWNLOAD }))
.on('entry', (entry) => {
// console.log(` processing ${entry.path}`);
})
.on('error', err => {
console.error(` decompressing error!`);
reject(err);
})
.on('end', () => {
console.log(` decompressing done!`);
resolve();
});
});
};

const fnCompileBoost = async () => {
if (await fs.pathExists(BCP_PATH)) {
console.log(` File ${BCP_PATH} already exists, skip compilation`);
return;
}
const bootstrap = path.join(BOOST_DECOMPRESSED_PATH, execName('bootstrap', '.bat', '.sh'));
const b2 = path.join(BOOST_DECOMPRESSED_PATH, execName('b2'));
await runCommand(BOOST_DECOMPRESSED_PATH, bootstrap);
await runCommand(BOOST_DECOMPRESSED_PATH, b2, 'headers');
await runCommand(BOOST_DECOMPRESSED_PATH, b2, 'tools/bcp');

if (await fs.pathExists(BCP_PATH)) {
console.log(` Compile bcp.exe successed!`);
} else {
console.error(` Failed to compile bcp.exe, file not found!`)
}
};

const fnCollectBoostHeaders = async () => {
console.log(` Analyzing source files...`);
const allSource = await filterFiles(ENGINE_ROOT, (path, isdir) => {
return isdir ? !/native(\\|\/)+external/.test(path) : ['.h', '.hpp', '.cpp', '.mm'].filter(e => path.endsWith(e)).length > 0;
});
console.log(`${allSource.length} files found!`);
const R = /#include\s+[<"]boost\/([^">]+)/
const boostHeaders = {};
await batcher(allSource, 20, async (file) => {
const data = await fs.readFile(file, 'utf8')
const includes = data.split(`\n`).filter(x => x.trim().startsWith("#include")).map(x => x.trim()).filter(x => R.test(x));
includes.forEach(x => {
const m = x.match(R);
if (m) {
boostHeaders[m[1]] = true;
}
})
})
headersUsedByEngine = Object.keys(boostHeaders).sort();
console.log(` headers: ${headersUsedByEngine.join(', ')}`);
};

const fnRunBCP = async () => {
await runCommand(BOOST_DECOMPRESSED_PATH, BCP_PATH,
`--boost="${BOOST_DECOMPRESSED_PATH}"`,
'graph', /* manually add components here */
...headersUsedByEngine,
BOOST_DEST
);

if (await fs.pathExists(BOOST_ORIG_PATH)) {
console.log(`Remove old folder ${BOOST_ORIG_PATH}`);
await rm(BOOST_ORIG_PATH);
}
console.log(`Link to minimized boost folder ${BOOST_DEST}`);
await fs.rename(path.join(BOOST_DEST, 'boost'), BOOST_ORIG_PATH);
if (await fs.pathExists(BOOST_DEST)) {
console.log(`Remove folder ${BOOST_DEST}`);
await rm(BOOST_DEST);
}
};

const fnCleanUp = async () => {
if (await fs.pathExists(BOOST_DOWNLOAD)) {
await rm(BOOST_DOWNLOAD);
}
}


await fnCheckDirectories();
await fnDownloadBoost();
await fnDecompressBoost();
await fnCompileBoost();
await fnCollectBoostHeaders();
await fnRunBCP();
await fnCleanUp(); // skip this step when debugging
}
Loading

0 comments on commit 9ce02b5

Please sign in to comment.