From 6610d87bbb91589cc2c4ac33c7c809b920163c96 Mon Sep 17 00:00:00 2001 From: kanno <812137533@qq.com> Date: Mon, 2 Sep 2024 15:44:05 +0800 Subject: [PATCH] chore: release v0.1.3 --- CHANGELOG.md | 6 ++++++ __tests__/stream.spec.ts | 26 +++++++++++++++++++++++--- example/README.md | 4 ++++ example/extract.js | 29 +++++++++++++++++++++++++++++ example/package.json | 7 +++++++ example/setup.js | 22 ++++++++++++++++++++++ package.json | 2 +- pnpm-lock.yaml | 10 ++++------ pnpm-workspace.yaml | 2 ++ src/stream.ts | 31 +++++++++++++++++++++++++------ 10 files changed, 123 insertions(+), 16 deletions(-) create mode 100644 example/README.md create mode 100644 example/extract.js create mode 100644 example/package.json create mode 100644 example/setup.js create mode 100644 pnpm-workspace.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 702dd15..cf6e627 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 0.1.3 + +## Patches + +- Fix incorrect split chunk when process large files. + # 0.1.2 ## Patches diff --git a/__tests__/stream.spec.ts b/__tests__/stream.spec.ts index 7bf9311..7bdcbbb 100644 --- a/__tests__/stream.spec.ts +++ b/__tests__/stream.spec.ts @@ -90,7 +90,7 @@ describe('Stream', () => { const targetPath = path.join(__dirname, 'tpl') const output = path.join(targetPath, 'node-v22.7.0') fs.mkdirSync(targetPath, { recursive: true }) - await x('tar', [`-xf${nodeTar}`, `-C${targetPath}`]) + await x('tar', [`-xzf${nodeTar}`, `-C${targetPath}`]) const files = await readAll(output) const extract = createExtract() let c = 0 @@ -100,8 +100,6 @@ describe('Stream', () => { } }) - extract.on('error', (e) => console.log(e)) - const p = await new Promise((resolve) => { const reader = fs.createReadStream(nodeTar) const binary: Buffer[] = [] @@ -113,6 +111,28 @@ describe('Stream', () => { extract.receiver.write(p) extract.receiver.end() + await new Promise((resolve) => extract.on('close', resolve)) + await fsp.rm(targetPath, { recursive: true }) + expect(c).toBe(files.length) + }) + it('@LongLink GNU tar2', async () => { + const nodeTar = path.join(fixturesPath, 'node-v22.7.0.tar.gz') + const targetPath = path.join(__dirname, 'tpl') + const output = path.join(targetPath, 'node-v22.7.0') + fs.mkdirSync(targetPath, { recursive: true }) + await x('tar', [`-xzf${nodeTar}`, `-C${targetPath}`]) + const files = await readAll(output) + const extract = createExtract() + let c = 0 + extract.on('entry', (head) => { + if (head.typeflag === TypeFlag.REG_TYPE) { + c += 1 + } + }) + + const reader = fs.createReadStream(nodeTar) + reader.pipe(zlib.createUnzip()).pipe(extract.receiver) + await new Promise((resolve) => extract.on('close', resolve)) await fsp.rm(targetPath, { recursive: true }) expect(c).toBe(files.length) diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..946414c --- /dev/null +++ b/example/README.md @@ -0,0 +1,4 @@ +# Example + +> [!IMPORTANT] +> This sample example might not support work on windows. diff --git a/example/extract.js b/example/extract.js new file mode 100644 index 0000000..83a6684 --- /dev/null +++ b/example/extract.js @@ -0,0 +1,29 @@ +import fs from 'fs' +import path from 'path' +import zlib from 'zlib' +import { createExtract } from 'tar-mini' +import { setupEnv } from './setup.js' + +const [ok, symPath] = setupEnv() + +async function main() { + if (!ok) { + console.error('An error occurred') + process.exit(1) + } + const extract = createExtract() + const gz = path.join(symPath, 'node-v22.7.0.tar.gz') + const readable = fs.createReadStream(gz) + + const unzip = zlib.createUnzip() + + extract.on('entry', (head) => { + console.log(head.name) + }) + + readable.pipe(unzip).pipe(extract.receiver) + + await new Promise((resolve) => extract.on('close', resolve)) +} + +main() diff --git a/example/package.json b/example/package.json new file mode 100644 index 0000000..498a712 --- /dev/null +++ b/example/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "tar-mini": "workspace:*" + } +} diff --git a/example/setup.js b/example/setup.js new file mode 100644 index 0000000..d602724 --- /dev/null +++ b/example/setup.js @@ -0,0 +1,22 @@ +import fs from 'fs' +import url from 'url' +import path from 'path' + +const __dirname = path.dirname(url.fileURLToPath(import.meta.url)) + +const TARGET_FIXTURES_PATH = path.join(path.dirname(__dirname), '__tests__', 'fixtures') + +const OUTPUT_PATH = path.join(__dirname, 'tpl') + +export function setupEnv() { + if (fs.existsSync(OUTPUT_PATH)) { + return [true, OUTPUT_PATH] + } + + try { + fs.symlinkSync(TARGET_FIXTURES_PATH, OUTPUT_PATH, 'dir') + return [true, OUTPUT_PATH] + } catch (error) { + return [false, ''] + } +} diff --git a/package.json b/package.json index 46cceb8..9adc424 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "tar-mini", - "version": "0.1.2", + "version": "0.1.3", "description": "It's an implementation based on the `ustar` format", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e6fc298..6ffc2f6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,13 +59,11 @@ importers: specifier: ^2.0.4 version: 2.0.4(@types/node@20.14.11) - packages/core: {} - - packages/stream: - dependencies: - '@tar/core': + example: + devDependencies: + tar-mini: specifier: workspace:* - version: link:../core + version: link:.. packages: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..d40538d --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,2 @@ +packages: + - 'example' \ No newline at end of file diff --git a/src/stream.ts b/src/stream.ts index 086188d..02372cc 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -162,9 +162,11 @@ class FastBytes { throw new Error(FAST_BYTES_ERROR_MESSAGES.EXCEED_BYTES_LEN) } const elt = this.queue.peek() + if (!elt) { throw new Error(FAST_BYTES_ERROR_MESSAGES.EXCEED_BYTES_LEN) } + return elt.subarray(0, size) } } @@ -186,6 +188,7 @@ export class Extract { private isNonUSTAR: boolean private paxMeta: Record private gnuMeta: Record + private pause: boolean constructor(options: DecodingHeadOptions) { this.decodeOptions = options this.matrix = new FastBytes() @@ -198,9 +201,19 @@ export class Extract { this.isNonUSTAR = false this.paxMeta = Object.create(null) this.gnuMeta = Object.create(null) + this.pause = false this.writer = createWriteableStream({ write: (chunk, _, callback) => { - this.matrix.push(chunk) + if (this.pause) { + const bb = this.matrix.shift(this.matrix.bytesLen) + const next = new Uint8Array(bb.length + chunk.length) + next.set(bb) + next.set(chunk, bb.length) + this.matrix.push(next) + this.pause = false + } else { + this.matrix.push(chunk) + } this.transport() callback() } @@ -221,6 +234,7 @@ export class Extract { const decodeHead = () => { try { this.head = decode(this.matrix.shift(512), this.decodeOptions) + if (!ensureIsStandardUSTARFormat(this.head.typeflag)) { this.isNonUSTAR = true return true @@ -322,11 +336,16 @@ export class Extract { continue } - const c = this.matrix.peek(512) - - if (c[0] === Magic.NULL_CHAR && c[511] === Magic.NULL_CHAR) { - this.matrix.shift(512) - continue + try { + const c = this.matrix.peek(512) + if (c[0] === Magic.NULL_CHAR && c[511] === Magic.NULL_CHAR) { + this.matrix.shift(512) + continue + } + } catch (_) { + // In some case the last chunk of the file is not enough to fill the 512 bytes + this.pause = true + return } if (!decodeHead()) return