Pack/Extract (#2)

* feat: complete size padding

* chore: test

* chore: workflow

.github/workflows/publish.yaml

@@ -0,0 +1,25 @@
+name: publish
+on:
+  push:
+    tags: ['v*']
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22.x'
+          registry-url: 'https://registry.npmjs.org'
+      - name: Install Dependices
+        run: make
+      - name: Pack and Publish
+        run: |
+          make build
+          npm publish --provenance --access public
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

Makefile

@@ -14,3 +14,7 @@ lint:
 format:
 	@echo "Formatting code..."
 	pnpm exec dprint fmt
+
+build:
+	@echo "Building project..."
+	@pnpm run build

README.md

@@ -1,8 +1,6 @@
-# Tar
+# TarMini
 
-It's an implementation based on the `ustar` format. support pack field. This package only provides low-level API's.
-
-WIP.
+It's an implementation based on the `ustar` format. This package only provides low-level API's.
 
 ### Sponsors
 

__tests__/stream.spec.ts

@@ -0,0 +1,60 @@
+import { Writable } from 'stream'
+import { describe, expect, it } from 'vitest'
+import { createExtract, createPack } from '../src'
+
+describe('Stream', () => {
+  describe('Uniform Standard Type Archive', () => {
+    const assets: Record<string, string> = {
+      'assets/a.mjs': 'const a = 1;',
+      'assets/b.mjs': 'import "./c.css"; import { a } from "./a.mjs"; console.log(a);',
+      'assets/c.css': 'body { background: red; }'
+    }
+    describe('Pack', () => {
+      it('Normal', async () => {
+        const pack = createPack()
+
+        let expectByteLen = 0
+        let actualByteLen = 0
+
+        for (const [path, content] of Object.entries(assets)) {
+          pack.add(new TextEncoder().encode(content), {
+            filename: path
+          })
+          expectByteLen += content.length
+          expectByteLen += 512 - (content.length % 512)
+          expectByteLen += 512
+        }
+        expectByteLen += 1024
+        pack.done()
+        const writer = new Writable({
+          write(chunk, _, callback) {
+            actualByteLen += chunk.length
+            callback()
+          }
+        })
+        pack.receiver.pipe(writer)
+        await new Promise((resolve) => writer.on('finish', resolve))
+        expect(actualByteLen).toBe(expectByteLen)
+      })
+    })
+    describe('Extract', () => {
+      it('Normal', async () => {
+        const pack = createPack()
+        const extract = createExtract()
+        const textDecode = new TextDecoder()
+        for (const [path, content] of Object.entries(assets)) {
+          pack.add(new TextEncoder().encode(content), {
+            filename: path
+          })
+        }
+
+        extract.on('entry', (head, file) => {
+          const content = assets[head.name]
+          expect(content).toBe(textDecode.decode(file))
+          expect(head.size).toBe(content.length)
+        })
+        pack.receiver.pipe(extract.receiver)
+      })
+    })
+  })
+})

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "tar-archive",
+  "name": "tar-mini",
   "version": "0.0.0",
   "description": "It's an implementation based on the `ustar` format",
   "main": "dist/index.js",
@@ -36,7 +36,6 @@
     "rollup": "^4.19.0",
     "rollup-plugin-dts": "^6.1.1",
     "rollup-plugin-swc3": "^0.11.2",
-    "tsx": "^4.16.2",
     "typescript": "^5.5.4",
     "vitest": "^2.0.4"
   },

src/list.ts

@@ -77,39 +77,6 @@ export class List<T> {
     }
     return v
   }
-
-  private walker(pos: number, handler: (elts: (T | undefined)[], pos: number, result: T) => void) {
-    if (pos < this.cap) {
-      handler(this.tail.items, pos, this.tail.items[pos]!)
-      return
-    }
-    let elt: Elt<T> | undefined = this.tail
-    let p = pos
-    while (elt) {
-      if (!elt.next && p >= elt.mask) {
-        throw new Error('Index out of range')
-      }
-
-      if (p <= elt.mask) {
-        handler(elt.items, pos, elt.items[p]!)
-        return
-      }
-      if (elt.next) {
-        p -= elt.items.length
-        elt = elt.next
-      }
-    }
-  }
-
-  at(pos: number) {
-    let elt: T | undefined
-    this.walker(pos, (_, __, result) => elt = result)
-    return elt
-  }
-
-  update(pos: number, data: T) {
-    this.walker(pos, (elts, pos) => elts[pos] = data)
-  }
 }
 
 export function createList<T>(cap?: number) {

src/stream.ts

@@ -62,19 +62,33 @@ export class Pack {
     this.reader.push(null)
   }
 
+  private fix(size: number) {
+    const padding = (512 - (size % 512)) % 512
+    if (padding > 0) this.reader.push(new Uint8Array(padding))
+  }
+
   private transport(binary: Uint8Array, resolvedOptions: EncodingHeadOptions) {
     const writer = createWriteableStream({
-      write: (chunk, encoding, callback) => {
-        this.reader.push(encode(resolvedOptions)) 
-        this.reader.push(chunk)
-        callback()
+      write: (chunk, _, callback) => {
+        try {
+          this.reader.push(encode(resolvedOptions)) 
+          this.reader.push(chunk)
+          callback()
+        } catch (error) {
+          callback(error as Error)
+        }
       },
       final: (callback) => {
-        this.reader.push(null)
+        this.reader.push(this.fix(resolvedOptions.size))
         callback()
       }
     })
-    writer.write(binary)
+    writer.write(binary, (e) => {
+      if (e) {
+        this.reader.emit('error', e)
+      }
+    })
+    writer.end()
   }
 
   get receiver() {
@@ -96,17 +110,17 @@ const FAST_BYTES_ERROR_MESSAGES = {
 class FastBytes {
   private queue: List<Uint8Array>
   bytesLen: number
-  private offset: number
-  private position: number
+  insertedBytesLen: number
+
   constructor() {
     this.queue = createList<Uint8Array>()
     this.bytesLen = 0
-    this.offset = 0
-    this.position = 0
+    this.insertedBytesLen = 0
   }
 
   push(b: Uint8Array) {
     this.bytesLen += b.length
+    this.insertedBytesLen += b.length
     this.queue.push(b)
   }
 
@@ -121,39 +135,38 @@ class FastBytes {
     if (!elt) {
       throw new Error(FAST_BYTES_ERROR_MESSAGES.EXCEED_BYTES_LEN)
     }
-    let bb: Uint8Array
-    const overflow = size >= elt.length
-    if (overflow) {
-      bb = new Uint8Array(size)
-      const preBinary = elt.subarray(this.offset)
-      bb.set(preBinary, 0)
-      this.queue.shift()
-      const next = this.queue.peek()!
-      const nextBinary = next.subarray(0, size - preBinary.length)
-      bb.set(nextBinary, preBinary.length)
-      this.position++
-      this.queue.update(this.position, next.subarray(size - preBinary.length)) 
-      while (size - preBinary.length - nextBinary.length > 0) {
-        bb.set(this.shift(size - preBinary.length - nextBinary.length), preBinary.length + nextBinary.length)
-      }
-    } else {
-      this.queue.update(this.position, elt.subarray(size))
-      bb = elt.subarray(0, size)
+
+    const b = new Uint8Array(size)
+    this.queue.shift()
+    const bb = elt.subarray(size)
+    if (bb.length > 0) {
+      this.queue.push(bb)
     }
-    this.offset += size
+    b.set(elt.subarray(0, size))
     this.bytesLen -= size
-
-    return bb
+    return b
   }
 }
 
 export class Extract {
   private writer: Writable
   private decodeOptions: DecodingHeadOptions
   matrix: FastBytes
+  private head: ReturnType<typeof decode>
+  private missing: number
+  private offset: number
+  private flag: boolean
+  private elt: Uint8Array | null
+  private total: number
   constructor(options: DecodingHeadOptions) {
     this.decodeOptions = options
     this.matrix = new FastBytes()
+    this.head = Object.create(null)
+    this.missing = 0
+    this.flag = false
+    this.offset = 0
+    this.elt = null
+    this.total = 0
     this.writer = createWriteableStream({
       write: (chunk, _, callback) => {
         this.matrix.push(chunk)
@@ -163,26 +176,64 @@ export class Extract {
     })
   }
 
-  private scan() {
-    try {
-      if (this.matrix.bytesLen === 512 * 2) {
-        return false
-      } 
-      const head = decode(this.matrix.shift(512), this.decodeOptions)
-      const b = this.matrix.shift(head.size)
-      this.writer.emit('entry', head, new Uint8Array(b))
-      return true
-    } catch (error) {
-      return false
+  private removePadding(size: number) {
+    const padding = (512 - (size % 512)) % 512
+    if (padding > 0) {
+      this.matrix.shift(padding)
+      return padding
     }
+
+    return 0
   }
 
   private transport() {
+    const decodeHead = () => {
+      try {
+        this.head = decode(this.matrix.shift(512), this.decodeOptions)
+        this.missing = this.head.size
+        this.elt = new Uint8Array(this.head.size)
+        this.flag = true
+        this.offset = 0
+        return true
+      } catch (error) {
+        this.writer.emit('error', error)
+        return false
+      }
+    }
+
+    const consume = () => {
+      const leak = this.missing > this.matrix.bytesLen
+      if (leak) {
+        const b = this.matrix.shift(this.matrix.bytesLen)
+        this.missing -= b.length
+        this.elt!.set(b, this.offset)
+        this.offset += b.length
+        return
+      }
+      this.elt!.set(this.matrix.shift(this.missing), this.offset)
+
+      this.total += this.elt!.length + 512
+      this.writer.emit('entry', this.head, this.elt!)
+      this.flag = false
+    }
+
     while (this.matrix.bytesLen > 0) {
-      if (this.matrix.bytesLen < 512) {
-        break
+      if (this.flag) {
+        consume()
+        continue
+      }
+
+      if (this.head && this.head.size && !this.flag) {
+        const padding = this.removePadding(this.head.size)
+        this.total += padding
+        this.head = Object.create(null)
+        continue
+      }
+      if (this.total + 1024 === this.matrix.insertedBytesLen) {
+        this.matrix.shift(1024)
+        return
       }
-      if (!this.scan()) return
+      if (!decodeHead()) return
     }
   }