Faster encoding and decoding thanks Chromium lookup tables (#19)

swift-extras · Oct 26, 2020 · b8af496 · b8af496
1 parent 35d3358
commit b8af496
Show file tree

Hide file tree

Showing 18 changed files with 1,077 additions and 573 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -1,21 +1,3 @@
-coverage:
-  status:
-    project:
-      default: off
-    base64:
-      flags: base64
-      target: 100%
-    unittest:
-      flags: unittest
-    performance:
-      flags: performance
-flags:
-  base64:  
-    paths:
-    - Sources/Base64
-  performance:  
-    paths:
-    - Sources/PerformanceTest
-  unittests:
-    paths:
-    - Tests/Base64Tests
+ignore:
+  - "Tests"
+  - ".build"
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -9,15 +9,15 @@ on:
 
 jobs:
 
-  "sanity-Tests":
+  "validity-Tests":
     runs-on: macOS-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v2
     - name: Install swiftformat
       run: brew install swiftformat
-    - name: Run sanity
-      run: ./scripts/sanity.sh .
+    - name: Run validity
+      run: ./scripts/validity.sh .
 
   "tuxOS-Tests":
     runs-on: ubuntu-latest
@@ -26,8 +26,8 @@ jobs:
         images:
         - swift:5.1
         - swift:5.2
-        - swiftlang/swift:nightly-5.3-bionic
-        - swiftlang/swift:nightly-amazonlinux2
+        - swift:5.3
+        - swiftlang/swift:nightly-master
     container:
       image: ${{ matrix.images }}
     steps:
@@ -52,30 +52,28 @@ jobs:
         images:
         - swift:5.1
         - swift:5.2
-        - swiftlang/swift:nightly-5.3-bionic
-        - swiftlang/swift:nightly-amazonlinux2
+        - swift:5.3
+        - swiftlang/swift:nightly-master
     container:
       image: ${{ matrix.images }}
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Build
-      run: swift build -c release
-    - name: Run test
-      run: .build/release/Base64KitPerformanceTest
+    - name: Build & run
+      run: swift run -c release
 
   "tuxOS-Integration-Tests":
     runs-on: ubuntu-latest
     strategy:
       matrix:
         images:
-        - swift:5.2
-        - swiftlang/swift:nightly-5.3
+        - swift:5.3
+        - swiftlang/swift:nightly-master
     container:
       image: ${{ matrix.images }}
     env:
       MAX_ALLOCS_ALLOWED_base64_decoding: 1000
-      MAX_ALLOCS_ALLOWED_base64_encoding: 2000
+      MAX_ALLOCS_ALLOWED_base64_encoding: 1000
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -108,7 +106,5 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Build
-      run: swift build -c release
-    - name: Run test
-      run: .build/release/Base64KitPerformanceTest
+    - name: Build & run
+      run: swift run -c release
diff --git a/.swiftformat b/.swiftformat
@@ -0,0 +1,13 @@
+# file options
+
+--swiftversion 5.3
+--exclude .build
+
+# format options
+
+--self insert
+--patternlet inline
+--stripunusedargs unnamed-only
+--ifdef no-indent
+
+# rules
diff --git a/IntegrationTests/allocation-counter-tests-framework/template/scaffolding.swift b/IntegrationTests/allocation-counter-tests-framework/template/scaffolding.swift
@@ -15,9 +15,9 @@
 import AtomicCounter
 import Foundation
 #if os(macOS) || os(iOS) || os(watchOS) || os(tvOS)
-    import Darwin
+import Darwin
 #else
-    import Glibc
+import Glibc
 #endif
 
 func measureAll(_ fn: () -> Int) -> [[String: Int]] {
@@ -26,11 +26,11 @@ func measureAll(_ fn: () -> Int) -> [[String: Int]] {
         AtomicCounter.reset_malloc_counter()
         AtomicCounter.reset_malloc_bytes_counter()
         #if os(macOS) || os(iOS) || os(watchOS) || os(tvOS)
-            autoreleasepool {
-                _ = fn()
-            }
-        #else
+        autoreleasepool {
             _ = fn()
+        }
+        #else
+        _ = fn()
         #endif
         usleep(100_000) // allocs/frees happen on multiple threads, allow some cool down time
         let frees = AtomicCounter.read_free_counter()

diff --git a/IntegrationTests/tests_04_performance/test_01_resources/test_base64_decoding.swift b/IntegrationTests/tests_04_performance/test_01_resources/test_base64_decoding.swift
@@ -6,7 +6,7 @@ func run(identifier: String) {
 
     measure(identifier: identifier) {
         for _ in 0 ..< 1000 {
-            bytes = try! base64.base64decoded()
+            bytes = try! Base64.decode(string: base64)
         }
 
         return bytes?.count ?? 0

diff --git a/IntegrationTests/tests_04_performance/test_01_resources/test_base64_encoding.swift b/IntegrationTests/tests_04_performance/test_01_resources/test_base64_encoding.swift
@@ -6,7 +6,7 @@ func run(identifier: String) {
 
     measure(identifier: identifier) {
         for _ in 0 ..< 1000 {
-            base64 = String(base64Encoding: bytes)
+            base64 = Base64.encodeString(bytes: bytes)
         }
 
         return base64?.count ?? 0

diff --git a/Package.swift b/Package.swift
@@ -6,24 +6,12 @@ import PackageDescription
 let package = Package(
     name: "swift-base64-kit",
     products: [
-        .library(
-            name: "Base64Kit",
-            targets: ["Base64Kit"]
-        ),
+        .library(name: "Base64Kit", targets: ["Base64Kit"]),
     ],
     dependencies: [],
     targets: [
-        .target(
-            name: "Base64KitPerformanceTest",
-            dependencies: ["Base64Kit"]
-        ),
-        .target(
-            name: "Base64Kit",
-            dependencies: []
-        ),
-        .testTarget(
-            name: "Base64KitTests",
-            dependencies: ["Base64Kit"]
-        ),
+        .target(name: "Base64Kit", dependencies: []),
+        .target(name: "PerformanceTest", dependencies: ["Base64Kit"]),
+        .testTarget(name: "Base64KitTests", dependencies: ["Base64Kit"]),
     ]
 )
diff --git a/README.md b/README.md
@@ -2,63 +2,56 @@
 
 [![Swift 5.1](https://img.shields.io/badge/Swift-5.1-blue.svg)](https://swift.org/download/)
 [![github-actions](https://github.com/fabianfett/swift-base64-kit/workflows/CI/badge.svg)](https://github.com/fabianfett/swift-base64-kit/actions)
-[![codecov](https://codecov.io/gh/fabianfett/swift-base64-kit/branch/master/graph/badge.svg)](https://codecov.io/gh/fabianfett/swift-base64)
+[![codecov](https://codecov.io/gh/fabianfett/swift-base64-kit/branch/main/graph/badge.svg)](https://codecov.io/gh/fabianfett/swift-base64)
 ![macOS](https://img.shields.io/badge/os-macOS-green.svg?style=flat)
 ![tuxOS](https://img.shields.io/badge/os-tuxOS-green.svg?style=flat)
 
 
-This package provides a base64 encoder and decoder in pure Swift (without the use of Foundation). The implementation is [RFC4648](https://tools.ietf.org/html/rfc4648) complient.
+This package provides a base64 encoder and decoder in Swift without the use of Foundation. The implementation is [RFC4648](https://tools.ietf.org/html/rfc4648) complient and is faster than the Foundation base64 implementation.
 
-Today the implementation is rather simple. No fancy precomputed lookup tables, no fancy SIMD instructions. Therefore, there is definitely room for improvement performance-wise. See also [Literature for a faster algorithm](#user-content-literature-for-a-faster-algorithm).
-
-Everything began with [an issue](https://github.com/apple/swift-nio/issues/1265) on [`swift-nio`](https://github.com/apple/swift-nio).
+To achieve performance the implementation uses [Chromium precomputed lookup tables](https://github.com/lemire/fastbase64/blob/master/src/chromiumbase64.c) and makes heavy use of unsafe swift API. When Swift has better support for SIMD instructions this might be an area worth exploring.
 
 ## Status
 
 - [x] support for base64 and base64url 
 - [x] faster than Foundation
+- [x] padding can be omitted
 - [ ] decoding can ignore line breaks
 - [ ] encoding can insert line breaks
 
-This package's encoding implementation [is used in `swift-nio`'s websocket implementation](https://github.com/apple/swift-nio/blob/master/Sources/NIOWebSocket/Base64.swift).
+A former implementation of this package [is used in `swift-nio`'s websocket implementation](https://github.com/apple/swift-nio/blob/main/Sources/NIOWebSocket/Base64.swift).
 
 ## Performance
 
-Super [simple performance test](https://github.com/fabianfett/swift-base64-kit/blob/master/Sources/Base64KitPerformanceTest/main.swift) 
+Super [simple performance test](https://github.com/fabianfett/swift-base64-kit/blob/main/Sources/Base64KitPerformanceTest/main.swift) 
 to ensure speediness of this implementation. Encoding and decoding 1m times the base64 string:
 
 ```
 AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==
 ```
 
-#### macOS
+Tests were run on a MacBook Pro (16-inch, late 2019). Processor: 2.4 GHz 8-Core Intel Core i9.
 
-MacBook Pro (15-inch, late 2016 - the first one with the butterfly keyboard). 
-Quad Core 2.7 GHz Intel Core i7
+#### macOS - swift 5.3
 
 |  | Encoding | Decoding |
 |:--|:--|:--|
-| Foundation   | 2.21s | 2.28s |
-| swift-base64-kit | 1.01s | 1.06s |
-| Speedup | 2.18x | 2.14x |
-
-#### linux
+| Foundation   | 2.08s | 2.15s |
+| swift-base64-kit | 0.66s | 0.54s |
+| Speedup | 3x | 4x |
 
-Whatevar runs GitHub Actions 😉
+#### Linux - swift 5.3
 
 |  | Encoding | Decoding |
 |:--|:--|:--|
-| Foundation   | 33.64s | 3.49s |
-| swift-base64-kit | 1.07s | 1.27s |
-| Speedup | **31.18x** | 2.74x |
-
-I have no idea why Foundation base64 encoding is so slow on linux. 🤷‍♂️
+| Foundation   | 1.01s | 5.5s |
+| swift-base64-kit | 0.27s | 0.41s |
+| Speedup | 3x | **~10x** |
 
 ## Literature for a faster algorithm
 
 I would really like to speed up this project further to be way faster than it is today. Some food for thought of how this could be tackled can be found here:
 
-- [Chromium precomputed lookup tables](https://github.com/lemire/fastbase64/blob/master/src/chromiumbase64.c)
 - [Wojciech Muła, Daniel Lemire: Faster Base64 Encoding and Decoding using AVX2 Instructions](https://arxiv.org/pdf/1704.00605.pdf).
 - [Daniel Lemire's blog - Ridiculously fast base64 encoding and decoding](https://lemire.me/blog/2018/01/17/ridiculously-fast-base64-encoding-and-decoding/)
 - [Swift SIMD support](https://github.com/apple/swift-evolution/blob/master/proposals/0229-simd.md)
@@ -68,4 +61,3 @@ I would really like to speed up this project further to be way faster than it is
 As of today (2019-12-10), the author is aware of only one alternative that offers merely encoding.
 
 - [SwiftyBase64](https://github.com/drichardson/SwiftyBase64)
-