modules: Add ZSTD Decoder module

This commit adds an initial implementation of the ZSTD Decoder.
It is capable of decoding simple ZSTD frames containing raw and rle
blocks.

This is a squashed commit that was created from the following changes:

modules/zstd: Add buffer library
modules/zstd: Add Buffer use-case example
modules/zstd: Add library for parsing magic number
modules/zstd: Add library for parsing frame header
dependency_support/libzstd: Make zstd_errors.h public
dependency_support: Add decodecorpus binary
modules/zstd: Add data generator library
modules/zstd: Add zstd frame header tests
modules/zstd: Add common zstd definitions
modules/zstd: Add raw block decoder
modules/zstd: Add rle block decoder
modules/zstd: Add block header parsing library
modules/zstd: Add SequenceExecutorPacket to common definitions
modules/zstd: Add block data muxer library
modules/zstd: Add block demuxer library
modules/zstd: Add block decoder module
modules/zstd/common: Specify decoder output format
examples/ram: Export internal RAM API to other modules
modules/zstd: Add Offset type to common zstd definitions
modules/zstd: Add RamPrinter Proc
modules/zstd: Add SequenceExecutor Proc
modules/zstd: Add repacketizer
modules/zstd: Add ZSTD decoder
modules/zstd: Add ZSTD Decoder documentation
CI: Add custom ZSTD module workflow

Co-authored-by: Maciej Dudek <[email protected]>
Co-authored-by: Pawel Czarnecki <[email protected]>
Co-authored-by: Robert Winkler <[email protected]>
Co-authored-by: Roman Dobrodii <[email protected]>
Internal-tag: [#52186]
Signed-off-by: Maciej Dudek <[email protected]>
Signed-off-by: Pawel Czarnecki <[email protected]>
Signed-off-by: Robert Winkler <[email protected]>
Signed-off-by: Roman Dobrodii <[email protected]>

.github/workflows/modules-zstd.yml

@@ -0,0 +1,73 @@
+name: Modules (ZSTD)
+on:
+  # Avoid triggering on pushes to /all/ open PR branches.
+  push:
+    branches:
+      - main
+    paths:
+      - 'xls/modules/zstd/**'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'xls/modules/zstd/**'
+  # This lets us trigger manually from the UI.
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Test ZSTD module (opt)
+    runs-on:
+      labels: ubuntu-22.04-64core
+    timeout-minutes: 600
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Restore Nightly Bazel Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: "~/.cache/bazel"
+          key: bazel-cache-nightly-${{ runner.os }}-${{ github.sha }}
+          restore-keys: bazel-cache-nightly-${{ runner.os }}-
+
+      - name: Install dependencies via apt
+        run: sudo apt-get install python3-distutils python3-dev python-is-python3 libtinfo5  build-essential liblapack-dev libblas-dev gfortran
+
+      - name: Bazel Build Tools (opt)
+        run: |
+          bazel build -c opt --test_output=errors -- //xls/dslx:interpreter_main //xls/dslx/ir_convert:ir_converter_main //xls/tools:opt_main //xls/tools:codegen_main
+
+      - name: Build ZSTD Module (opt)
+        run: |
+          bazel build -c opt --test_output=errors -- //xls/modules/zstd:all
+
+      - name: Test ZSTD Module - DSLX Tests (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_dslx_test", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Test ZSTD Module - CC Tests (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel test -c opt --test_output=errors -- $(bazel query 'filter(".*_cc_test", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build ZSTD verilog targets (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel build -c opt -- $(bazel query 'filter(".*_verilog", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build and run ZSTD IR benchmark rules (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel run -c opt -- $(bazel query 'filter(".*_ir_benchmark", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build and run synthesis benchmarks of the ZSTD module (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel run -c opt -- $(bazel query 'filter(".*_benchmark_synth", kind(rule, //xls/modules/zstd/...))')
+
+      - name: Build ZSTD place and route targets (opt)
+        if: ${{ !cancelled() }}
+        run: |
+          bazel build -c opt -- $(bazel query 'filter(".*_place_and_route", kind(rule, //xls/modules/zstd/...))')

dependency_support/com_github_facebook_zstd/bundled.BUILD.bazel

@@ -174,3 +174,42 @@ cc_test(
         ":zstd",
     ],
 )
+
+# NOTE: Required because of direct zstd_compress.c include in decodecorpus sources
+cc_library(
+    name = "decodecorpus_includes",
+    hdrs = [
+        "lib/compress/zstd_compress.c",
+    ],
+)
+
+cc_binary(
+    name = "decodecorpus",
+    srcs = [
+        "tests/decodecorpus.c",
+    ] + glob(
+        [
+            "programs/*.c",
+            "programs/*.h",
+        ],
+        exclude = [
+            "programs/zstdcli.c",
+        ],
+    ),
+    deps = [
+        ":zstd",
+        ":decodecorpus_includes",
+    ],
+    includes = [
+        "lib/",
+        "lib/common/",
+        "lib/compress/",
+        "lib/dictBuilder/",
+        "lib/deprecated/",
+        "programs/",
+    ],
+    local_defines = [
+        "XXH_NAMESPACE=ZSTD_",
+    ],
+    visibility = ["//visibility:public"],
+)

xls/examples/ram.x

@@ -55,7 +55,7 @@ pub fn ReadWordReq<NUM_PARTITIONS:u32, ADDR_WIDTH:u32>(addr:uN[ADDR_WIDTH]) ->
 }
 
 // Behavior of reads and writes to the same address in the same "tick".
-enum SimultaneousReadWriteBehavior : u2 {
+pub enum SimultaneousReadWriteBehavior : u2 {
   // The read shows the contents at the address before the write.
   READ_BEFORE_WRITE = 0,
   // The read shows the contents at the address after the write.
@@ -160,7 +160,7 @@ fn write_word_test() {
 
 // Function to compute num partitions (e.g. mask width) for a data_width-wide
 // word divided into word_partition_size-chunks.
-fn num_partitions(word_partition_size: u32, data_width: u32) -> u32 {
+pub fn num_partitions(word_partition_size: u32, data_width: u32) -> u32 {
   match word_partition_size {
     u32:0 => u32:0,
     _ => (word_partition_size + data_width - u32:1) / word_partition_size,
@@ -251,7 +251,7 @@ proc RamModel<DATA_WIDTH:u32, SIZE:u32, WORD_PARTITION_SIZE:u32={u32:0},
     if read_req_valid && ASSERT_VALID_READ {
       let mem_initialized_as_bits =
         std::convert_to_bits_msb0(array_rev(mem_initialized[read_req.addr]));
-      assert_eq(read_req.mask & !mem_initialized_as_bits, uN[NUM_PARTITIONS]:0)
+      assert!(read_req.mask & !mem_initialized_as_bits == uN[NUM_PARTITIONS]:0, "memory_not_initialized")
     } else { () };
 
     let (value_to_write, written_mem_initialized) = write_word(
@@ -265,11 +265,8 @@ proc RamModel<DATA_WIDTH:u32, SIZE:u32, WORD_PARTITION_SIZE:u32={u32:0},
       match SIMULTANEOUS_READ_WRITE_BEHAVIOR {
         SimultaneousReadWriteBehavior::READ_BEFORE_WRITE => mem[read_req.addr],
         SimultaneousReadWriteBehavior::WRITE_BEFORE_READ => value_to_write,
-        SimultaneousReadWriteBehavior::ASSERT_NO_CONFLICT => {
-          // Assertion failure, we have a conflicting read and write.
-          assert_eq(true, false);
-          mem[read_req.addr]  // Need to return something.
-        },
+        SimultaneousReadWriteBehavior::ASSERT_NO_CONFLICT => fail!("conflicting_read_and_write", mem[read_req.addr]),
+        _ => fail!("impossible_case", uN[DATA_WIDTH]:0),
       }
     } else { mem[read_req.addr] };
     let read_resp_value = ReadResp<DATA_WIDTH> {