add blob codec tooling

ethereum-consensus/Cargo.toml

@@ -28,6 +28,7 @@ ec = [
     "ctr",
     "base64",
     "unicode-normalization",
+    "bitvec",
 ]
 
 [dependencies]
@@ -70,6 +71,7 @@ aes = { version = "0.8.3", optional = true }
 ctr = { version = "0.9.2", optional = true }
 base64 = { version = "0.21.4", optional = true }
 unicode-normalization = { version = "0.1.22", optional = true }
+bitvec = { version = "1.0.1", optional = true }
 
 [dev-dependencies]
 serde_with = "1.13.0"

ethereum-consensus/src/bin/ec/blobs/command.rs

@@ -0,0 +1,36 @@
+use crate::blobs::{decode, encode};
+use clap::{Args, Subcommand};
+use std::io;
+
+#[derive(Debug, Subcommand)]
+enum Commands {
+    Encode { framing: String },
+    Decode { framing: String },
+}
+
+#[derive(Debug, Args)]
+#[clap(about = "utilities for blobspace")]
+pub struct Command {
+    #[clap(subcommand)]
+    command: Commands,
+}
+
+impl Command {
+    pub fn execute(self) -> eyre::Result<()> {
+        match self.command {
+            Commands::Encode { framing, .. } => {
+                let stdin = io::stdin().lock();
+                let blobs = encode::from_reader(stdin, framing.try_into()?)?;
+                let result = serde_json::to_string_pretty(&blobs)?;
+                println!("{}", result);
+                Ok(())
+            }
+            Commands::Decode { framing, .. } => {
+                let stdin = io::stdin().lock();
+                let stdout = io::stdout().lock();
+                decode::to_writer_from_json(stdin, stdout, framing.try_into()?)?;
+                Ok(())
+            }
+        }
+    }
+}

ethereum-consensus/src/bin/ec/blobs/decode.rs

@@ -0,0 +1,47 @@
+use crate::blobs::{
+    framing::{payload_from_sized, Mode as Framing},
+    Blob, Error, BITS_PER_FIELD_ELEMENT, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT,
+};
+use bitvec::prelude::*;
+use std::io::{Read, Write};
+
+pub fn unpack_from_blobs(blobs: &[Blob]) -> Result<Vec<u8>, Error> {
+    let mut stream = vec![0u8; blobs.len() * BYTES_PER_BLOB];
+    let stream_bits = stream.view_bits_mut::<Lsb0>();
+
+    let mut i = 0;
+    for blob in blobs {
+        let blob_bits = blob.as_ref().view_bits::<Lsb0>();
+        // chunks of serialized field element bits
+        let mut chunks = blob_bits.chunks_exact(8 * BYTES_PER_FIELD_ELEMENT);
+        for chunk in chunks.by_ref() {
+            // only grab the first bits for a field element
+            let src = &chunk[..BITS_PER_FIELD_ELEMENT];
+            stream_bits[i * BITS_PER_FIELD_ELEMENT..(i + 1) * BITS_PER_FIELD_ELEMENT]
+                .copy_from_bitslice(src);
+            i += 1;
+        }
+
+        let remainder = chunks.remainder();
+        assert!(remainder.is_empty());
+    }
+
+    Ok(stream)
+}
+
+// Expects a `Vec<Blob>` with `serde_json` encoding read from `reader`.
+// Writes recovered byte stream to `writer`.
+pub fn to_writer_from_json(
+    reader: impl Read,
+    mut writer: impl Write,
+    framing: Framing,
+) -> Result<(), Error> {
+    let blobs: Vec<Blob> = serde_json::from_reader(reader)?;
+    let result = unpack_from_blobs(&blobs)?;
+    let result = match framing {
+        Framing::Raw => result,
+        Framing::Sized => payload_from_sized(result),
+    };
+    writer.write_all(&result)?;
+    Ok(())
+}

ethereum-consensus/src/bin/ec/blobs/encode.rs

@@ -0,0 +1,65 @@
+use crate::blobs::{
+    framing::{sized_header, Mode as Framing},
+    verify_field_element_bytes, BitSlice, Blob, Error, BITS_PER_FIELD_ELEMENT, BYTES_PER_BLOB,
+    BYTES_PER_FIELD_ELEMENT,
+};
+use bitvec::prelude::*;
+use std::io::Read;
+
+fn field_element_from_bits(src: &BitSlice) -> Result<Vec<u8>, Error> {
+    let mut field_element = vec![0u8; BYTES_PER_FIELD_ELEMENT];
+    let dst = &mut field_element.view_bits_mut()[..src.len()];
+    dst.copy_from_bitslice(src);
+
+    verify_field_element_bytes(&field_element)?;
+    Ok(field_element)
+}
+
+// Pack a buffer of an arbitrary number of bytes into a series of `Blob`s.
+pub fn pack_into_blobs(buffer: Vec<u8>) -> Result<Vec<Blob>, Error> {
+    let mut blobs = vec![];
+    let bits = BitSlice::from_slice(&buffer);
+    let mut blob_buffer = Vec::with_capacity(BYTES_PER_BLOB);
+    let mut chunks = bits.chunks_exact(BITS_PER_FIELD_ELEMENT);
+    for src in chunks.by_ref() {
+        if blob_buffer.len() == BYTES_PER_BLOB {
+            let blob = Blob::try_from(blob_buffer.as_ref()).expect("is the right size");
+            blobs.push(blob);
+            blob_buffer.clear();
+        }
+        let mut field_element = field_element_from_bits(src)?;
+        blob_buffer.append(&mut field_element);
+    }
+
+    let remainder = chunks.remainder();
+    if !remainder.is_empty() {
+        let mut field_element = field_element_from_bits(remainder)?;
+        blob_buffer.append(&mut field_element);
+    }
+
+    // ensure we have only packed complete field elements so far
+    assert!(blob_buffer.len() % BYTES_PER_FIELD_ELEMENT == 0);
+
+    blob_buffer.resize(BYTES_PER_BLOB, 0);
+    let blob = Blob::try_from(blob_buffer.as_ref()).expect("is the right size");
+    blobs.push(blob);
+
+    Ok(blobs)
+}
+
+pub fn from_reader(mut reader: impl Read, framing: Framing) -> Result<Vec<Blob>, Error> {
+    let mut buffer = Vec::with_capacity(BYTES_PER_BLOB);
+
+    reader.read_to_end(&mut buffer).expect("can read data");
+
+    let prepared_buffer = if matches!(framing, Framing::Sized) {
+        let mut header = sized_header(buffer.len())?;
+        let mut framed_buffer = Vec::with_capacity(buffer.len() + header.len());
+        framed_buffer.append(&mut header);
+        framed_buffer.append(&mut buffer);
+        framed_buffer
+    } else {
+        buffer
+    };
+    pack_into_blobs(prepared_buffer)
+}

ethereum-consensus/src/bin/ec/blobs/framing.rs

@@ -0,0 +1,37 @@
+use crate::blobs::Error;
+
+pub const FRAMING_VERSION: u8 = 0;
+
+pub enum Mode {
+    Raw,
+    Sized,
+}
+
+impl TryFrom<String> for Mode {
+    type Error = Error;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        match value.as_str() {
+            "raw" => Ok(Self::Raw),
+            "sized" => Ok(Self::Sized),
+            other => Err(Error::InvalidFrameMode(other.into())),
+        }
+    }
+}
+
+pub fn sized_header(data_byte_length: usize) -> Result<Vec<u8>, Error> {
+    let mut header = vec![0u8; 5];
+    header[0] = FRAMING_VERSION;
+    let size = u32::try_from(data_byte_length)
+        .map_err(|_| Error::ExceedsMaxFrameSize(data_byte_length))?;
+    header[1..].copy_from_slice(&size.to_be_bytes());
+    Ok(header)
+}
+
+pub fn payload_from_sized(stream: Vec<u8>) -> Vec<u8> {
+    assert!(stream.len() >= 5);
+    let (header, payload) = stream.split_at(5);
+    assert!(header[0] == FRAMING_VERSION);
+    let size = u32::from_be_bytes(header[1..5].try_into().expect("correct size bytes")) as usize;
+    payload[..size].to_vec()
+}

ethereum-consensus/src/bin/ec/main.rs

@@ -1,3 +1,4 @@
+mod blobs;
 mod bls;
 mod validator;
 
@@ -7,6 +8,7 @@ use clap::{Parser, Subcommand};
 pub enum Commands {
     Validator(validator::Command),
     Bls(bls::Command),
+    Blobs(blobs::Command),
 }
 
 #[derive(Debug, Parser)]
@@ -22,5 +24,6 @@ fn main() -> eyre::Result<()> {
     match cli.command {
         Commands::Validator(cmd) => cmd.execute(),
         Commands::Bls(cmd) => cmd.execute(),
+        Commands::Blobs(cmd) => cmd.execute(),
     }
 }