Merge pull request #345 from ralexstokes/shuffling-opts

implement optimized index shuffling and add support in spec tests

ethereum-consensus/Cargo.toml

@@ -9,6 +9,8 @@ license = "MIT OR Apache-2.0"
 default = ["serde", "async"]
 serde = ["hex", "serde_json", "serde_yaml"]
 async = ["tokio", "tokio-stream", "async-stream"]
+optimized = ["shuffling"]
+shuffling = [] # supports optimized shuffling routines
 secret-key-debug = [
 ] # enable if you want to be able to print `crypto::SecretKey`
 spec-tests = [] # enable extra features for testing

ethereum-consensus/src/altair/spec/mod.rs

@@ -1154,6 +1154,69 @@ pub fn compute_shuffled_index(
     }
     Ok(index)
 }
+pub fn compute_shuffled_indices(
+    indices: &[ValidatorIndex],
+    seed: &Bytes32,
+    context: &Context,
+) -> ShuffledIndices {
+    let mut input = indices.to_vec();
+    if input.is_empty() {
+        return input;
+    }
+    let index_count = input.len();
+    let mut pivot_input = [0u8; 33];
+    pivot_input[..32].copy_from_slice(seed.as_ref());
+    let mut source_input = [0u8; 37];
+    source_input[..32].copy_from_slice(seed.as_ref());
+    for current_round in (0..=context.shuffle_round_count - 1).rev() {
+        pivot_input[32] = current_round as u8;
+        let pivot_bytes: [u8; 8] = hash(pivot_input).as_ref()[..8].try_into().unwrap();
+        let pivot = u64::from_le_bytes(pivot_bytes) as usize % index_count;
+        source_input[32] = current_round as u8;
+        let position = (pivot >> 8) as u32;
+        source_input[33..].copy_from_slice(&position.to_le_bytes());
+        let mut source = hash(source_input);
+        let mut byte_source = source[(pivot & 0xff) >> 3];
+        let mirror = (pivot + 1) >> 1;
+        for i in 0..mirror {
+            let j = pivot - i;
+            if j & 0xff == 0xff {
+                let position = (j >> 8) as u32;
+                source_input[33..].copy_from_slice(&position.to_le_bytes());
+                source = hash(source_input);
+            }
+            if j & 0x07 == 0x07 {
+                byte_source = source[(j & 0xff) >> 3];
+            }
+            let bit_source = (byte_source >> (j & 0x07)) & 0x01;
+            if bit_source == 1 {
+                input.swap(i, j);
+            }
+        }
+        let end = index_count - 1;
+        let position = (end >> 8) as u32;
+        source_input[33..].copy_from_slice(&position.to_le_bytes());
+        source = hash(source_input);
+        byte_source = source[(end & 0xff) >> 3];
+        let mirror = (pivot + index_count + 1) >> 1;
+        for (k, i) in ((pivot + 1)..mirror).enumerate() {
+            let j = end - k;
+            if j & 0xff == 0xff {
+                let position = (j >> 8) as u32;
+                source_input[33..].copy_from_slice(&position.to_le_bytes());
+                source = hash(source_input);
+            }
+            if j & 0x07 == 0x07 {
+                byte_source = source[(j & 0xff) >> 3];
+            }
+            let bit_source = (byte_source >> (j & 0x07)) & 0x01;
+            if bit_source == 1 {
+                input.swap(i, j);
+            }
+        }
+    }
+    input
+}
 pub fn compute_proposer_index<
     const SLOTS_PER_HISTORICAL_ROOT: usize,
     const HISTORICAL_ROOTS_LIMIT: usize,
@@ -1206,14 +1269,23 @@ pub fn compute_committee(
     count: usize,
     context: &Context,
 ) -> Result<Vec<ValidatorIndex>> {
-    let start = (indices.len() * index) / count;
-    let end = (indices.len()) * (index + 1) / count;
-    let mut committee = vec![0usize; end - start];
-    for i in start..end {
-        let index = compute_shuffled_index(i, indices.len(), seed, context)?;
-        committee[i - start] = indices[index];
+    if cfg!(feature = "shuffling") {
+        let shuffled_indices = compute_shuffled_indices(indices, seed, context);
+        let index_count = indices.len();
+        let start = index_count * index / count;
+        let end = index_count * (index + 1) / count;
+        let committee = shuffled_indices[start..end].to_vec();
+        Ok(committee)
+    } else {
+        let start = indices.len() * index / count;
+        let end = indices.len() * (index + 1) / count;
+        let mut committee = vec![0usize; end - start];
+        for i in start..end {
+            let index = compute_shuffled_index(i, indices.len(), seed, context)?;
+            committee[i - start] = indices[index];
+        }
+        Ok(committee)
     }
-    Ok(committee)
 }
 pub fn compute_epoch_at_slot(slot: Slot, context: &Context) -> Epoch {
     slot / context.slots_per_epoch

ethereum-consensus/src/bellatrix/spec/mod.rs

@@ -2103,6 +2103,69 @@ pub fn compute_shuffled_index(
     }
     Ok(index)
 }
+pub fn compute_shuffled_indices(
+    indices: &[ValidatorIndex],
+    seed: &Bytes32,
+    context: &Context,
+) -> ShuffledIndices {
+    let mut input = indices.to_vec();
+    if input.is_empty() {
+        return input;
+    }
+    let index_count = input.len();
+    let mut pivot_input = [0u8; 33];
+    pivot_input[..32].copy_from_slice(seed.as_ref());
+    let mut source_input = [0u8; 37];
+    source_input[..32].copy_from_slice(seed.as_ref());
+    for current_round in (0..=context.shuffle_round_count - 1).rev() {
+        pivot_input[32] = current_round as u8;
+        let pivot_bytes: [u8; 8] = hash(pivot_input).as_ref()[..8].try_into().unwrap();
+        let pivot = u64::from_le_bytes(pivot_bytes) as usize % index_count;
+        source_input[32] = current_round as u8;
+        let position = (pivot >> 8) as u32;
+        source_input[33..].copy_from_slice(&position.to_le_bytes());
+        let mut source = hash(source_input);
+        let mut byte_source = source[(pivot & 0xff) >> 3];
+        let mirror = (pivot + 1) >> 1;
+        for i in 0..mirror {
+            let j = pivot - i;
+            if j & 0xff == 0xff {
+                let position = (j >> 8) as u32;
+                source_input[33..].copy_from_slice(&position.to_le_bytes());
+                source = hash(source_input);
+            }
+            if j & 0x07 == 0x07 {
+                byte_source = source[(j & 0xff) >> 3];
+            }
+            let bit_source = (byte_source >> (j & 0x07)) & 0x01;
+            if bit_source == 1 {
+                input.swap(i, j);
+            }
+        }
+        let end = index_count - 1;
+        let position = (end >> 8) as u32;
+        source_input[33..].copy_from_slice(&position.to_le_bytes());
+        source = hash(source_input);
+        byte_source = source[(end & 0xff) >> 3];
+        let mirror = (pivot + index_count + 1) >> 1;
+        for (k, i) in ((pivot + 1)..mirror).enumerate() {
+            let j = end - k;
+            if j & 0xff == 0xff {
+                let position = (j >> 8) as u32;
+                source_input[33..].copy_from_slice(&position.to_le_bytes());
+                source = hash(source_input);
+            }
+            if j & 0x07 == 0x07 {
+                byte_source = source[(j & 0xff) >> 3];
+            }
+            let bit_source = (byte_source >> (j & 0x07)) & 0x01;
+            if bit_source == 1 {
+                input.swap(i, j);
+            }
+        }
+    }
+    input
+}
 pub fn compute_proposer_index<
     const SLOTS_PER_HISTORICAL_ROOT: usize,
     const HISTORICAL_ROOTS_LIMIT: usize,
@@ -2159,14 +2222,23 @@ pub fn compute_committee(
     count: usize,
     context: &Context,
 ) -> Result<Vec<ValidatorIndex>> {
-    let start = (indices.len() * index) / count;
-    let end = (indices.len()) * (index + 1) / count;
-    let mut committee = vec![0usize; end - start];
-    for i in start..end {
-        let index = compute_shuffled_index(i, indices.len(), seed, context)?;
-        committee[i - start] = indices[index];
-    }
-    Ok(committee)
+    if cfg!(feature = "shuffling") {
+        let shuffled_indices = compute_shuffled_indices(indices, seed, context);
+        let index_count = indices.len();
+        let start = index_count * index / count;
+        let end = index_count * (index + 1) / count;
+        let committee = shuffled_indices[start..end].to_vec();
+        Ok(committee)
+    } else {
+        let start = indices.len() * index / count;
+        let end = indices.len() * (index + 1) / count;
+        let mut committee = vec![0usize; end - start];
+        for i in start..end {
+            let index = compute_shuffled_index(i, indices.len(), seed, context)?;
+            committee[i - start] = indices[index];
+        }
+        Ok(committee)
+    }
 }
 pub fn compute_epoch_at_slot(slot: Slot, context: &Context) -> Epoch {
     slot / context.slots_per_epoch

ethereum-consensus/src/capella/spec/mod.rs

@@ -2366,6 +2366,69 @@ pub fn compute_shuffled_index(
     }
     Ok(index)
 }
+pub fn compute_shuffled_indices(
+    indices: &[ValidatorIndex],
+    seed: &Bytes32,
+    context: &Context,
+) -> ShuffledIndices {
+    let mut input = indices.to_vec();
+    if input.is_empty() {
+        return input;
+    }
+    let index_count = input.len();
+    let mut pivot_input = [0u8; 33];
+    pivot_input[..32].copy_from_slice(seed.as_ref());
+    let mut source_input = [0u8; 37];
+    source_input[..32].copy_from_slice(seed.as_ref());
+    for current_round in (0..=context.shuffle_round_count - 1).rev() {
+        pivot_input[32] = current_round as u8;
+        let pivot_bytes: [u8; 8] = hash(pivot_input).as_ref()[..8].try_into().unwrap();
+        let pivot = u64::from_le_bytes(pivot_bytes) as usize % index_count;
+        source_input[32] = current_round as u8;
+        let position = (pivot >> 8) as u32;
+        source_input[33..].copy_from_slice(&position.to_le_bytes());
+        let mut source = hash(source_input);
+        let mut byte_source = source[(pivot & 0xff) >> 3];
+        let mirror = (pivot + 1) >> 1;
+        for i in 0..mirror {
+            let j = pivot - i;
+            if j & 0xff == 0xff {
+                let position = (j >> 8) as u32;
+                source_input[33..].copy_from_slice(&position.to_le_bytes());
+                source = hash(source_input);
+            }
+            if j & 0x07 == 0x07 {
+                byte_source = source[(j & 0xff) >> 3];
+            }
+            let bit_source = (byte_source >> (j & 0x07)) & 0x01;
+            if bit_source == 1 {
+                input.swap(i, j);
+            }
+        }
+        let end = index_count - 1;
+        let position = (end >> 8) as u32;
+        source_input[33..].copy_from_slice(&position.to_le_bytes());
+        source = hash(source_input);
+        byte_source = source[(end & 0xff) >> 3];
+        let mirror = (pivot + index_count + 1) >> 1;
+        for (k, i) in ((pivot + 1)..mirror).enumerate() {
+            let j = end - k;
+            if j & 0xff == 0xff {
+                let position = (j >> 8) as u32;
+                source_input[33..].copy_from_slice(&position.to_le_bytes());
+                source = hash(source_input);
+            }
+            if j & 0x07 == 0x07 {
+                byte_source = source[(j & 0xff) >> 3];
+            }
+            let bit_source = (byte_source >> (j & 0x07)) & 0x01;
+            if bit_source == 1 {
+                input.swap(i, j);
+            }
+        }
+    }
+    input
+}
 pub fn compute_proposer_index<
     const SLOTS_PER_HISTORICAL_ROOT: usize,
     const HISTORICAL_ROOTS_LIMIT: usize,
@@ -2422,14 +2485,23 @@ pub fn compute_committee(
     count: usize,
     context: &Context,
 ) -> Result<Vec<ValidatorIndex>> {
-    let start = (indices.len() * index) / count;
-    let end = (indices.len()) * (index + 1) / count;
-    let mut committee = vec![0usize; end - start];
-    for i in start..end {
-        let index = compute_shuffled_index(i, indices.len(), seed, context)?;
-        committee[i - start] = indices[index];
-    }
-    Ok(committee)
+    if cfg!(feature = "shuffling") {
+        let shuffled_indices = compute_shuffled_indices(indices, seed, context);
+        let index_count = indices.len();
+        let start = index_count * index / count;
+        let end = index_count * (index + 1) / count;
+        let committee = shuffled_indices[start..end].to_vec();
+        Ok(committee)
+    } else {
+        let start = indices.len() * index / count;
+        let end = indices.len() * (index + 1) / count;
+        let mut committee = vec![0usize; end - start];
+        for i in start..end {
+            let index = compute_shuffled_index(i, indices.len(), seed, context)?;
+            committee[i - start] = indices[index];
+        }
+        Ok(committee)
+    }
 }
 pub fn compute_epoch_at_slot(slot: Slot, context: &Context) -> Epoch {
     slot / context.slots_per_epoch