Skip to content

Commit

Permalink
Add erasure coding to ObjectFetcher
Browse files Browse the repository at this point in the history
  • Loading branch information
teor2345 committed Sep 25, 2024
1 parent 462d65a commit 6e55699
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 6 deletions.
20 changes: 17 additions & 3 deletions shared/subspace-data-retrieval/src/object_fetcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use subspace_archiving::archiver::{Segment, SegmentItem};
use subspace_core_primitives::{
Piece, PieceIndex, RawRecord, RecordedHistorySegment, SegmentIndex,
};
use subspace_erasure_coding::ErasureCoding;
use tracing::{debug, trace};

/// Object fetching errors.
Expand Down Expand Up @@ -128,21 +129,29 @@ pub struct ObjectFetcher {
/// The piece getter used to fetch pieces.
piece_getter: Arc<dyn DsnSyncPieceGetter + Send + Sync + 'static>,

/// The erasure coding configuration of those pieces.
erasure_coding: ErasureCoding,

/// The maximum number of data bytes we'll read for a single object.
max_object_len: usize,
}

impl ObjectFetcher {
/// Create a new object fetcher with the given piece getter.
/// Create a new object fetcher with the given configuration.
///
/// `max_object_len` is the amount of data bytes we'll read for a single object before giving
/// up and returning an error, or `None` for no limit (`usize::MAX`).
pub fn new<PG>(piece_getter: PG, max_object_len: Option<usize>) -> Self
pub fn new<PG>(
piece_getter: PG,
erasure_coding: ErasureCoding,
max_object_len: Option<usize>,
) -> Self
where
PG: DsnSyncPieceGetter + Send + Sync + 'static,
{
Self {
piece_getter: Arc::new(piece_getter),
erasure_coding,
max_object_len: max_object_len.unwrap_or(usize::MAX),
}
}
Expand Down Expand Up @@ -440,7 +449,12 @@ impl ObjectFetcher {

/// Read the whole segment by its index (just records, skipping witnesses).
async fn read_segment(&self, segment_index: SegmentIndex) -> Result<Segment, Error> {
Ok(download_segment(segment_index, &self.piece_getter).await?)
Ok(download_segment(
segment_index,
&self.piece_getter,
self.erasure_coding.clone(),
)
.await?)
}

/// Concurrently read multiple pieces by their indexes
Expand Down
5 changes: 2 additions & 3 deletions shared/subspace-data-retrieval/src/segment_fetcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,12 @@ pub enum SegmentGetterError {
pub async fn download_segment<PG>(
segment_index: SegmentIndex,
piece_getter: &PG,
erasure_coding: ErasureCoding,
) -> Result<Segment, SegmentGetterError>
where
PG: DsnSyncPieceGetter,
{
let reconstructor = Reconstructor::new(
/*TODO*/ ErasureCoding::new(1.try_into().unwrap()).unwrap(),
);
let reconstructor = Reconstructor::new(erasure_coding);

let segment_pieces = download_segment_pieces(segment_index, piece_getter).await?;

Expand Down

0 comments on commit 6e55699

Please sign in to comment.