Skip to content

Commit

Permalink
Merge branch 'sprint-july-1' into free-reads
Browse files Browse the repository at this point in the history
  • Loading branch information
@din-mukhammed
din-mukhammed authored Jul 9, 2023
2 parents bb14972 + d59d1f6 commit ae1fa90
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 48 deletions.
17 changes: 12 additions & 5 deletions zboxcore/sdk/allocation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ type ConsolidatedFileMeta struct {
Hash string
MimeType string
Size int64
NumBlocks int64
ActualFileSize int64
ActualNumBlocks int64
EncryptedKey string
Expand Down Expand Up @@ -1407,11 +1408,14 @@ func (a *Allocation) GetFileMeta(path string) (*ConsolidatedFileMeta, error) {
result.LookupHash = ref.LookupHash
result.MimeType = ref.MimeType
result.Path = ref.Path
result.Size = ref.ActualFileSize
result.Size = ref.Size
result.NumBlocks = ref.NumBlocks
result.EncryptedKey = ref.EncryptedKey
result.Collaborators = ref.Collaborators
result.ActualFileSize = ref.ActualFileSize
result.ActualNumBlocks = ref.NumBlocks
if result.ActualFileSize > 0 {
result.ActualNumBlocks = (ref.ActualFileSize + CHUNK_SIZE - 1) / CHUNK_SIZE
}
return result, nil
}
return nil, errors.New("file_meta_error", "Error getting the file meta data from blobbers")
Expand Down Expand Up @@ -1461,9 +1465,12 @@ func (a *Allocation) GetFileMetaFromAuthTicket(authTicket string, lookupHash str
result.LookupHash = ref.LookupHash
result.MimeType = ref.MimeType
result.Path = ref.Path
result.Size = ref.ActualFileSize
result.ActualFileSize = ref.Size
result.ActualNumBlocks = ref.NumBlocks
result.Size = ref.Size
result.NumBlocks = ref.NumBlocks
result.ActualFileSize = ref.ActualFileSize
if result.ActualFileSize > 0 {
result.ActualNumBlocks = (result.ActualFileSize + CHUNK_SIZE - 1) / CHUNK_SIZE
}
return result, nil
}
return nil, errors.New("file_meta_error", "Error getting the file meta data from blobbers")
Expand Down
20 changes: 12 additions & 8 deletions zboxcore/sdk/allocation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1606,7 +1606,7 @@ func TestAllocation_ListDirFromAuthTicket(t *testing.T) {
lookupHash: mockLookupHash,
expectedResult: &ListResult{
Type: mockType,
Size: -1,
Size: 0,
},
},
setup: func(t *testing.T, testCaseName string, a *Allocation, mockClient *mocks.HttpClient) (teardown func(t *testing.T)) {
Expand Down Expand Up @@ -1641,9 +1641,11 @@ func TestAllocation_ListDirFromAuthTicket(t *testing.T) {
ClientKey: mockClientKey,
}
a := &Allocation{
ID: mockAllocationId,
Tx: mockAllocationTxId,
FileOptions: 63,
ID: mockAllocationId,
Tx: mockAllocationTxId,
FileOptions: 63,
DataShards: 2,
ParityShards: 2,
}

if tt.setup != nil {
Expand Down Expand Up @@ -1929,7 +1931,7 @@ func TestAllocation_listDir(t *testing.T) {
path: mockPath,
expectedResult: &ListResult{
Type: mockType,
Size: -1,
Size: 0,
},
},
setup: func(t *testing.T, testCaseName string, a *Allocation, mockClient *mocks.HttpClient) (teardown func(t *testing.T)) {
Expand Down Expand Up @@ -1959,9 +1961,11 @@ func TestAllocation_listDir(t *testing.T) {

require := require.New(t)
a := &Allocation{
ID: mockAllocationId,
Tx: mockAllocationTxId,
FileOptions: 63,
ID: mockAllocationId,
Tx: mockAllocationTxId,
FileOptions: 63,
DataShards: 2,
ParityShards: 2,
}
a.InitAllocation()
sdkInitialized = true
Expand Down
49 changes: 23 additions & 26 deletions zboxcore/sdk/listworker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (req *ListRequest) getListInfoFromBlobber(blobber *blockchain.StorageNode,
return errors.Wrap(err, "Error: Resp")
}
s.WriteString(string(resp_body))
l.Logger.Debug("List result:", string(resp_body))
l.Logger.Debug("List result from Blobber:", string(resp_body))
if resp.StatusCode == http.StatusOK {
listResult := &fileref.ListResult{}
err = json.Unmarshal(resp_body, listResult)
Expand Down Expand Up @@ -137,7 +137,7 @@ func (req *ListRequest) getlistFromBlobbers() []*listResponse {
go req.getListInfoFromBlobber(req.blobbers[i], i, rspCh)
}
req.wg.Wait()
listInfos := make([]*listResponse, len(req.blobbers))
listInfos := make([]*listResponse, numList)
for i := 0; i < numList; i++ {
listInfos[i] = <-rspCh
}
Expand All @@ -151,8 +151,8 @@ func (req *ListRequest) GetListFromBlobbers() (*ListResult, error) {
childResultMap := make(map[string]*ListResult)
var err error
var errNum int
req.consensus = 0
for i := 0; i < len(lR); i++ {
req.consensus = 0
ti := lR[i]
if ti.err != nil {
err = ti.err
Expand All @@ -170,30 +170,19 @@ func (req *ListRequest) GetListFromBlobbers() (*ListResult, error) {
result.UpdatedAt = ti.ref.UpdatedAt
result.LookupHash = ti.ref.LookupHash
result.FileMetaHash = ti.ref.FileMetaHash
if result.Type == fileref.DIRECTORY {
result.Size = -1
result.ActualSize = ti.ref.ActualSize
if ti.ref.ActualSize > 0 {
result.ActualNumBlocks = (ti.ref.ActualSize + CHUNK_SIZE - 1) / CHUNK_SIZE
}
result.Size += ti.ref.Size
result.NumBlocks += ti.ref.NumBlocks

if len(lR[i].ref.Children) > 0 {
result.populateChildren(lR[i].ref.Children, childResultMap, selected, req)

if req.forRepair {
for _, child := range childResultMap {
if child.consensus < child.fullconsensus {
if _, ok := selected[child.LookupHash]; !ok {
result.Children = append(result.Children, child)
selected[child.LookupHash] = child
}
}
}
}

for _, child := range result.Children {
result.Size += child.Size
result.NumBlocks += child.NumBlocks
result.ActualSize += child.ActualSize
result.ActualNumBlocks += child.ActualNumBlocks
}
}
req.consensus++
if req.isConsensusOk() {
break
}
}

Expand Down Expand Up @@ -235,9 +224,11 @@ func (lr *ListResult) populateChildren(children []fileref.RefEntity, childResult
childResult.MimeType = (child.(*fileref.FileRef)).MimeType
childResult.EncryptionKey = (child.(*fileref.FileRef)).EncryptedKey
childResult.ActualSize = (child.(*fileref.FileRef)).ActualFileSize
if childResult.ActualSize > 0 {
childResult.ActualNumBlocks = childResult.ActualSize / CHUNK_SIZE
}
} else {
childResult.ActualSize = (child.(*fileref.Ref)).ActualSize
}
if childResult.ActualSize > 0 {
childResult.ActualNumBlocks = (childResult.ActualSize + CHUNK_SIZE - 1) / CHUNK_SIZE
}
childResult.Size += child.GetSize()
childResult.NumBlocks += child.GetNumBlocks()
Expand All @@ -248,5 +239,11 @@ func (lr *ListResult) populateChildren(children []fileref.RefEntity, childResult
selected[child.GetLookupHash()] = childResult
}
}
if req.forRepair {
if _, ok := selected[child.GetLookupHash()]; !ok {
lr.Children = append(lr.Children, childResult)
selected[child.GetLookupHash()] = childResult
}
}
}
}
22 changes: 13 additions & 9 deletions zboxcore/sdk/listworker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,10 @@ func TestListRequest_GetListFromBlobbers(t *testing.T) {
}

tests := []struct {
name string
numBlobbers int
name string
numBlobbers int
consensusThresh int
fullconsensus int

setup func(*testing.T, string, int)
wantFunc func(require *require.Assertions, req *ListRequest)
Expand All @@ -261,12 +263,14 @@ func TestListRequest_GetListFromBlobbers(t *testing.T) {
wantErr: true,
},
{
name: "Test_Success",
numBlobbers: 4,
setup: setupHttpResponses,
name: "Test_Success",
numBlobbers: 4,
consensusThresh: 2,
fullconsensus: 4,
setup: setupHttpResponses,
wantFunc: func(require *require.Assertions, req *ListRequest) {
require.NotNil(req)
require.Equal(float32(4), req.consensus)
require.Equal(float32(2), req.consensus)
},
},
}
Expand All @@ -283,9 +287,9 @@ func TestListRequest_GetListFromBlobbers(t *testing.T) {
blobbers: []*blockchain.StorageNode{},
wg: &sync.WaitGroup{},
Consensus: Consensus{
consensusThresh: tt.consensusThresh,
fullconsensus: tt.fullconsensus,
RWMutex: &sync.RWMutex{},
consensusThresh: 2,
fullconsensus: 4,
},
}
for i := 0; i < tt.numBlobbers; i++ {
Expand All @@ -296,7 +300,7 @@ func TestListRequest_GetListFromBlobbers(t *testing.T) {
got, _ := req.GetListFromBlobbers()
expectedResult := &ListResult{
Type: mockType,
Size: -1,
Size: 0,
}
if !tt.wantErr {
require.EqualValues(expectedResult, got)
Expand Down

0 comments on commit ae1fa90

Please sign in to comment.