Skip to content

Commit

Permalink
enhance: Make skip load work for all branches (#37160)
Browse files Browse the repository at this point in the history
Related to #37112

Skip load logic used to work only when there is multiple segment load
info entires in load request. In continous delete case, delegator still
loads l0 segment, which occupies lot of memory.

Signed-off-by: Congqi Xia <[email protected]>
  • Loading branch information
congqixia authored Oct 25, 2024
1 parent ed37c27 commit 05f8807
Showing 1 changed file with 10 additions and 9 deletions.
19 changes: 10 additions & 9 deletions internal/querynodev2/delegator/delegator_data.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,16 @@ func (sd *shardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSeg
log.Debug("worker loads segments...")

sLoad := func(ctx context.Context, req *querypb.LoadSegmentsRequest) error {
info := req.GetInfos()[0]
// put meta l0, instead of load actual delta data
if info.GetLevel() == datapb.SegmentLevel_L0 && sd.l0ForwardPolicy == L0ForwardPolicyRemoteLoad {
l0Seg, err := segments.NewL0Segment(sd.collection, segments.SegmentTypeSealed, req.GetVersion(), info)
if err != nil {
return err
}
sd.segmentManager.Put(ctx, segments.SegmentTypeSealed, l0Seg)
return nil
}
segmentID := req.GetInfos()[0].GetSegmentID()
nodeID := req.GetDstNodeID()
_, err, _ := sd.sf.Do(fmt.Sprintf("%d-%d", nodeID, segmentID), func() (struct{}, error) {
Expand All @@ -441,15 +451,6 @@ func (sd *shardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSeg
if len(req.GetInfos()) > 1 {
var reqs []*querypb.LoadSegmentsRequest
for _, info := range req.GetInfos() {
// put meta l0, instead of load actual delta data
if info.GetLevel() == datapb.SegmentLevel_L0 && sd.l0ForwardPolicy == L0ForwardPolicyRemoteLoad {
l0Seg, err := segments.NewL0Segment(sd.collection, segments.SegmentTypeSealed, req.GetVersion(), info)
if err != nil {
return err
}
sd.segmentManager.Put(ctx, segments.SegmentTypeSealed, l0Seg)
continue
}
newReq := typeutil.Clone(req)
newReq.Infos = []*querypb.SegmentLoadInfo{info}
reqs = append(reqs, newReq)
Expand Down

0 comments on commit 05f8807

Please sign in to comment.