Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PBM-1391: Enabling PITR after physical restore causes PSMDB to crash #1019

Merged
merged 4 commits into from
Sep 26, 2024
Merged

PBM-1391: Enabling PITR after physical restore causes PSMDB to crash #1019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5344a7c
Fix inconsistent data after physical restore
boris-ilijic Sep 20, 2024
717044e
Expand setupNewDB logic with all PBM's collections
boris-ilijic Sep 23, 2024
ca4209b
Merge remote-tracking branch 'origin/dev' into PBM-1391-crash-after-p…
boris-ilijic Sep 24, 2024
3b1d8a5
Merge remote-tracking branch 'origin/dev' into PBM-1391-crash-after-p…
boris-ilijic Sep 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion cmd/pbm-agent/setup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,14 @@ func setupNewDB(ctx context.Context, conn connect.Client) error {
return errors.Wrap(err, "ensure pitr chunks index")
}

err = conn.AdminCommand(
ctx,
bson.D{{"create", defs.PITRCollection}},
).Err()
if err != nil && !strings.Contains(err.Error(), "already exists") {
return errors.Wrap(err, "ensure pitr collection")
}

_, err = conn.BcpCollection().Indexes().CreateMany(
ctx,
[]mongo.IndexModel{
Expand All @@ -124,6 +132,25 @@ func setupNewDB(ctx context.Context, conn connect.Client) error {
},
},
)
if err != nil && !strings.Contains(err.Error(), "already exists") {
return errors.Wrap(err, "ensure backup collection index")
}

err = conn.AdminCommand(
ctx,
bson.D{{"create", defs.RestoresCollection}},
).Err()
if err != nil && !strings.Contains(err.Error(), "already exists") {
return errors.Wrap(err, "ensure restore collection")
}

err = conn.AdminCommand(
ctx,
bson.D{{"create", defs.AgentsStatusCollection}},
).Err()
if err != nil && !strings.Contains(err.Error(), "already exists") {
return errors.Wrap(err, "ensure agent status collection")
}

return err
return nil
}
8 changes: 4 additions & 4 deletions pbm/restore/physical.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1524,13 +1524,13 @@ func (r *PhysRestore) resetRS() error {
return errors.Wrap(err, "turn off pitr")
}

r.dropPBMCollections(ctx, c)
r.cleanUpPBMCollections(ctx, c)
}

return r.shutdown(c)
}

func (r *PhysRestore) dropPBMCollections(ctx context.Context, c *mongo.Client) {
func (r *PhysRestore) cleanUpPBMCollections(ctx context.Context, c *mongo.Client) {
pbmCollections := []string{
defs.LockCollection,
defs.LogCollection,
Expand All @@ -1554,9 +1554,9 @@ func (r *PhysRestore) dropPBMCollections(ctx context.Context, c *mongo.Client) {
defer wg.Done()

r.log.Debug("dropping 'admin.%s'", coll)
err := c.Database(defs.DB).Collection(coll).Drop(ctx)
_, err := c.Database(defs.DB).Collection(coll).DeleteMany(ctx, bson.D{})
if err != nil {
r.log.Warning("failed to drop 'admin.%s': %v", coll, err)
r.log.Warning("failed to delete all from 'admin.%s': %v", coll, err)
}
}()
}
Expand Down
Loading