Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

improve deployment query performance #1211

Merged
merged 5 commits into from
Jul 10, 2023
Merged

improve deployment query performance #1211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bdfe2fc
deployment queries index
theganyo Jul 6, 2023
4ac5f02
use idx_ prefix like other gorm indexes
theganyo Jul 6, 2023
d2a7b24
Merge branch 'main' into theganyo/issue1208
theganyo Jul 7, 2023
e3a25e9
add page size option to visitors
theganyo Jul 10, 2023
f5fdd8c
set page size to 1k on get command
theganyo Jul 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions cmd/registry/cmd/annotate/annotate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func annotateAPIs(ctx context.Context,
filterFlag string,
labeling *label.Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListAPIs(ctx, client, api, filterFlag, func(ctx context.Context, api *rpc.Api) error {
return visitor.ListAPIs(ctx, client, api, 0, filterFlag, func(ctx context.Context, api *rpc.Api) error {
taskQueue <- &annotateApiTask{
client: client,
api: api,
Expand All @@ -142,7 +142,7 @@ func annotateVersions(
filterFlag string,
labeling *label.Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListVersions(ctx, client, version, filterFlag, func(ctx context.Context, version *rpc.ApiVersion) error {
return visitor.ListVersions(ctx, client, version, 0, filterFlag, func(ctx context.Context, version *rpc.ApiVersion) error {
taskQueue <- &annotateVersionTask{
client: client,
version: version,
Expand All @@ -159,7 +159,7 @@ func annotateSpecs(
filterFlag string,
labeling *label.Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListSpecs(ctx, client, spec, filterFlag, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, 0, filterFlag, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &annotateSpecTask{
client: client,
spec: spec,
Expand All @@ -176,7 +176,7 @@ func annotateDeployments(
filterFlag string,
labeling *label.Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListDeployments(ctx, client, deployment, filterFlag, func(ctx context.Context, deployment *rpc.ApiDeployment) error {
return visitor.ListDeployments(ctx, client, deployment, 0, filterFlag, func(ctx context.Context, deployment *rpc.ApiDeployment) error {
taskQueue <- &annotateDeploymentTask{
client: client,
deployment: deployment,
Expand Down
2 changes: 1 addition & 1 deletion cmd/registry/cmd/check/rules/rule1000/rule1000.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ var requiredArtifacts = &lint.ProjectRule{
}
projectName, _ := names.ParseProject(a.GetName())
var probs []*check.Problem
if err := visitor.ListArtifacts(ctx, client, projectName.Artifact("-"), filter, true, func(ctx context.Context, a *rpc.Artifact) error {
if err := visitor.ListArtifacts(ctx, client, projectName.Artifact("-"), 0, filter, true, func(ctx context.Context, a *rpc.Artifact) error {
found[a.GetName()] = true

name, _ := names.ParseArtifact(a.GetName())
Expand Down
4 changes: 2 additions & 2 deletions cmd/registry/cmd/compute/complexity/complexity.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func Command() *cobra.Command {
}

if parsed.RevisionID == "" {
err = visitor.ListSpecs(ctx, client, parsed.Spec(), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecs(ctx, client, parsed.Spec(), 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeComplexityTask{
client: client,
specName: spec.Name,
Expand All @@ -74,7 +74,7 @@ func Command() *cobra.Command {
return nil
})
} else {
err = visitor.ListSpecRevisions(ctx, client, parsed, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecRevisions(ctx, client, parsed, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeComplexityTask{
client: client,
specName: spec.Name,
Expand Down
6 changes: 3 additions & 3 deletions cmd/registry/cmd/compute/conformance/conformance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ func Command() *cobra.Command {

specs := make([]*rpc.ApiSpec, 0)
if name.RevisionID == "" {
err = visitor.ListSpecs(ctx, client, name.Spec(), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecs(ctx, client, name.Spec(), 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
specs = append(specs, spec)
return nil
})
} else {
err = visitor.ListSpecRevisions(ctx, client, name, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecRevisions(ctx, client, name, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
specs = append(specs, spec)
return nil
})
Expand All @@ -76,7 +76,7 @@ func Command() *cobra.Command {
}

guides := make([]*style.StyleGuide, 0)
if err := visitor.ListArtifacts(ctx, client, name.Project().Artifact("-"), styleguideFilter, true, func(ctx context.Context, artifact *rpc.Artifact) error {
if err := visitor.ListArtifacts(ctx, client, name.Project().Artifact("-"), 0, styleguideFilter, true, func(ctx context.Context, artifact *rpc.Artifact) error {
guide := new(style.StyleGuide)
if err := patch.UnmarshalContents(artifact.GetContents(), artifact.GetMimeType(), guide); err != nil {
log.FromContext(ctx).WithError(err).Debugf("Unmarshal() to StyleGuide failed on artifact: %s", artifact.GetName())
Expand Down
2 changes: 1 addition & 1 deletion cmd/registry/cmd/compute/lint/lint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func Command() *cobra.Command {
}

// Iterate through a collection of specs and evaluate each.
return visitor.ListSpecs(ctx, client, spec, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeLintTask{
client: client,
spec: spec,
Expand Down
14 changes: 7 additions & 7 deletions cmd/registry/cmd/compute/lintstats/lintstats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func computeLintStatsSpecs(ctx context.Context,
filter string,
linter string,
dryRun bool) error {
return visitor.ListSpecs(ctx, client, spec, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
// Iterate through a collection of specs and evaluate each.
log.Debug(ctx, spec.GetName())
// get the lint results
Expand Down Expand Up @@ -181,10 +181,10 @@ func computeLintStatsProjects(ctx context.Context,
filter string,
linter string,
dryRun bool) error {
return visitor.ListProjects(ctx, adminClient, projectName, nil, filter, func(ctx context.Context, project *rpc.Project) error {
return visitor.ListProjects(ctx, adminClient, projectName, nil, 0, filter, func(ctx context.Context, project *rpc.Project) error {
project_stats := &style.LintStats{}

if err := visitor.ListAPIs(ctx, client, projectName.Api(""), filter, func(ctx context.Context, api *rpc.Api) error {
if err := visitor.ListAPIs(ctx, client, projectName.Api(""), 0, filter, func(ctx context.Context, api *rpc.Api) error {
aggregateLintStats(ctx, client, api.GetName(), linter, project_stats)
return nil
}); err != nil {
Expand All @@ -207,10 +207,10 @@ func computeLintStatsAPIs(ctx context.Context,
filter string,
linter string,
dryRun bool) error {
return visitor.ListAPIs(ctx, client, apiName, filter, func(ctx context.Context, api *rpc.Api) error {
return visitor.ListAPIs(ctx, client, apiName, 0, filter, func(ctx context.Context, api *rpc.Api) error {
api_stats := &style.LintStats{}

if err := visitor.ListVersions(ctx, client, apiName.Version(""), filter, func(ctx context.Context, version *rpc.ApiVersion) error {
if err := visitor.ListVersions(ctx, client, apiName.Version(""), 0, filter, func(ctx context.Context, version *rpc.ApiVersion) error {
aggregateLintStats(ctx, client, version.GetName(), linter, api_stats)
return nil
}); err != nil {
Expand All @@ -234,9 +234,9 @@ func computeLintStatsVersions(ctx context.Context,
filter string,
linter string,
dryRun bool) error {
return visitor.ListVersions(ctx, client, versionName, filter, func(ctx context.Context, version *rpc.ApiVersion) error {
return visitor.ListVersions(ctx, client, versionName, 0, filter, func(ctx context.Context, version *rpc.ApiVersion) error {
stats := &style.LintStats{}
if err := visitor.ListSpecs(ctx, client, versionName.Spec(""), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
if err := visitor.ListSpecs(ctx, client, versionName.Spec(""), 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
aggregateLintStats(ctx, client, spec.GetName(), linter, stats)
return nil
}); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/registry/cmd/compute/vocabulary/vocabulary.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func Command() *cobra.Command {

// Iterate through a collection of specs and summarize each.
if parsed.RevisionID == "" {
err = visitor.ListSpecs(ctx, client, parsed.Spec(), filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecs(ctx, client, parsed.Spec(), 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeVocabularyTask{
client: client,
spec: spec,
Expand All @@ -76,7 +76,7 @@ func Command() *cobra.Command {
return nil
})
} else {
err = visitor.ListSpecRevisions(ctx, client, parsed, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
err = visitor.ListSpecRevisions(ctx, client, parsed, 0, filter, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &computeVocabularyTask{
client: client,
spec: spec,
Expand Down
1 change: 1 addition & 0 deletions cmd/registry/cmd/get/get.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ func Command() *cobra.Command {
RegistryClient: registryClient,
AdminClient: adminClient,
Pattern: pattern,
PageSize: 1000,
Filter: filter,
}); err != nil {
if status.Code(err) == codes.NotFound {
Expand Down
8 changes: 4 additions & 4 deletions cmd/registry/cmd/label/label.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func labelAPIs(ctx context.Context,
filterFlag string,
labeling *Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListAPIs(ctx, client, api, filterFlag, func(ctx context.Context, api *rpc.Api) error {
return visitor.ListAPIs(ctx, client, api, 0, filterFlag, func(ctx context.Context, api *rpc.Api) error {
taskQueue <- &labelApiTask{
client: client,
api: api,
Expand All @@ -141,7 +141,7 @@ func labelVersions(
filterFlag string,
labeling *Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListVersions(ctx, client, version, filterFlag, func(ctx context.Context, version *rpc.ApiVersion) error {
return visitor.ListVersions(ctx, client, version, 0, filterFlag, func(ctx context.Context, version *rpc.ApiVersion) error {
taskQueue <- &labelVersionTask{
client: client,
version: version,
Expand All @@ -158,7 +158,7 @@ func labelSpecs(
filterFlag string,
labeling *Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListSpecs(ctx, client, spec, filterFlag, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
return visitor.ListSpecs(ctx, client, spec, 0, filterFlag, false, func(ctx context.Context, spec *rpc.ApiSpec) error {
taskQueue <- &labelSpecTask{
client: client,
spec: spec,
Expand All @@ -175,7 +175,7 @@ func labelDeployments(
filterFlag string,
labeling *Labeling,
taskQueue chan<- tasks.Task) error {
return visitor.ListDeployments(ctx, client, deployment, filterFlag, func(ctx context.Context, deployment *rpc.ApiDeployment) error {
return visitor.ListDeployments(ctx, client, deployment, 0, filterFlag, func(ctx context.Context, deployment *rpc.ApiDeployment) error {
taskQueue <- &labelDeploymentTask{
client: client,
deployment: deployment,
Expand Down
8 changes: 4 additions & 4 deletions cmd/registry/controller/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,19 @@ type RegistryLister struct {
}

func (r *RegistryLister) ListAPIs(ctx context.Context, api names.Api, filter string, handler visitor.ApiHandler) error {
return visitor.ListAPIs(ctx, r.RegistryClient, api, filter, handler)
return visitor.ListAPIs(ctx, r.RegistryClient, api, 0, filter, handler)
}

func (r *RegistryLister) ListVersions(ctx context.Context, version names.Version, filter string, handler visitor.VersionHandler) error {
return visitor.ListVersions(ctx, r.RegistryClient, version, filter, handler)
return visitor.ListVersions(ctx, r.RegistryClient, version, 0, filter, handler)
}

func (r *RegistryLister) ListSpecs(ctx context.Context, spec names.Spec, filter string, handler visitor.SpecHandler) error {
return visitor.ListSpecs(ctx, r.RegistryClient, spec, filter, false, handler)
return visitor.ListSpecs(ctx, r.RegistryClient, spec, 0, filter, false, handler)
}

func (r *RegistryLister) ListArtifacts(ctx context.Context, artifact names.Artifact, filter string, contents bool, handler visitor.ArtifactHandler) error {
return visitor.ListArtifacts(ctx, r.RegistryClient, artifact, filter, contents, handler)
return visitor.ListArtifacts(ctx, r.RegistryClient, artifact, 0, filter, contents, handler)
}

func listResources(ctx context.Context, client listingClient, pattern, filter string) ([]patterns.ResourceInstance, error) {
Expand Down
6 changes: 3 additions & 3 deletions cmd/registry/patch/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func NewApi(ctx context.Context, client *gapic.RegistryClient, message *rpc.Api,
var artifacts []*encoding.Artifact
if nested {
versions = make([]*encoding.ApiVersion, 0)
if err = visitor.ListVersions(ctx, client, apiName.Version("-"), "", func(ctx context.Context, message *rpc.ApiVersion) error {
if err = visitor.ListVersions(ctx, client, apiName.Version("-"), 0, "", func(ctx context.Context, message *rpc.ApiVersion) error {
var version *encoding.ApiVersion
version, err := NewApiVersion(ctx, client, message, true)
if err != nil {
Expand All @@ -63,7 +63,7 @@ func NewApi(ctx context.Context, client *gapic.RegistryClient, message *rpc.Api,
return nil, err
}
deployments = make([]*encoding.ApiDeployment, 0)
if err = visitor.ListDeployments(ctx, client, apiName.Deployment("-"), "", func(ctx context.Context, message *rpc.ApiDeployment) error {
if err = visitor.ListDeployments(ctx, client, apiName.Deployment("-"), 0, "", func(ctx context.Context, message *rpc.ApiDeployment) error {
var deployment *encoding.ApiDeployment
deployment, err = NewApiDeployment(ctx, client, message, true)
if err != nil {
Expand Down Expand Up @@ -109,7 +109,7 @@ func NewApi(ctx context.Context, client *gapic.RegistryClient, message *rpc.Api,

func collectChildArtifacts(ctx context.Context, client *gapic.RegistryClient, artifactPattern names.Artifact) ([]*encoding.Artifact, error) {
artifacts := make([]*encoding.Artifact, 0)
if err := visitor.ListArtifacts(ctx, client, artifactPattern, "", true, func(ctx context.Context, message *rpc.Artifact) error {
if err := visitor.ListArtifacts(ctx, client, artifactPattern, 0, "", true, func(ctx context.Context, message *rpc.Artifact) error {
artifact, err := NewArtifact(ctx, client, message)
if err != nil {
log.FromContext(ctx).Warnf("Skipping %s: %s", message.Name, err)
Expand Down
Loading