Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Remove per-entity tables #4279

Closed
wants to merge 2 commits into from
Closed

WIP: Remove per-entity tables #4279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1769938
Remove per-entity tables
JAORMX Aug 26, 2024
100f2b4
Add per-entity tables as views to migrate
JAORMX Aug 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 41 additions & 17 deletions cmd/server/app/history_purge_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,12 @@ func TestRecordSize(t *testing.T) {
db.ListEvaluationHistoryStaleRecordsRow{
ID: uuid.Nil,
EvaluationTime: time.Now(),
EntityType: int32(1),
EntityID: uuid.Nil,
RuleID: uuid.Nil,
EntityType: db.NullEntities{
Entities: db.EntitiesRepository,
Valid: true,
},
EntityID: uuid.Nil,
RuleID: uuid.Nil,
},
)

Expand Down Expand Up @@ -76,8 +79,11 @@ func TestPurgeLoop(t *testing.T) {
EvaluationTime: time.Now(),
ID: uuid1,
RuleID: ruleID1,
EntityType: int32(1),
EntityID: entityID1,
EntityType: db.NullEntities{
Entities: db.EntitiesRepository,
Valid: true,
},
EntityID: entityID1,
},
),
withTransactionStuff(),
Expand All @@ -104,8 +110,11 @@ func TestPurgeLoop(t *testing.T) {
EvaluationTime: time.Now(),
ID: uuid1,
RuleID: ruleID1,
EntityType: int32(1),
EntityID: entityID1,
EntityType: db.NullEntities{
Entities: db.EntitiesRepository,
Valid: true,
},
EntityID: entityID1,
},
),
),
Expand All @@ -126,22 +135,31 @@ func TestPurgeLoop(t *testing.T) {
EvaluationTime: time.Now(),
ID: uuid1,
RuleID: ruleID1,
EntityType: int32(1),
EntityID: entityID1,
EntityType: db.NullEntities{
Entities: db.EntitiesRepository,
Valid: true,
},
EntityID: entityID1,
},
db.ListEvaluationHistoryStaleRecordsRow{
EvaluationTime: time.Now(),
ID: uuid2,
RuleID: ruleID2,
EntityType: int32(1),
EntityID: entityID2,
EntityType: db.NullEntities{
Entities: db.EntitiesRepository,
Valid: true,
},
EntityID: entityID2,
},
db.ListEvaluationHistoryStaleRecordsRow{
EvaluationTime: time.Now(),
ID: uuid3,
RuleID: ruleID3,
EntityType: int32(1),
EntityID: entityID3,
EntityType: db.NullEntities{
Entities: db.EntitiesRepository,
Valid: true,
},
EntityID: entityID3,
},
),
withTransactionStuff(),
Expand Down Expand Up @@ -201,8 +219,11 @@ func TestPurgeLoop(t *testing.T) {
EvaluationTime: time.Now(),
ID: uuid1,
RuleID: ruleID1,
EntityType: int32(1),
EntityID: entityID1,
EntityType: db.NullEntities{
Entities: db.EntitiesRepository,
Valid: true,
},
EntityID: entityID1,
},
),
withTransactionStuff(),
Expand Down Expand Up @@ -434,14 +455,17 @@ var (
ruleID3 = uuid.MustParse("00000000-0000-0000-0000-000000000333")
evaluatedAt1 = time.Now()
evaluatedAt2 = evaluatedAt1.Add(-1 * time.Hour)
entityType = int32(1)
entityType = db.NullEntities{
Entities: db.EntitiesRepository,
Valid: true,
}
)

//nolint:unparam
func makeHistoryRow(
id uuid.UUID,
evaluatedAt time.Time,
entityType int32,
entityType db.NullEntities,
entityID uuid.UUID,
ruleID uuid.UUID,
) db.ListEvaluationHistoryStaleRecordsRow {
Expand Down
Empty file added database/migrations/000102_remove_entities_tables.down.sql
View file
Open in desktop
Empty file.
91 changes: 91 additions & 0 deletions database/migrations/000102_remove_entities_tables.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
-- Copyright 2024 Stacklok, Inc
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.

BEGIN;

-- remove the repositories, artifacts and pull_request tables

DROP TABLE IF EXISTS repositories;
DROP TABLE IF EXISTS artifacts;
DROP TABLE IF EXISTS pull_requests;

CREATE VIEW repositories AS
SELECT
ei.id,
ei.project_id,
pr.name AS provider,
ei.provider_id,
(prop_owner.value->>'text')::TEXT AS repo_owner,
(prop_name.value->>'text')::TEXT AS repo_name,
(prop_repo_id.value->>'number')::BIGINT AS repo_id,
(prop_is_private.value->>'boolean')::BOOLEAN AS is_private,
(prop_is_fork.value->>'boolean')::BOOLEAN AS is_fork,
(prop_webhook_id.value->>'number')::BIGINT AS webhook_id,
(prop_webhook_url.value->>'text')::TEXT AS webhook_url,
(prop_deploy_url.value->>'text')::TEXT AS deploy_url,
(prop_clone_url.value->>'text')::TEXT AS clone_url,
(prop_default_branch.value->>'text')::TEXT AS default_branch,
(prop_license.value->>'text')::TEXT AS license,
ei.created_at
FROM
entity_instances ei
JOIN providers pr ON ei.provider_id = pr.id
LEFT JOIN properties prop_owner ON ei.id = prop_owner.entity_id AND prop_owner.key = 'repo_owner'
LEFT JOIN properties prop_name ON ei.id = prop_name.entity_id AND prop_name.key = 'repo_name'
LEFT JOIN properties prop_repo_id ON ei.id = prop_repo_id.entity_id AND prop_repo_id.key = 'repo_id'
LEFT JOIN properties prop_is_private ON ei.id = prop_is_private.entity_id AND prop_is_private.key = 'is_private'
LEFT JOIN properties prop_is_fork ON ei.id = prop_is_fork.entity_id AND prop_is_fork.key = 'is_fork'
LEFT JOIN properties prop_webhook_id ON ei.id = prop_webhook_id.entity_id AND prop_webhook_id.key = 'webhook_id'
LEFT JOIN properties prop_webhook_url ON ei.id = prop_webhook_url.entity_id AND prop_webhook_url.key = 'webhook_url'
LEFT JOIN properties prop_deploy_url ON ei.id = prop_deploy_url.entity_id AND prop_deploy_url.key = 'deploy_url'
LEFT JOIN properties prop_clone_url ON ei.id = prop_clone_url.entity_id AND prop_clone_url.key = 'clone_url'
LEFT JOIN properties prop_default_branch ON ei.id = prop_default_branch.entity_id AND prop_default_branch.key = 'default_branch'
LEFT JOIN properties prop_license ON ei.id = prop_license.entity_id AND prop_license.key = 'license'
WHERE
ei.entity_type = 'repository';

CREATE VIEW artifacts AS
SELECT
ei.id,
ei.project_id,
pr.name AS provider_name,
ei.provider_id,
ei.originated_from AS repository_id,
(prop_artifact_name.value->>'text')::TEXT AS artifact_name,
(prop_artifact_type.value->>'text')::TEXT AS artifact_type,
(prop_artifact_visibility.value->>'text')::TEXT AS artifact_visibility,
ei.created_at
FROM
entity_instances ei
JOIN providers pr ON ei.provider_id = pr.id
LEFT JOIN properties prop_artifact_name ON ei.id = prop_artifact_name.entity_id AND prop_artifact_name.key = 'artifact_name'
LEFT JOIN properties prop_artifact_type ON ei.id = prop_artifact_type.entity_id AND prop_artifact_type.key = 'artifact_type'
LEFT JOIN properties prop_artifact_visibility ON ei.id = prop_artifact_visibility.entity_id AND prop_artifact_visibility.key = 'artifact_visibility'
WHERE
ei.entity_type = 'artifact';

CREATE VIEW pull_requests AS
SELECT
ei.id,
ei.originated_from AS repository_id,
(prop_pr_number.value->>'number')::BIGINT AS pr_number,
ei.created_at
FROM
entity_instances ei
LEFT JOIN properties prop_pr_number ON ei.id = prop_pr_number.entity_id AND prop_pr_number.key = 'pr_number'
WHERE
ei.entity_type = 'pull_request';


COMMIT;
116 changes: 0 additions & 116 deletions database/mock/store.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 1 addition & 22 deletions database/query/artifacts.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,3 @@
-- name: UpsertArtifact :one
INSERT INTO artifacts (
repository_id,
artifact_name,
artifact_type,
artifact_visibility,
project_id,
provider_id,
provider_name
) VALUES ($1, $2, $3, $4, sqlc.arg(project_id), sqlc.arg(provider_id), sqlc.arg(provider_name))
ON CONFLICT (project_id, LOWER(artifact_name))
DO UPDATE SET
artifact_type = $3,
artifact_visibility = $4
WHERE artifacts.repository_id = $1 AND artifacts.artifact_name = $2
RETURNING *;

-- name: GetArtifactByID :one
SELECT * FROM artifacts
WHERE artifacts.id = $1 AND artifacts.project_id = $2;
Expand All @@ -27,8 +10,4 @@ AND artifacts.repository_id = $1 AND artifacts.project_id = $2;
-- name: ListArtifactsByRepoID :many
SELECT * FROM artifacts
WHERE repository_id = $1
ORDER BY id;

-- name: DeleteArtifact :exec
DELETE FROM artifacts
WHERE id = $1;
ORDER BY id;
Loading
Loading