From 1612f9348da9214200765d34d04889342e82836e Mon Sep 17 00:00:00 2001 From: Yogananth Subramanian Date: Sat, 21 Sep 2024 05:29:10 +0530 Subject: [PATCH 1/4] chore(RHIDP-4040): 1.3-mvp-catalog-size-component-20g --- config/cluster_read_config.test.yaml | 21 +++++++++++++++++++++ test.env | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/config/cluster_read_config.test.yaml b/config/cluster_read_config.test.yaml index 2499d1d..25e6821 100644 --- a/config/cluster_read_config.test.yaml +++ b/config/cluster_read_config.test.yaml @@ -215,6 +215,27 @@ - name: measurements.postgresql.backstage-plugin-search.index command: oc exec $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.name') -n $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.namespace') -- psql -h localhost -U postgres backstage_plugin_search -c "SELECT relname, 100 * idx_scan / (seq_scan + idx_scan) percent_of_times_index_used, n_live_tup rows_in_table FROM pg_stat_user_tables ORDER BY n_live_tup DESC;" -A -F ',' |head -n -1|yq -p csv -o json output: json + +- name: measurements.postgresql.db.size.readable + command: oc exec $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.name') -n $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.namespace') -- psql -h localhost -U postgres backstage_plugin_search -c "SELECT pg_size_pretty(SUM(pg_database_size(datname))) AS total_size FROM pg_database;" -A -F ',' |head -n -1|yq -p csv -o json + output: json + +- name: measurements.postgresql.db.size.raw + command: oc exec $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.name') -n $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.namespace') -- psql -h localhost -U postgres backstage_plugin_search -c "SELECT SUM(pg_database_size(datname)) AS total_size_in_bytes FROM pg_database;" -A -F ',' |head -n -1|yq -p csv -o json + output: json + +- name: measurements.postgresql.dbs.size.readable + command: oc exec $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.name') -n $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.namespace') -- psql -h localhost -U postgres backstage_plugin_search -c "SELECT datname, pg_size_pretty(pg_database_size(datname)) AS size FROM pg_database;" -A -F ',' |head -n -1|yq -p csv -o json + output: json + +- name: measurements.postgresql.dbs.size.raw + command: oc exec $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.name') -n $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.namespace') -- psql -h localhost -U postgres backstage_plugin_search -c "SELECT datname, pg_database_size(datname) AS size_in_bytes FROM pg_database;" -A -F ',' |head -n -1|yq -p csv -o json + output: json + +- name: measurements.postgresql.disk.usage.readable + command: oc exec $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.name') -n $(oc get pods -A -o json | jq -r '.items[] | select(.metadata.namespace | match("{{ namespace_regex }}")) | select(.metadata.name | match("{{ pod_regex }}")).metadata.namespace') -- df -h /var/lib/pgsql/data |yq -p csv -o json + output: json + {%- endmacro %} {{ collect_index_usage('rhdh-performance.*', '(rhdh|backstage)-(postgresql|psql)-(primary|developer-hub)-0') }} diff --git a/test.env b/test.env index f9286e8..ebbd833 100644 --- a/test.env +++ b/test.env @@ -66,3 +66,21 @@ # export SCALE_DB_STORAGES="1Gi 2Gi" # export SCALE_CPU_REQUESTS_LIMITS=": 1:1" # export SCALE_MEMORY_REQUESTS_LIMITS=": 1Gi:1Gi" + +export DURATION=15m +export PRE_LOAD_DB=true +export SCALE_ACTIVE_USERS_SPAWN_RATES='100:5' +export SCALE_BS_USERS_GROUPS='10000:2000' +export SCALE_CATALOG_SIZES='1:1 1:10000 1:25000 1:30000 1:35000 1:40000 1:45000 1:50000' +export SCALE_CPU_REQUESTS_LIMITS=: +export SCALE_DB_STORAGES=20Gi +export SCALE_MEMORY_REQUESTS_LIMITS=: +export SCALE_REPLICAS=1 +export SCALE_WORKERS=20 +export SCENARIO=mvp +export USE_PR_BRANCH=true +export WAIT_FOR_SEARCH_INDEX=false +export RHDH_HELM_CHART=redhat-developer-hub +export AUTH_PROVIDER=keycloak +export RHDH_HELM_REPO=https://raw.githubusercontent.com/rhdh-bot/openshift-helm-charts/refs/heads/rhdh-1.3-rhel-9/installation/ + From db61a0ebaab0ba35e8c9f84c160b25a5534ff5c9 Mon Sep 17 00:00:00 2001 From: Yogananth Subramanian Date: Thu, 3 Oct 2024 06:11:58 +0530 Subject: [PATCH 2/4] chore(RHIDP-4040): 1.3-mvp-catalog-size-component-20g Increase entity timeout to 1800 --- test.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.env b/test.env index ebbd833..460044b 100644 --- a/test.env +++ b/test.env @@ -71,7 +71,7 @@ export DURATION=15m export PRE_LOAD_DB=true export SCALE_ACTIVE_USERS_SPAWN_RATES='100:5' export SCALE_BS_USERS_GROUPS='10000:2000' -export SCALE_CATALOG_SIZES='1:1 1:10000 1:25000 1:30000 1:35000 1:40000 1:45000 1:50000' +export SCALE_CATALOG_SIZES='1:1 1:10000 1:30000 1:50000 1:70000 1:90000' export SCALE_CPU_REQUESTS_LIMITS=: export SCALE_DB_STORAGES=20Gi export SCALE_MEMORY_REQUESTS_LIMITS=: From 11b94d63c07b755779104cef5272095d50de3840 Mon Sep 17 00:00:00 2001 From: Yogananth Subramanian Date: Tue, 8 Oct 2024 16:26:43 +0530 Subject: [PATCH 3/4] chore(RHIDP-4040): 1.3-mvp-catalog-size-component-20g rebase --- test.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.env b/test.env index 460044b..5318e32 100644 --- a/test.env +++ b/test.env @@ -71,7 +71,7 @@ export DURATION=15m export PRE_LOAD_DB=true export SCALE_ACTIVE_USERS_SPAWN_RATES='100:5' export SCALE_BS_USERS_GROUPS='10000:2000' -export SCALE_CATALOG_SIZES='1:1 1:10000 1:30000 1:50000 1:70000 1:90000' +export SCALE_CATALOG_SIZES='1:1 1:10000 1:30000 1:35000 1:40000 1:45000' export SCALE_CPU_REQUESTS_LIMITS=: export SCALE_DB_STORAGES=20Gi export SCALE_MEMORY_REQUESTS_LIMITS=: From 93ab97395b27086d64635253027a887d9f67150d Mon Sep 17 00:00:00 2001 From: Yogananth Subramanian Date: Wed, 9 Oct 2024 06:28:25 +0530 Subject: [PATCH 4/4] chore(RHIDP-4040): 1.3-mvp-catalog-size-component-20g increate entity timeout to 3600 --- ci-scripts/rhdh-setup/create_resource.sh | 2 +- test.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/rhdh-setup/create_resource.sh b/ci-scripts/rhdh-setup/create_resource.sh index e4b94bc..06fc692 100755 --- a/ci-scripts/rhdh-setup/create_resource.sh +++ b/ci-scripts/rhdh-setup/create_resource.sh @@ -117,7 +117,7 @@ clone_and_upload() { ACCESS_TOKEN=$(get_token "rhdh") curl -k "$(backstage_url)/api/catalog/locations" --cookie "$COOKIE" --cookie-jar "$COOKIE" -X POST -H 'Accept-Encoding: gzip, deflate, br' -H 'Authorization: Bearer '"$ACCESS_TOKEN" -H 'Content-Type: application/json' --data-raw '{"type":"url","target":"'"${upload_url}"'"}' - timeout=1800 + timeout=3600 timeout_timestamp=$(date -d "$timeout seconds" "+%s") last_count=-1 while true; do diff --git a/test.env b/test.env index 5318e32..3de8fde 100644 --- a/test.env +++ b/test.env @@ -71,7 +71,7 @@ export DURATION=15m export PRE_LOAD_DB=true export SCALE_ACTIVE_USERS_SPAWN_RATES='100:5' export SCALE_BS_USERS_GROUPS='10000:2000' -export SCALE_CATALOG_SIZES='1:1 1:10000 1:30000 1:35000 1:40000 1:45000' +export SCALE_CATALOG_SIZES='1:1 1:10000 1:20000 1:30000 1:35000 1:40000 1:45000' export SCALE_CPU_REQUESTS_LIMITS=: export SCALE_DB_STORAGES=20Gi export SCALE_MEMORY_REQUESTS_LIMITS=: