Skip to content

Commit

Permalink
fix: restart
Browse files Browse the repository at this point in the history
  • Loading branch information
sesheffield committed Sep 24, 2024
1 parent 1e01e10 commit 24a4aab
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 18 deletions.
29 changes: 22 additions & 7 deletions .github/workflows/cd-protonet.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,24 @@
name: Continuous Deployment (Protonet)
# run after every successful CI job of new commits to the master branch
on:
workflow_run:
workflows: [Continuous Integration (Kava Master)]
types:
- completed
push
# workflow_run:
# workflows: [Continuous Integration (Kava Master)]
# types:
# - completed

jobs:
# in order:
# enter standby (prevents autoscaling group from killing node during deploy)
# stop doctor and kava
# download updated binary and genesis
# reset application database state
# restart all once all have been reset
# start kava and doctor
# enter inService
reset-chain-to-zero-state:
# only start cd pipeline if last ci run was successful
if: ${{ github.event.workflow_run.conclusion == 'success' }}
#if: ${{ github.event.workflow_run.conclusion == 'success' }}
uses: ./.github/workflows/cd-reset-protonet.yml
with:
aws-region: us-east-1
Expand All @@ -25,9 +27,22 @@ jobs:
auto_scaling_group_names: kava-protonet-iavl-v1-validator-api-node,kava-protonet-iavl-v1-rpc-api-node
ssm-document-name: kava-protonet-iavl-v1-instance-update
playbook-name: reset-chain.yml
playbook-infrastructure-branch: master
playbook-infrastructure-branch: shef-protonet-validator-network-data-reset
secrets: inherit

restart-chain:
uses: ./.github/workflows/cd-restart-protonet.yml
with:
aws-region: us-east-1
chain-tag-name: ChainId
chain-id: proto_2221-17000
auto_scaling_group_names: kava-protonet-iavl-v1-validator-api-node,kava-protonet-iavl-v1-rpc-api-node
ssm-document-name: kava-protonet-iavl-v1-instance-update
playbook-name: restart-chain.yml
playbook-infrastructure-branch: shef-protonet-validator-network-data-reset
secrets: inherit
needs: [ reset-chain-to-zero-state ]

# setup test and development accounts and balances, deploy contracts by calling the chain's api
seed-chain-state:
uses: ./.github/workflows/cd-seed-chain.yml
Expand All @@ -39,7 +54,7 @@ jobs:
genesis_validator_addresses: "kavavaloper1jaw3g097lq9jdrnscchspr2233yhpsxdlq7ula"
kava_version_filepath: ./ci/env/kava-protonet/KAVA.VERSION
secrets: inherit
needs: [reset-chain-to-zero-state]
needs: [restart-chain]

post-pipeline-metrics:
uses: ./.github/workflows/metric-pipeline.yml
Expand Down
12 changes: 2 additions & 10 deletions .github/workflows/cd-reset-protonet.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
repository: Kava-Labs/infrastructure
token: ${{ secrets.KAVA_PRIVATE_GITHUB_ACCESS_TOKEN }}
path: infrastructure
ref: master
ref: shef-protonet-validator-network-data-reset
- name: Set up Go
uses: actions/setup-go@v4
with:
Expand Down Expand Up @@ -91,12 +91,4 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
AWS_SDK_LOAD_CONFIG: 1
PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}
- name: bring the chain online
run: bash ${GITHUB_WORKSPACE}/.github/scripts/exit-standby-all-chain-nodes.sh
env:
CHAIN_TAG_NAME: ${{ inputs.chain-tag-name }}
CHAIN_ID: ${{ inputs.chain-id }}
AWS_REGION: ${{ inputs.aws-region }}
AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}
102 changes: 102 additions & 0 deletions .github/workflows/cd-restart-protonet.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
name: Restart Protonet

on:
workflow_call:
inputs:
auto_scaling_group_names:
required: true
type: string
description: 'Comma-separated list of auto-scaling group names'
chain-tag-name:
required: false
type: string
default: 'KavaChainId'
chain-id:
required: true
type: string
aws-region:
required: true
type: string
ssm-document-name:
required: true
type: string
playbook-name:
required: true
type: string
playbook-infrastructure-branch:
required: true
type: string
secrets:
CI_AWS_KEY_ID:
required: true
CI_AWS_KEY_SECRET:
required: true
KAVA_PRIVATE_GITHUB_ACCESS_TOKEN:
required: true

# in order:
# enter standby (prevents autoscaling group from killing node during deploy)
# stop doctor and kava
# download updated binary and genesis
# reset application database state
# start kava and doctor
# enter inService
jobs:
restart-traffic:
runs-on: ubuntu-latest
steps:
- name: checkout repo from current commit
uses: actions/checkout@v4
- name: take the chain offline
run: bash ${GITHUB_WORKSPACE}/.github/scripts/put-all-chain-nodes-on-standby.sh
env:
CHAIN_TAG_NAME: ${{ inputs.chain-tag-name }}
CHAIN_ID: ${{ inputs.chain-id }}
AWS_REGION: ${{ inputs.aws-region }}
AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
- name: checkout infrastructure repo
uses: actions/checkout@v4
with:
repository: Kava-Labs/infrastructure
token: ${{ secrets.KAVA_PRIVATE_GITHUB_ACCESS_TOKEN }}
path: infrastructure
ref: shef-protonet-validator-network-data-reset
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version-file: go.mod
- name: build kava node updater
run: cd infrastructure/cli/kava-node-updater && make install && cd ../../../
- name: run reset playbook on all chain nodes
run: |
IFS=',' read -r -a auto_scaling_group_names <<< "$AUTO_SCALING_GROUP_NAMES"
for auto_scaling_group_name in "${auto_scaling_group_names[@]}"; do
kava-node-updater \
--debug \
--max-retries=2 \
--aws-ssm-document-name="$SSM_DOCUMENT_NAME" \
--infrastructure-git-pointer="$PLAYBOOK_INFRASTRUCTURE_BRANCH" \
--update-playbook-filename="$PLAYBOOK_NAME" \
--autoscaling-group-name="$auto_scaling_group_name" \
--max-upgrade-batch-size=0 \
--node-types="" \
--wait-for-node-sync-after-upgrade=true
done
env:
SSM_DOCUMENT_NAME: ${{ inputs.ssm-document-name }}
PLAYBOOK_NAME: ${{ inputs.playbook-name }}
AUTO_SCALING_GROUP_NAMES: ${{ inputs.auto_scaling_group_names }}
AWS_REGION: ${{ inputs.aws-region }}
AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
AWS_SDK_LOAD_CONFIG: 1
PLAYBOOK_INFRASTRUCTURE_BRANCH: ${{ inputs.playbook-infrastructure-branch }}
- name: bring the chain online
run: bash ${GITHUB_WORKSPACE}/.github/scripts/exit-standby-all-chain-nodes.sh
env:
CHAIN_TAG_NAME: ${{ inputs.chain-tag-name }}
CHAIN_ID: ${{ inputs.chain-id }}
AWS_REGION: ${{ inputs.aws-region }}
AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_KEY_SECRET }}
2 changes: 1 addition & 1 deletion ci/env/kava-protonet/KAVA.VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
master
shef-protonet-validator-network-ci

0 comments on commit 24a4aab

Please sign in to comment.