From e00e01fc97c6d6a762185617d1c9177ecf6ccbf0 Mon Sep 17 00:00:00 2001 From: Andy Pliszka Date: Mon, 16 Oct 2023 11:53:30 -0400 Subject: [PATCH] Feat add addrbook (#369) * feat: adds download-addrbook.sh script * feat: adds addrbook download * fix(CosmosFullNode): updates CRD with addrbook --- api/v1/cosmosfullnode_types.go | 21 ++++++ api/v1/zz_generated.deepcopy.go | 10 +++ .../cosmos.strange.love_cosmosfullnodes.yaml | 21 ++++++ internal/fullnode/addrbook.go | 40 +++++++++++ internal/fullnode/addrbook_test.go | 72 +++++++++++++++++++ internal/fullnode/pod_builder.go | 12 +++- internal/fullnode/pod_builder_test.go | 31 +++++--- internal/fullnode/script/download-addrbook.sh | 53 ++++++++++++++ 8 files changed, 248 insertions(+), 12 deletions(-) create mode 100644 internal/fullnode/addrbook.go create mode 100644 internal/fullnode/addrbook_test.go create mode 100755 internal/fullnode/script/download-addrbook.sh diff --git a/api/v1/cosmosfullnode_types.go b/api/v1/cosmosfullnode_types.go index 0294a2d6..9c04a0ff 100644 --- a/api/v1/cosmosfullnode_types.go +++ b/api/v1/cosmosfullnode_types.go @@ -445,6 +445,27 @@ type ChainSpec struct { // +optional LogFormat *string `json:"logFormat"` + // URL to address book file to download from the internet. + // The operator detects and properly handles the following file extensions: + // .json, .json.gz, .tar, .tar.gz, .tar.gzip, .zip + // Use AddrbookScript if the chain has an unconventional file format or address book location. + // +optional + AddrbookURL *string `json:"addrbookURL"` + + // Specify shell (sh) script commands to properly download and save the address book file. + // Prefer AddrbookURL if the file is in a conventional format. + // The available shell commands are from docker image ghcr.io/strangelove-ventures/infra-toolkit, including wget and curl. + // Save the file to env var $ADDRBOOK_FILE. + // E.g. curl https://url-to-addrbook.com > $ADDRBOOK_FILE + // Takes precedence over AddrbookURL. + // Hint: Use "set -eux" in your script. + // Available env vars: + // $HOME: The home directory. + // $ADDRBOOK_FILE: The location of the final address book file. + // $CONFIG_DIR: The location of the config dir that houses the address book file. Used for extracting from archives. The archive must have a single file called "addrbook.json". + // +optional + AddrbookScript *string `json:"addrbookScript"` + // URL to genesis file to download from the internet. // Although this field is optional, you will almost always want to set it. // If not set, uses the genesis file created from the init subcommand. (This behavior may be desirable for new chains or testing.) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index a3c13d7f..4a0a7676 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -64,6 +64,16 @@ func (in *ChainSpec) DeepCopyInto(out *ChainSpec) { *out = new(string) **out = **in } + if in.AddrbookURL != nil { + in, out := &in.AddrbookURL, &out.AddrbookURL + *out = new(string) + **out = **in + } + if in.AddrbookScript != nil { + in, out := &in.AddrbookScript, &out.AddrbookScript + *out = new(string) + **out = **in + } if in.GenesisURL != nil { in, out := &in.GenesisURL, &out.GenesisURL *out = new(string) diff --git a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml index f0b78e5b..ce869fa6 100644 --- a/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml +++ b/config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml @@ -45,6 +45,27 @@ spec: chain: description: Blockchain-specific configuration. properties: + addrbookScript: + description: 'Specify shell (sh) script commands to properly download + and save the address book file. Prefer AddrbookURL if the file + is in a conventional format. The available shell commands are + from docker image ghcr.io/strangelove-ventures/infra-toolkit, + including wget and curl. Save the file to env var $ADDRBOOK_FILE. + E.g. curl https://url-to-addrbook.com > $ADDRBOOK_FILE Takes + precedence over AddrbookURL. Hint: Use "set -eux" in your script. + Available env vars: $HOME: The home directory. $ADDRBOOK_FILE: + The location of the final address book file. $CONFIG_DIR: The + location of the config dir that houses the address book file. + Used for extracting from archives. The archive must have a single + file called "addrbook.json".' + type: string + addrbookURL: + description: 'URL to address book file to download from the internet. + The operator detects and properly handles the following file + extensions: .json, .json.gz, .tar, .tar.gz, .tar.gzip, .zip + Use AddrbookScript if the chain has an unconventional file format + or address book location.' + type: string app: description: App configuration applied to app.toml. properties: diff --git a/internal/fullnode/addrbook.go b/internal/fullnode/addrbook.go new file mode 100644 index 00000000..d1f94664 --- /dev/null +++ b/internal/fullnode/addrbook.go @@ -0,0 +1,40 @@ +package fullnode + +import ( + _ "embed" + "fmt" + + cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1" +) + +var ( + //go:embed script/download-addrbook.sh + scriptDownloadAddrbook string +) + +const addrbookScriptWrapper = `ls $CONFIG_DIR/addrbook.json 1> /dev/null 2>&1 +ADDRBOOK_EXISTS=$? +if [ $ADDRBOOK_EXISTS -eq 0 ]; then + echo "Address book already exists" + exit 0 +fi +ls -l $CONFIG_DIR/addrbook.json +%s +ls -l $CONFIG_DIR/addrbook.json + +echo "Address book $ADDRBOOK_FILE downloaded" +` + +// DownloadGenesisCommand returns a proper address book command for use in an init container. +func DownloadAddrbookCommand(cfg cosmosv1.ChainSpec) (string, []string) { + args := []string{"-c"} + switch { + case cfg.AddrbookScript != nil: + args = append(args, fmt.Sprintf(addrbookScriptWrapper, *cfg.AddrbookScript)) + case cfg.AddrbookURL != nil: + args = append(args, fmt.Sprintf(addrbookScriptWrapper, scriptDownloadAddrbook), "-s", *cfg.AddrbookURL) + default: + args = append(args, "echo Using default address book") + } + return "sh", args +} diff --git a/internal/fullnode/addrbook_test.go b/internal/fullnode/addrbook_test.go new file mode 100644 index 00000000..5ca8e783 --- /dev/null +++ b/internal/fullnode/addrbook_test.go @@ -0,0 +1,72 @@ +package fullnode + +import ( + "testing" + + cosmosv1 "github.com/strangelove-ventures/cosmos-operator/api/v1" + "github.com/stretchr/testify/require" +) + +func TestDownloadAddrbookCommand(t *testing.T) { + t.Parallel() + + requireValidScript := func(t *testing.T, script string) { + t.Helper() + require.NotEmpty(t, script) + require.Contains(t, script, `if [ $ADDRBOOK_EXISTS -eq 0 ]`) + } + + t.Run("default", func(t *testing.T) { + var cfg cosmosv1.ChainSpec + + cmd, args := DownloadAddrbookCommand(cfg) + require.Equal(t, "sh", cmd) + + require.Len(t, args, 2) + + require.Equal(t, "-c", args[0]) + + got := args[1] + require.NotContains(t, got, "ADDRBOOK_EXISTS") + require.Contains(t, got, "Using default address book") + }) + + t.Run("download", func(t *testing.T) { + cfg := cosmosv1.ChainSpec{ + AddrbookURL: ptr("https://example.com/addrbook.json"), + } + cmd, args := DownloadAddrbookCommand(cfg) + require.Equal(t, "sh", cmd) + + require.Len(t, args, 4) + + require.Equal(t, "-c", args[0]) + got := args[1] + requireValidScript(t, got) + require.Contains(t, got, `ADDRBOOK_URL`) + require.Contains(t, got, "download_json") + + require.Equal(t, "-s", args[2]) + require.Equal(t, "https://example.com/addrbook.json", args[3]) + }) + + t.Run("custom", func(t *testing.T) { + cfg := cosmosv1.ChainSpec{ + // Keeping this to assert that custom script takes precedence. + AddrbookURL: ptr("https://example.com/addrbook.json"), + AddrbookScript: ptr("echo hi"), + } + cmd, args := DownloadAddrbookCommand(cfg) + require.Equal(t, "sh", cmd) + + require.Len(t, args, 2) + + require.Equal(t, "-c", args[0]) + + got := args[1] + requireValidScript(t, got) + + require.NotContains(t, got, "ADDRBOOK_URL") + require.Contains(t, got, "echo hi") + }) +} diff --git a/internal/fullnode/pod_builder.go b/internal/fullnode/pod_builder.go index 8e94e47f..fba3dfe3 100644 --- a/internal/fullnode/pod_builder.go +++ b/internal/fullnode/pod_builder.go @@ -278,6 +278,7 @@ func envVars(crd *cosmosv1.CosmosFullNode) []corev1.EnvVar { {Name: "HOME", Value: workDir}, {Name: "CHAIN_HOME", Value: home}, {Name: "GENESIS_FILE", Value: path.Join(home, "config", "genesis.json")}, + {Name: "ADDRBOOK_FILE", Value: path.Join(home, "config", "addrbook.json")}, {Name: "CONFIG_DIR", Value: path.Join(home, "config")}, {Name: "DATA_DIR", Value: path.Join(home, "data")}, } @@ -287,6 +288,7 @@ func initContainers(crd *cosmosv1.CosmosFullNode, moniker string) []corev1.Conta tpl := crd.Spec.PodTemplate binary := crd.Spec.ChainSpec.Binary genesisCmd, genesisArgs := DownloadGenesisCommand(crd.Spec.ChainSpec) + addrbookCmd, addrbookArgs := DownloadAddrbookCommand(crd.Spec.ChainSpec) env := envVars(crd) initCmd := fmt.Sprintf("%s init %s --chain-id %s", binary, moniker, crd.Spec.ChainSpec.ChainID) @@ -332,7 +334,15 @@ echo "Initializing into tmp dir for downstream processing..." ImagePullPolicy: tpl.ImagePullPolicy, WorkingDir: workDir, }, - + { + Name: "addrbook-init", + Image: infraToolImage, + Command: []string{addrbookCmd}, + Args: addrbookArgs, + Env: env, + ImagePullPolicy: tpl.ImagePullPolicy, + WorkingDir: workDir, + }, { Name: "config-merge", Image: infraToolImage, diff --git a/internal/fullnode/pod_builder_test.go b/internal/fullnode/pod_builder_test.go index 9558f090..3ba1b8a3 100644 --- a/internal/fullnode/pod_builder_test.go +++ b/internal/fullnode/pod_builder_test.go @@ -222,10 +222,12 @@ func TestPodBuilder(t *testing.T) { require.Equal(t, startContainer.Env[1].Value, "/home/operator/cosmos") require.Equal(t, startContainer.Env[2].Name, "GENESIS_FILE") require.Equal(t, startContainer.Env[2].Value, "/home/operator/cosmos/config/genesis.json") - require.Equal(t, startContainer.Env[3].Name, "CONFIG_DIR") - require.Equal(t, startContainer.Env[3].Value, "/home/operator/cosmos/config") - require.Equal(t, startContainer.Env[4].Name, "DATA_DIR") - require.Equal(t, startContainer.Env[4].Value, "/home/operator/cosmos/data") + require.Equal(t, startContainer.Env[3].Name, "ADDRBOOK_FILE") + require.Equal(t, startContainer.Env[3].Value, "/home/operator/cosmos/config/addrbook.json") + require.Equal(t, startContainer.Env[4].Name, "CONFIG_DIR") + require.Equal(t, startContainer.Env[4].Value, "/home/operator/cosmos/config") + require.Equal(t, startContainer.Env[5].Name, "DATA_DIR") + require.Equal(t, startContainer.Env[5].Value, "/home/operator/cosmos/data") require.Equal(t, envVars(&crd), startContainer.Env) healthContainer := pod.Spec.Containers[1] @@ -242,7 +244,7 @@ func TestPodBuilder(t *testing.T) { } require.Equal(t, healthPort, healthContainer.Ports[0]) - require.Len(t, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Name }), 5) + require.Len(t, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Name }), 6) wantInitImages := []string{ "ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1", @@ -250,6 +252,7 @@ func TestPodBuilder(t *testing.T) { "ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1", "ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1", "ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1", + "ghcr.io/strangelove-ventures/infra-toolkit:v0.0.1", } require.Equal(t, wantInitImages, lo.Map(pod.Spec.InitContainers, func(c corev1.Container, _ int) string { return c.Image @@ -267,7 +270,11 @@ func TestPodBuilder(t *testing.T) { require.Contains(t, initCont.Args[1], `osmosisd init osmosis-6 --chain-id osmosis-123 --home "$CHAIN_HOME"`) require.Contains(t, initCont.Args[1], `osmosisd init osmosis-6 --chain-id osmosis-123 --home "$HOME/.tmp"`) - mergeConfig := pod.Spec.InitContainers[3] + mergeConfig1 := pod.Spec.InitContainers[3] + // The order of config-merge arguments is important. Rightmost takes precedence. + require.Contains(t, mergeConfig1.Args[1], `echo Using default address book`) + + mergeConfig := pod.Spec.InitContainers[4] // The order of config-merge arguments is important. Rightmost takes precedence. require.Contains(t, mergeConfig.Args[1], `config-merge -f toml "$TMP_DIR/config.toml" "$OVERLAY_DIR/config-overlay.toml" > "$CONFIG_DIR/config.toml"`) require.Contains(t, mergeConfig.Args[1], `config-merge -f toml "$TMP_DIR/app.toml" "$OVERLAY_DIR/app-overlay.toml" > "$CONFIG_DIR/app.toml`) @@ -293,10 +300,12 @@ func TestPodBuilder(t *testing.T) { require.Equal(t, container.Env[1].Value, "/home/operator/.osmosisd") require.Equal(t, container.Env[2].Name, "GENESIS_FILE") require.Equal(t, container.Env[2].Value, "/home/operator/.osmosisd/config/genesis.json") - require.Equal(t, container.Env[3].Name, "CONFIG_DIR") - require.Equal(t, container.Env[3].Value, "/home/operator/.osmosisd/config") - require.Equal(t, container.Env[4].Name, "DATA_DIR") - require.Equal(t, container.Env[4].Value, "/home/operator/.osmosisd/data") + require.Equal(t, container.Env[3].Name, "ADDRBOOK_FILE") + require.Equal(t, container.Env[3].Value, "/home/operator/.osmosisd/config/addrbook.json") + require.Equal(t, container.Env[4].Name, "CONFIG_DIR") + require.Equal(t, container.Env[4].Value, "/home/operator/.osmosisd/config") + require.Equal(t, container.Env[5].Name, "DATA_DIR") + require.Equal(t, container.Env[5].Value, "/home/operator/.osmosisd/data") require.NotEmpty(t, pod.Spec.InitContainers) @@ -554,7 +563,7 @@ gaiad start --home /home/operator/cosmos` require.Equal(t, "/foo", extraVol[0].MountPath) initConts := lo.SliceToMap(pod.Spec.InitContainers, func(c corev1.Container) (string, corev1.Container) { return c.Name, c }) - require.ElementsMatch(t, []string{"clean-init", "chain-init", "new-init", "genesis-init", "config-merge"}, lo.Keys(initConts)) + require.ElementsMatch(t, []string{"clean-init", "chain-init", "new-init", "genesis-init", "addrbook-init", "config-merge"}, lo.Keys(initConts)) require.Equal(t, "foo:latest", initConts["chain-init"].Image) }) diff --git a/internal/fullnode/script/download-addrbook.sh b/internal/fullnode/script/download-addrbook.sh new file mode 100755 index 00000000..4eaad8d6 --- /dev/null +++ b/internal/fullnode/script/download-addrbook.sh @@ -0,0 +1,53 @@ +set -eu + +# $ADDRBOOK_FILE and $CONFIG_DIR already set via pod env vars. + +ADDRBOOK_URL="$1" + +echo "Downloading address book file $ADDRBOOK_URL to $ADDRBOOK_FILE..." + +download_json() { + echo "Downloading plain json..." + wget -c -O "$ADDRBOOK_FILE" "$ADDRBOOK_URL" +} + +download_jsongz() { + echo "Downloading json.gz..." + wget -c -O - "$ADDRBOOK_URL" | gunzip -c >"$ADDRBOOK_FILE" +} + +download_tar() { + echo "Downloading and extracting tar..." + wget -c -O - "$ADDRBOOK_URL" | tar -x -C "$CONFIG_DIR" +} + +download_targz() { + echo "Downloading and extracting compressed tar..." + wget -c -O - "$ADDRBOOK_URL" | tar -xz -C "$CONFIG_DIR" +} + +download_zip() { + echo "Downloading and extracting zip..." + wget -c -O tmp_genesis.zip "$ADDRBOOK_URL" + unzip tmp_genesis.zip + rm tmp_genesis.zip + mv genesis.json "$ADDRBOOK_FILE" +} + +rm -f "$ADDRBOOK_FILE" + +case "$ADDRBOOK_URL" in +*.json.gz) download_jsongz ;; +*.json) download_json ;; +*.tar.gz) download_targz ;; +*.tar.gzip) download_targz ;; +*.tar) download_tar ;; +*.zip) download_zip ;; +*) + echo "Unable to handle file extension for $ADDRBOOK_URL" + exit 1 + ;; +esac + +echo "Saved address book file to $ADDRBOOK_FILE." +echo "Download address book file complete."