Add ability to fetch testnet chains and paths + force-add ability (#1285

)

* add testnet and force-add

* update cmd examples

* improve usage description

cmd/chains.go

@@ -291,12 +291,12 @@ func chainsAddCmd(a *appState) *cobra.Command {
 			"                the chain-registry or passing a file (-f) or url (-u)",
 		Args: withUsage(cobra.MinimumNArgs(0)),
 		Example: fmt.Sprintf(` $ %s chains add cosmoshub
- $ %s chains add testnets/cosmoshubtestnet
  $ %s chains add cosmoshub osmosis
+ $ %s chains add cosmoshubtestnet --testnet
  $ %s chains add --file chains/ibc0.json ibc0
  $ %s chains add --url https://relayer.com/ibc0.json ibc0`, appName, appName, appName, appName, appName),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			file, url, err := getAddInputs(cmd)
+			file, url, forceAdd, testnet, err := getAddInputs(cmd)
 			if err != nil {
 				return err
 			}
@@ -330,7 +330,7 @@ func chainsAddCmd(a *appState) *cobra.Command {
 						return err
 					}
 				default:
-					if err := addChainsFromRegistry(cmd.Context(), a, args); err != nil {
+					if err := addChainsFromRegistry(cmd.Context(), a, forceAdd, testnet, args); err != nil {
 						return err
 					}
 				}
@@ -435,7 +435,7 @@ func addChainFromURL(a *appState, chainName string, rawurl string) error {
 	return nil
 }
 
-func addChainsFromRegistry(ctx context.Context, a *appState, chains []string) error {
+func addChainsFromRegistry(ctx context.Context, a *appState, forceAdd, testnet bool, chains []string) error {
 	chainRegistry := cregistry.DefaultChainRegistry(a.log)
 
 	var existed, failed, added []string
@@ -451,7 +451,7 @@ func addChainsFromRegistry(ctx context.Context, a *appState, chains []string) er
 			continue
 		}
 
-		chainInfo, err := chainRegistry.GetChain(ctx, chain)
+		chainInfo, err := chainRegistry.GetChain(ctx, testnet, chain)
 		if err != nil {
 			a.log.Warn(
 				"Error retrieving chain",
@@ -462,7 +462,7 @@ func addChainsFromRegistry(ctx context.Context, a *appState, chains []string) er
 			continue
 		}
 
-		chainConfig, err := chainInfo.GetChainConfig(ctx, chain)
+		chainConfig, err := chainInfo.GetChainConfig(ctx, forceAdd, testnet, chain)
 		if err != nil {
 			a.log.Warn(
 				"Error generating chain config",

cmd/errors.go

@@ -18,5 +18,6 @@ func errChainNotFound(chainName string) error {
 }
 
 var (
-	errMultipleAddFlags = errors.New("expected either --file/-f OR --url/u, found multiple")
+	errMultipleAddFlags   = errors.New("expected either --file/-f OR --url/u, found multiple")
+	errInvalidTestnetFlag = errors.New("cannot use --testnet with --file/-f OR --url/u, must be used alone")
 )

cmd/flags.go

@@ -18,7 +18,9 @@ const (
 	flagJSON                    = "json"
 	flagYAML                    = "yaml"
 	flagFile                    = "file"
+	flagForceAdd                = "force-add"
 	flagPath                    = "path"
+	flagTestnet                 = "testnet"
 	flagMaxTxSize               = "max-tx-size"
 	flagMaxMsgLength            = "max-msgs"
 	flagIBCDenoms               = "ibc-denoms"
@@ -125,6 +127,8 @@ func skipConfirm(v *viper.Viper, cmd *cobra.Command) *cobra.Command {
 func chainsAddFlags(v *viper.Viper, cmd *cobra.Command) *cobra.Command {
 	fileFlag(v, cmd)
 	urlFlag(v, cmd)
+	forceAddFlag(v, cmd)
+	testnetFlag(v, cmd)
 	return cmd
 }
 
@@ -164,6 +168,22 @@ func fileFlag(v *viper.Viper, cmd *cobra.Command) *cobra.Command {
 	return cmd
 }
 
+func testnetFlag(v *viper.Viper, cmd *cobra.Command) *cobra.Command {
+	cmd.Flags().Bool(flagTestnet, false, "fetches testnet data from the chain registry")
+	if err := v.BindPFlag(flagTestnet, cmd.Flags().Lookup(flagTestnet)); err != nil {
+		panic(err)
+	}
+	return cmd
+}
+
+func forceAddFlag(v *viper.Viper, cmd *cobra.Command) *cobra.Command {
+	cmd.Flags().Bool(flagForceAdd, false, "adds chain data even if there are no working RPC's in the chain registry")
+	if err := v.BindPFlag(flagForceAdd, cmd.Flags().Lookup(flagForceAdd)); err != nil {
+		panic(err)
+	}
+	return cmd
+}
+
 func pathFilterFlags(v *viper.Viper, cmd *cobra.Command) *cobra.Command {
 	flags := cmd.Flags()
 	flags.String(flagFilterRule, blankValue, `filter rule ("allowlist", "denylist", or "" for no filtering)`)
@@ -238,7 +258,7 @@ func strategyFlag(v *viper.Viper, cmd *cobra.Command) *cobra.Command {
 	return cmd
 }
 
-func getAddInputs(cmd *cobra.Command) (file string, url string, err error) {
+func getAddInputs(cmd *cobra.Command) (file string, url string, forceAdd bool, testNet bool, err error) {
 	file, err = cmd.Flags().GetString(flagFile)
 	if err != nil {
 		return
@@ -249,8 +269,22 @@ func getAddInputs(cmd *cobra.Command) (file string, url string, err error) {
 		return
 	}
 
+	forceAdd, err = cmd.Flags().GetBool(flagForceAdd)
+	if err != nil {
+		return
+	}
+
+	testNet, err = cmd.Flags().GetBool(flagTestnet)
+	if err != nil {
+		return
+	}
+
 	if file != "" && url != "" {
-		return "", "", errMultipleAddFlags
+		return "", "", false, false, errMultipleAddFlags
+	}
+
+	if file != "" && testNet || url != "" && testNet {
+		return "", "", false, false, errInvalidTestnetFlag
 	}
 
 	return

cmd/paths.go

@@ -368,10 +368,11 @@ func pathsFetchCmd(a *appState) *cobra.Command {
 		Args:    withUsage(cobra.RangeArgs(0, 1)),
 		Example: strings.TrimSpace(fmt.Sprintf(`
 $ %s paths fetch --home %s
-$ %s pth fch
-$ %s pth fch cosmoshub`, appName, defaultHome, appName, appName)),
+$ %s paths fetch --testnet
+$ %s pth fch`, appName, defaultHome, appName, appName)),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			overwrite, _ := cmd.Flags().GetBool(flagOverwriteConfig)
+			testnet, _ := cmd.Flags().GetBool(flagTestnet)
 
 			// allow the relayer to only pull paths for a specific chain
 			chainReq := ""
@@ -418,9 +419,15 @@ $ %s pth fch cosmoshub`, appName, defaultHome, appName, appName)),
 						continue
 					}
 
-					// TODO: Don't use github api. Potentially use: https://github.com/eco-stake/cosmos-directory once they integrate IBC data into restAPI. This will avoid rate limits.
+					// TODO: Don't use github api. Potentially use http.get like GetChain() does to avoid rate limits
 					fileName := pthName + ".json"
-					regPath := path.Join("_IBC", fileName)
+					var regPath string
+					if testnet {
+						regPath = path.Join("testnets", "_IBC", fileName)
+					} else {
+						regPath = path.Join("_IBC", fileName)
+
+					}
 					client, _, err := client.Repositories.DownloadContents(cmd.Context(), "cosmos", "chain-registry", regPath, nil)
 					if err != nil {
 						if errors.As(err, new(*github.RateLimitError)) {
@@ -471,5 +478,7 @@ $ %s pth fch cosmoshub`, appName, defaultHome, appName, appName)),
 			})
 		},
 	}
-	return OverwriteConfigFlag(a.viper, cmd)
+	OverwriteConfigFlag(a.viper, cmd)
+	testnetFlag(a.viper, cmd)
+	return cmd
 }

cregistry/chain_info.go

@@ -186,14 +186,18 @@ func (c ChainInfo) GetRPCEndpoints(ctx context.Context) (out []string, err error
 }
 
 // GetRandomRPCEndpoint returns a string representing a random RPC endpoint from the cosmos chain registry for this chain.
-func (c ChainInfo) GetRandomRPCEndpoint(ctx context.Context) (string, error) {
+func (c ChainInfo) GetRandomRPCEndpoint(ctx context.Context, forceAdd bool) (string, error) {
 	rpcs, err := c.GetRPCEndpoints(ctx)
 	if err != nil {
 		return "", err
 	}
 
 	if len(rpcs) == 0 {
-		return "", fmt.Errorf("no working RPCs found")
+		if !forceAdd {
+			return "", fmt.Errorf("no working RPCs found, consider using --force-add")
+		} else {
+			return "", nil
+		}
 	}
 
 	randomGenerator := rand.New(rand.NewSource(time.Now().UnixNano()))
@@ -206,9 +210,15 @@ func (c ChainInfo) GetRandomRPCEndpoint(ctx context.Context) (string, error) {
 }
 
 // GetAssetList returns the asset metadata from the cosmos chain registry for this particular chain.
-func (c ChainInfo) GetAssetList(ctx context.Context, name string) (AssetList, error) {
-	chainRegURL := fmt.Sprintf("https://raw.githubusercontent.com/cosmos/chain-registry/master/%s/assetlist.json", name)
+func (c ChainInfo) GetAssetList(ctx context.Context, testnet bool, name string) (AssetList, error) {
+	var chainRegURL string
+	if testnet {
+		chainRegURL = fmt.Sprintf("https://raw.githubusercontent.com/cosmos/chain-registry/master/testnets/%s/assetlist.json", name)
 
+	} else {
+		chainRegURL = fmt.Sprintf("https://raw.githubusercontent.com/cosmos/chain-registry/master/%s/assetlist.json", name)
+
+	}
 	res, err := http.Get(chainRegURL)
 	if err != nil {
 		return AssetList{}, err
@@ -236,11 +246,11 @@ func (c ChainInfo) GetAssetList(ctx context.Context, name string) (AssetList, er
 
 // GetChainConfig returns a CosmosProviderConfig composed from the details found in the cosmos chain registry for
 // this particular chain.
-func (c ChainInfo) GetChainConfig(ctx context.Context, name string) (*cosmos.CosmosProviderConfig, error) {
+func (c ChainInfo) GetChainConfig(ctx context.Context, forceAdd, testnet bool, name string) (*cosmos.CosmosProviderConfig, error) {
 	debug := viper.GetBool("debug")
 	home := viper.GetString("home")
 
-	assetList, err := c.GetAssetList(ctx, name)
+	assetList, err := c.GetAssetList(ctx, testnet, name)
 	if err != nil {
 		return nil, err
 	}
@@ -250,7 +260,7 @@ func (c ChainInfo) GetChainConfig(ctx context.Context, name string) (*cosmos.Cos
 		gasPrices = fmt.Sprintf("%.2f%s", 0.01, assetList.Assets[0].Base)
 	}
 
-	rpc, err := c.GetRandomRPCEndpoint(ctx)
+	rpc, err := c.GetRandomRPCEndpoint(ctx, forceAdd)
 	if err != nil {
 		return nil, err
 	}

cregistry/chain_registry.go

@@ -8,7 +8,7 @@ import (
 
 // ChainRegistry is a slim interface that can be implemented to interact with a repository of chain info/metadata.
 type ChainRegistry interface {
-	GetChain(ctx context.Context, name string) (ChainInfo, error)
+	GetChain(ctx context.Context, testnet bool, name string) (ChainInfo, error)
 	ListChains(ctx context.Context) ([]string, error)
 	SourceLink() string
 }

cregistry/cosmos_github_registry.go

@@ -52,8 +52,13 @@ func (c CosmosGithubRegistry) ListChains(ctx context.Context) ([]string, error)
 }
 
 // GetChain attempts to fetch ChainInfo for the specified chain name from the cosmos chain registry.
-func (c CosmosGithubRegistry) GetChain(ctx context.Context, name string) (ChainInfo, error) {
-	chainRegURL := fmt.Sprintf("https://raw.githubusercontent.com/cosmos/chain-registry/master/%s/chain.json", name)
+func (c CosmosGithubRegistry) GetChain(ctx context.Context, testnet bool, name string) (ChainInfo, error) {
+	var chainRegURL string
+	if testnet {
+		chainRegURL = fmt.Sprintf("https://raw.githubusercontent.com/cosmos/chain-registry/master/testnets/%s/chain.json", name)
+	} else {
+		chainRegURL = fmt.Sprintf("https://raw.githubusercontent.com/cosmos/chain-registry/master/%s/chain.json", name)
+	}
 
 	res, err := http.Get(chainRegURL)
 	if err != nil {