Pretty major re-write of the config package including renaming Config…

…File to Config and moving type defs into configtypes package

src/apps/chifra/internal/scrape/options.go

@@ -24,6 +24,7 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/caps"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/rpc"
@@ -34,17 +35,17 @@ import (
 
 // ScrapeOptions provides all command options for the chifra scrape command.
 type ScrapeOptions struct {
-	BlockCnt  uint64                `json:"blockCnt,omitempty"`  // Maximum number of blocks to process per pass
-	Sleep     float64               `json:"sleep,omitempty"`     // Seconds to sleep between scraper passes
-	Publisher string                `json:"publisher,omitempty"` // For some query options, the publisher of the index
-	Touch     base.Blknum           `json:"touch,omitempty"`     // First block to visit when scraping (snapped back to most recent snap_to_grid mark)
-	RunCount  uint64                `json:"runCount,omitempty"`  // Run the scraper this many times, then quit
-	DryRun    bool                  `json:"dryRun,omitempty"`    // Show the configuration that would be applied if run,no changes are made
-	Notify    bool                  `json:"notify,omitempty"`    // Enable the notify feature
-	Settings  config.ScrapeSettings `json:"settings,omitempty"`  // Configuration items for the scrape
-	Globals   globals.GlobalOptions `json:"globals,omitempty"`   // The global options
-	Conn      *rpc.Connection       `json:"conn,omitempty"`      // The connection to the RPC server
-	BadFlag   error                 `json:"badFlag,omitempty"`   // An error flag if needed
+	BlockCnt  uint64                     `json:"blockCnt,omitempty"`  // Maximum number of blocks to process per pass
+	Sleep     float64                    `json:"sleep,omitempty"`     // Seconds to sleep between scraper passes
+	Publisher string                     `json:"publisher,omitempty"` // For some query options, the publisher of the index
+	Touch     base.Blknum                `json:"touch,omitempty"`     // First block to visit when scraping (snapped back to most recent snap_to_grid mark)
+	RunCount  uint64                     `json:"runCount,omitempty"`  // Run the scraper this many times, then quit
+	DryRun    bool                       `json:"dryRun,omitempty"`    // Show the configuration that would be applied if run,no changes are made
+	Notify    bool                       `json:"notify,omitempty"`    // Enable the notify feature
+	Settings  configtypes.ScrapeSettings `json:"settings,omitempty"`  // Configuration items for the scrape
+	Globals   globals.GlobalOptions      `json:"globals,omitempty"`   // The global options
+	Conn      *rpc.Connection            `json:"conn,omitempty"`      // The connection to the RPC server
+	BadFlag   error                      `json:"badFlag,omitempty"`   // An error flag if needed
 	// EXISTING_CODE
 	PublisherAddr base.Address `json:"-"`
 	// EXISTING_CODE

src/apps/chifra/pkg/config/chainGroup.go

@@ -4,27 +4,17 @@
 
 package config
 
-type chainGroup struct {
-	Chain          string         `toml:"chain,omitempty"`
-	ChainId        string         `toml:"chainId"`
-	IpfsGateway    string         `toml:"ipfsGateway,omitempty"`
-	KeyEndpoint    string         `toml:"keyEndpoint,omitempty"`
-	LocalExplorer  string         `toml:"localExplorer,omitempty"`
-	RemoteExplorer string         `toml:"remoteExplorer,omitempty"`
-	RpcProvider    string         `toml:"rpcProvider"`
-	Symbol         string         `toml:"symbol"`
-	Scrape         ScrapeSettings `toml:"scrape"`
-}
+import "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 
 // GetChain returns the chain for a given chain
-func GetChain(chain string) chainGroup {
+func GetChain(chain string) configtypes.ChainGroup {
 	return GetRootConfig().Chains[chain]
 }
 
 // GetChains returns a list of all chains configured in the config file. Note, there is no "official"
 // list. Users may add their own chains.
-func GetChains() []chainGroup {
-	chainArray := make([]chainGroup, 0, len(GetRootConfig().Chains))
+func GetChains() []configtypes.ChainGroup {
+	chainArray := make([]configtypes.ChainGroup, 0, len(GetRootConfig().Chains))
 	for _, v := range GetRootConfig().Chains {
 		chainArray = append(chainArray, v)
 	}
@@ -33,5 +23,5 @@ func GetChains() []chainGroup {
 
 // IsChainConfigured returns true if the chain is configured in the config file.
 func IsChainConfigured(needle string) bool {
-	return GetRootConfig().Chains != nil && GetRootConfig().Chains[needle] != chainGroup{}
+	return GetRootConfig().Chains != nil && GetRootConfig().Chains[needle] != configtypes.ChainGroup{}
 }

src/apps/chifra/pkg/config/config.go

@@ -1,7 +1,3 @@
-// Copyright 2021 The TrueBlocks Authors. All rights reserved.
-// Use of this source code is governed by a license that can
-// be found in the LICENSE file.
-
 package config
 
 import (
@@ -19,6 +15,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/usage"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
@@ -27,19 +24,10 @@ import (
 
 const envPrefix = "TB_"
 
-var trueBlocksConfig ConfigFile
+var trueBlocksConfig configtypes.Config
 var cachePath string
 var indexPath string
 
-type ConfigFile struct {
-	Version   versionGroup          `toml:"version"`
-	Settings  settingsGroup         `toml:"settings"`
-	Keys      map[string]keyGroup   `toml:"keys"`
-	Pinning   pinningGroup          `toml:"pinning"`
-	Unchained unchainedGroup        `toml:"unchained,omitempty" comment:"Do not edit these values unless instructed to do so."`
-	Chains    map[string]chainGroup `toml:"chains"`
-}
-
 // init sets up default values for the given configuration
 func init() {
 	// The location of the per chain caches
@@ -51,13 +39,13 @@ func init() {
 var configMutex sync.Mutex
 var configLoaded = false
 
-func loadFromTomlFile(filePath string, dest *ConfigFile) error {
+func loadFromTomlFile(filePath string, dest *configtypes.Config) error {
 	return ReadToml(filePath, dest)
 }
 
 // GetRootConfig reads and the configuration located in trueBlocks.toml file. Note
 // that this routine is local to the package
-func GetRootConfig() *ConfigFile {
+func GetRootConfig() *configtypes.Config {
 	if configLoaded {
 		return &trueBlocksConfig
 	}
@@ -67,7 +55,7 @@ func GetRootConfig() *ConfigFile {
 	configPath := PathToRootConfig()
 
 	// First load the default config
-	trueBlocksConfig = *defaultConfig
+	trueBlocksConfig = configtypes.NewConfig(cachePath, indexPath, defaultIpfsGateway)
 
 	// Load TOML file
 	tomlConfigFn := filepath.Join(configPath, "trueBlocks.toml")
@@ -139,7 +127,7 @@ func GetRootConfig() *ConfigFile {
 		}
 		ch.IpfsGateway = clean(ch.IpfsGateway)
 		if ch.Scrape.AppsPerChunk == 0 {
-			settings := ScrapeSettings{
+			settings := configtypes.ScrapeSettings{
 				AppsPerChunk: 2000000,
 				SnapToGrid:   250000,
 				FirstSnap:    2000000,

src/apps/chifra/pkg/config/env.go

@@ -6,10 +6,12 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 )
 
 // loadFromEnv loads configuration from environment variables
-func loadFromEnv(prefix string, destination *ConfigFile) (err error) {
+func loadFromEnv(prefix string, destination *configtypes.Config) (err error) {
 	// First we get all env variables then filter by prefix and finally parse the values
 	envs := os.Environ()
 	for i := 0; i < len(envs); i++ {
@@ -96,7 +98,7 @@ func deepSetByPath(structure *reflect.Value, path []string, value string) error
 				}
 			}
 
-			// Since all maps in ConfigFile have structs as values, we use recursion to set the value
+			// Since all maps in Config have structs as values, we use recursion to set the value
 			// of a correct field of the struct
 			if err := deepSetByPath(&mapValue, path[2:], value); err != nil {
 				return err

src/apps/chifra/pkg/config/env_test.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"testing"
+
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 )
 
 func Test_setByPath(t *testing.T) {
@@ -10,7 +12,7 @@ func Test_setByPath(t *testing.T) {
 		Uint   uint
 		Bool   bool
 		String string
-		ConfigFile
+		configtypes.Config
 	}
 
 	if err := setByPath(&result, []string{"INT"}, "42"); err != nil {
@@ -41,21 +43,21 @@ func Test_setByPath(t *testing.T) {
 		t.Fatal("wrong value", v)
 	}
 
-	if err := setByPath(&result, []string{"CONFIGFILE", "VERSION", "CURRENT"}, "v3.0.0-release"); err != nil {
+	if err := setByPath(&result, []string{"CONFIG", "VERSION", "CURRENT"}, "v3.0.0-release"); err != nil {
 		t.Fatal(err)
 	}
 	if v := result.Version.Current; v != "v3.0.0-release" {
 		t.Fatal("wrong value", v)
 	}
 
-	if err := setByPath(&result, []string{"CONFIGFILE", "KEYS", "PROVIDER", "APIKEY"}, "test-key"); err != nil {
+	if err := setByPath(&result, []string{"CONFIG", "KEYS", "PROVIDER", "APIKEY"}, "test-key"); err != nil {
 		t.Fatal(err)
 	}
 	if v := result.Keys["provider"].ApiKey; v != "test-key" {
 		t.Fatal("wrong value", v)
 	}
 	// Make sure we don't override the whole map when setting another key in already initialized map.
-	if err := setByPath(&result, []string{"CONFIGFILE", "KEYS", "PROVIDER", "SECRET"}, "secret"); err != nil {
+	if err := setByPath(&result, []string{"CONFIG", "KEYS", "PROVIDER", "SECRET"}, "secret"); err != nil {
 		t.Fatal(err)
 	}
 	if v := result.Keys["provider"].ApiKey; v != "test-key" {
@@ -66,7 +68,7 @@ func Test_setByPath(t *testing.T) {
 	}
 
 	// Make sure we don't override the whole map when setting another key in already initialized map.
-	if err := setByPath(&result, []string{"CONFIGFILE", "CHAINS", "MAINNET", "SCRAPE", "APPSPERCHUNK"}, "2000000"); err != nil {
+	if err := setByPath(&result, []string{"CONFIG", "CHAINS", "MAINNET", "SCRAPE", "APPSPERCHUNK"}, "2000000"); err != nil {
 		t.Fatal(err)
 	}
 	if v := result.Chains["mainnet"].Scrape.AppsPerChunk; v != 2000000 {

src/apps/chifra/pkg/config/keyGroup.go

@@ -4,13 +4,8 @@
 
 package config
 
-type keyGroup struct {
-	License string `toml:"license,omitempty"`
-	ApiKey  string `toml:"apiKey"`
-	Secret  string `toml:"secret,omitempty"`
-	Jwt     string `toml:"jwt,omitempty"`
-}
+import "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 
-func GetKey(set string) keyGroup {
+func GetKey(set string) configtypes.KeyGroup {
 	return GetRootConfig().Keys[set]
 }

src/apps/chifra/pkg/config/migrate.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/version"
@@ -25,7 +26,7 @@ func migrate(currentVer version.Version) error {
 	}
 	// note that this routine does not return if it gets this far.
 
-	var cfg ConfigFile
+	var cfg configtypes.Config
 	configFile := PathToConfigFile()
 	if err := ReadToml(configFile, &cfg); err != nil {
 		return err
@@ -36,7 +37,7 @@ func migrate(currentVer version.Version) error {
 		for chain := range cfg.Chains {
 			ch := cfg.Chains[chain]
 			ch.Chain = chain
-			scrape := ScrapeSettings{
+			scrape := configtypes.ScrapeSettings{
 				AppsPerChunk: 2000000,
 				SnapToGrid:   250000,
 				FirstSnap:    2000000,
@@ -68,7 +69,7 @@ func migrate(currentVer version.Version) error {
 
 	vers = version.NewVersion("v2.0.0-release")
 	if currentVer.Uint64() < vers.Uint64() {
-		pinning := pinningGroup{
+		pinning := configtypes.PinningGroup{
 			LocalPinUrl:  "http://localhost:5001",
 			RemotePinUrl: "https://api.pinata.cloud/pinning/pinFileToIPFS",
 		}
@@ -83,7 +84,7 @@ func migrate(currentVer version.Version) error {
 
 	// Re-write the file (after making a backup) with the new version
 	_, _ = file.Copy(configFile+".bak", configFile)
-	_ = cfg.writeFile(configFile, minVersion) // updates the version
+	_ = cfg.WriteFile(configFile, minVersion) // updates the version
 
 	msg := "Your configuration files were upgraded to {%s}. Rerun your command."
 	logger.Fatal(colors.Colored(fmt.Sprintf(msg, minVersion.String())))
@@ -104,7 +105,7 @@ type OldScrape struct {
 	Settings oldScrapeGroup `toml:"settings"`
 }
 
-func MergeScrapeConfig(fn string, scrape *ScrapeSettings) error {
+func MergeScrapeConfig(fn string, scrape *configtypes.ScrapeSettings) error {
 	var sCfg OldScrape
 	if err := ReadToml(fn, &sCfg); err != nil {
 		return err

src/apps/chifra/pkg/config/notifyGroup.go

@@ -3,8 +3,3 @@
 // be found in the LICENSE file.
 
 package config
-
-type notifyGroup struct {
-	Url    string `toml:"url" json:"url,omitempty"`
-	Author string `toml:"author" json:"author,omitempty"`
-}

src/apps/chifra/pkg/config/pinningGroup.go

@@ -7,18 +7,13 @@ package config
 import (
 	"strings"
 
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 	shell "github.com/ipfs/go-ipfs-api"
 )
 
 const defaultIpfsGateway = "https://ipfs.unchainedindex.io/ipfs/"
 
-type pinningGroup struct {
-	GatewayUrl   string `toml:"gatewayUrl" comment:"The pinning gateway to query when downloading the unchained index"`
-	LocalPinUrl  string `toml:"localPinUrl" comment:"The local endpoint for the IPFS daemon"`
-	RemotePinUrl string `toml:"remotePinUrl" comment:"The remote endpoint for pinning on Pinata"`
-}
-
-func GetPinning() pinningGroup {
+func GetPinning() configtypes.PinningGroup {
 	return GetRootConfig().Pinning
 }
 

src/apps/chifra/pkg/config/scrapeGroup.go

@@ -7,37 +7,18 @@ package config
 import (
 	"strconv"
 
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 )
 
-// ScrapeSettings carries config information for the scraper
-type ScrapeSettings struct {
-	AppsPerChunk uint64 `toml:"appsPerChunk" json:"appsPerChunk"`
-	SnapToGrid   uint64 `toml:"snapToGrid" json:"snapToGrid"`
-	FirstSnap    uint64 `toml:"firstSnap" json:"firstSnap"`
-	UnripeDist   uint64 `toml:"unripeDist" json:"unripeDist"`
-	AllowMissing bool   `toml:"allowMissing" json:"allowMissing,omitempty"`
-	ChannelCount uint64 `toml:"channelCount" json:"channelCount,omitempty"`
-}
-
 // GetScrape returns the scraper settings per chain
-func GetScrape(chain string) ScrapeSettings {
+func GetScrape(chain string) configtypes.ScrapeSettings {
 	return GetRootConfig().Chains[chain].Scrape
 }
 
-func (s *ScrapeSettings) TestLog(chain string, test bool) {
-	logger.TestLog(false, "AppsPerChunk: ", s.AppsPerChunk)
-	logger.TestLog(false, "SnapToGrid: ", s.SnapToGrid)
-	logger.TestLog(false, "FirstSnap: ", s.FirstSnap)
-	logger.TestLog(false, "UnripeDist: ", s.UnripeDist)
-	logger.TestLog(false, "ChannelCount: ", s.ChannelCount)
-	logger.TestLog(false, "AllowMissing: ", s.AllowMissing)
-}
-
 func SetScrapeArgs(chain string, args map[string]string) {
 	ch := trueBlocksConfig.Chains[chain]
 
-	empty := ScrapeSettings{}
+	empty := configtypes.ScrapeSettings{}
 	if trueBlocksConfig.Chains[chain].Scrape == empty {
 		ch.Scrape = GetScrape(chain)
 

src/apps/chifra/pkg/config/settingsGroup.go

@@ -4,14 +4,8 @@
 
 package config
 
-type settingsGroup struct {
-	CachePath      string      `toml:"cachePath" comment:"The location of the per chain caches"`
-	IndexPath      string      `toml:"indexPath" comment:"The location of the per chain unchained indexes"`
-	DefaultChain   string      `toml:"defaultChain" comment:"The default chain to use if none is provided"`
-	DefaultGateway string      `toml:"defaultGateway,omitempty"`
-	Notify         notifyGroup `toml:"notify"`
-}
+import "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 
-func GetSettings() settingsGroup {
+func GetSettings() configtypes.SettingsGroup {
 	return GetRootConfig().Settings
 }

src/apps/chifra/pkg/config/unchainedGroup.go

@@ -5,16 +5,12 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/configtypes"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/history"
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-type unchainedGroup struct {
-	PreferredPublisher string `toml:"preferredPublisher,omitempty" comment:"The default publisher of the index if none other is provided"`
-	SmartContract      string `toml:"smartContract,omitempty" comment:"The address of the current version of the Unchained Index"`
-}
-
-func GetUnchained() unchainedGroup {
+func GetUnchained() configtypes.UnchainedGroup {
 	return GetRootConfig().Unchained
 }