FEATURE: add csvsource backtesting support (rebase pull request #1389)

.gitignore

@@ -14,6 +14,7 @@
 *.out
 
 .idea
+.vscode
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
@@ -48,6 +49,7 @@ testoutput
 
 *.swp
 /pkg/backtest/assets.go
+/data/backtest
 
 coverage.txt
 coverage_dum.txt

doc/topics/back-testing.md

@@ -1,9 +1,13 @@
 ## Back-testing
 
-*Before you start back-testing, you need to setup [MySQL](../../README.md#configure-mysql-database) or [SQLite3
+Currently bbgo supports two ways to run backtests:
+
+  1: Through csv data source (supported right now are binance, bybit and OkEx)
+
+  2: Alternatively run backtests through [MySQL](../../README.md#configure-mysql-database) or [SQLite3
 ](../../README.md#configure-sqlite3-database). Using MySQL is highly recommended.*
 
-First, you need to add the back-testing config to your `bbgo.yaml`:
+Let's start by adding the back-testing section to your config eg: `bbgo.yaml`:
 
 ```yaml
 backtest:
@@ -41,8 +45,11 @@ Note on date formats, the following date formats are supported:
 * RFC822, which looks like `02 Jan 06 15:04 MST`
 * You can also use `2021-11-26T15:04:56`
 
-And then, you can sync remote exchange k-lines (candle bars) data for back-testing:
-
+And then, you can sync remote exchange k-lines (candle bars) data for back-testing through csv data source:
+```sh
+bbgo backtest -v --csv --verify --config config/grid.yaml
+```
+or use the sql data source like so:
 ```sh
 bbgo backtest -v --sync --config config/grid.yaml
 ```
@@ -67,6 +74,11 @@ Run back-test:
 ```sh
 bbgo backtest --base-asset-baseline --config config/grid.yaml
 ```
+or through csv data source
+
+```sh
+bbgo backtest -v --csv --base-asset-baseline --config config/grid.yaml --output data/backtest
+```
 
 If you're developing a strategy, you might want to start with a command like this:
 

pkg/backtest/exchange.go

@@ -55,7 +55,7 @@ var ErrEmptyOrderType = errors.New("order type can not be empty string")
 type Exchange struct {
 	sourceName     types.ExchangeName
 	publicExchange types.Exchange
-	srv            *service.BacktestService
+	srv            service.BackTestable
 	currentTime    time.Time
 
 	account *types.Account
@@ -78,7 +78,7 @@ type Exchange struct {
 }
 
 func NewExchange(
-	sourceName types.ExchangeName, sourceExchange types.Exchange, srv *service.BacktestService, config *bbgo.Backtest,
+	sourceName types.ExchangeName, sourceExchange types.Exchange, srv service.BackTestable, config *bbgo.Backtest,
 ) (*Exchange, error) {
 	ex := sourceExchange
 
@@ -366,6 +366,7 @@ func (e *Exchange) SubscribeMarketData(
 			loadedIntervals[sub.Options.Interval] = struct{}{}
 
 		default:
+			// todo support stream back test with csv tick source
 			// Since Environment is not yet been injected at this point, no hard error
 			log.Errorf("stream channel %s is not supported in backtest", sub.Channel)
 		}
@@ -394,6 +395,7 @@ func (e *Exchange) SubscribeMarketData(
 		log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals)
 	}
 
+	log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals)
 	if len(symbols) == 0 {
 		log.Warnf("empty symbols, will not query kline data from the database")
 

pkg/bbgo/config.go

@@ -150,7 +150,8 @@ type Backtest struct {
 	Sessions []string                   `json:"sessions" yaml:"sessions"`
 
 	// sync 1 second interval KLines
-	SyncSecKLines bool `json:"syncSecKLines,omitempty" yaml:"syncSecKLines,omitempty"`
+	SyncSecKLines bool             `json:"syncSecKLines,omitempty" yaml:"syncSecKLines,omitempty"`
+	CsvSource     *CsvSourceConfig `json:"csvConfig,omitempty" yaml:"csvConfig,omitempty"`
 }
 
 func (b *Backtest) GetAccount(n string) BacktestAccount {
@@ -706,3 +707,8 @@ func reUnmarshal(conf interface{}, tpe interface{}) (interface{}, error) {
 
 	return val.Elem().Interface(), nil
 }
+
+type CsvSourceConfig struct {
+	Market      types.MarketType     `json:"market"`
+	Granularity types.MarketDataType `json:"granularity"`
+}

pkg/bbgo/environment.go

@@ -42,9 +42,9 @@ var defaultSyncBufferPeriod = 30 * time.Minute
 // IsBackTesting is a global variable that indicates the current environment is back-test or not.
 var IsBackTesting = false
 
-var BackTestService *service.BacktestService
+var BackTestService service.BackTestable
 
-func SetBackTesting(s *service.BacktestService) {
+func SetBackTesting(s service.BackTestable) {
 	BackTestService = s
 	IsBackTesting = s != nil
 }
@@ -87,7 +87,7 @@ type Environment struct {
 	TradeService      *service.TradeService
 	ProfitService     *service.ProfitService
 	PositionService   *service.PositionService
-	BacktestService   *service.BacktestService
+	BacktestService   service.BackTestable
 	RewardService     *service.RewardService
 	MarginService     *service.MarginService
 	SyncService       *service.SyncService

pkg/cmd/backtest.go

@@ -12,12 +12,6 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/google/uuid"
-
-	"github.com/c9s/bbgo/pkg/cmd/cmdutil"
-	"github.com/c9s/bbgo/pkg/core"
-	"github.com/c9s/bbgo/pkg/data/tsv"
-	"github.com/c9s/bbgo/pkg/util"
-
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -26,13 +20,18 @@ import (
 	"github.com/c9s/bbgo/pkg/accounting/pnl"
 	"github.com/c9s/bbgo/pkg/backtest"
 	"github.com/c9s/bbgo/pkg/bbgo"
+	"github.com/c9s/bbgo/pkg/cmd/cmdutil"
+	"github.com/c9s/bbgo/pkg/core"
+	"github.com/c9s/bbgo/pkg/data/tsv"
 	"github.com/c9s/bbgo/pkg/exchange"
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/service"
 	"github.com/c9s/bbgo/pkg/types"
+	"github.com/c9s/bbgo/pkg/util"
 )
 
 func init() {
+	BacktestCmd.Flags().Bool("csv", false, "use csv data source for exchange (if supported)")
 	BacktestCmd.Flags().Bool("sync", false, "sync backtest data")
 	BacktestCmd.Flags().Bool("sync-only", false, "sync backtest data only, do not run backtest")
 	BacktestCmd.Flags().String("sync-from", "", "sync backtest data from the given time, which will override the time range in the backtest config")
@@ -77,6 +76,11 @@ var BacktestCmd = &cobra.Command{
 			return err
 		}
 
+		modeCsv, err := cmd.Flags().GetBool("csv")
+		if err != nil {
+			return err
+		}
+
 		wantSync, err := cmd.Flags().GetBool("sync")
 		if err != nil {
 			return err
@@ -156,15 +160,29 @@ var BacktestCmd = &cobra.Command{
 		log.Infof("starting backtest with startTime %s", startTime.Format(time.RFC3339))
 
 		environ := bbgo.NewEnvironment()
-		if err := bbgo.BootstrapBacktestEnvironment(ctx, environ); err != nil {
-			return err
-		}
 
-		if environ.DatabaseService == nil {
-			return errors.New("database service is not enabled, please check your environment variables DB_DRIVER and DB_DSN")
+		if userConfig.Backtest.CsvSource == nil {
+			return fmt.Errorf("user config backtest section needs csvsource config")
 		}
+		backtestService := service.NewBacktestServiceCSV(
+			outputDirectory,
+			userConfig.Backtest.CsvSource.Market,
+			userConfig.Backtest.CsvSource.Granularity,
+		)
+		if modeCsv {
+			if err := bbgo.BootstrapEnvironmentLightweight(ctx, environ, userConfig); err != nil {
+				return err
+			}
+		} else {
+			backtestService = service.NewBacktestService(environ.DatabaseService.DB)
+			if err := bbgo.BootstrapBacktestEnvironment(ctx, environ); err != nil {
+				return err
+			}
 
-		backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
+			if environ.DatabaseService == nil {
+				return errors.New("database service is not enabled, please check your environment variables DB_DRIVER and DB_DSN")
+			}
+		}
 		environ.BacktestService = backtestService
 		bbgo.SetBackTesting(backtestService)
 
@@ -692,7 +710,7 @@ func createSymbolReport(
 }
 
 func verify(
-	userConfig *bbgo.Config, backtestService *service.BacktestService,
+	userConfig *bbgo.Config, backtestService service.BackTestable,
 	sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time,
 ) error {
 	for _, sourceExchange := range sourceExchanges {
@@ -735,7 +753,7 @@ func getExchangeIntervals(ex types.Exchange) types.IntervalMap {
 }
 
 func sync(
-	ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService,
+	ctx context.Context, userConfig *bbgo.Config, backtestService service.BackTestable,
 	sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time,
 ) error {
 	for _, symbol := range userConfig.Backtest.Symbols {
@@ -750,10 +768,8 @@ func sync(
 			var intervals = supportIntervals.Slice()
 			intervals.Sort()
 
-			for _, interval := range intervals {
-				if err := backtestService.Sync(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
-					return err
-				}
+			if err := backtestService.Sync(ctx, sourceExchange, symbol, intervals, syncFrom, syncTo); err != nil {
+				return err
 			}
 		}
 	}

pkg/cmd/pnl.go

@@ -135,7 +135,8 @@ var PnLCmd = &cobra.Command{
 
 				// we need the backtest klines for the daily prices
 				backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
-				if err := backtestService.Sync(ctx, exchange, symbol, types.Interval1d, since, until); err != nil {
+				intervals := []types.Interval{types.Interval1d}
+				if err := backtestService.Sync(ctx, exchange, symbol, intervals, since, until); err != nil {
 					return err
 				}
 			}