SP500 names and sync delay added.

asset/README.md

@@ -24,13 +24,13 @@ The information provided on this project is strictly for informational purposes
 
 ## Index
 
+- [Constants](<#constants>)
 - [func SnapshotsAsClosings\(snapshots \<\-chan \*Snapshot\) \<\-chan float64](<#SnapshotsAsClosings>)
 - [func SnapshotsAsDates\(snapshots \<\-chan \*Snapshot\) \<\-chan time.Time](<#SnapshotsAsDates>)
 - [func SnapshotsAsHighs\(snapshots \<\-chan \*Snapshot\) \<\-chan float64](<#SnapshotsAsHighs>)
 - [func SnapshotsAsLows\(snapshots \<\-chan \*Snapshot\) \<\-chan float64](<#SnapshotsAsLows>)
 - [func SnapshotsAsOpenings\(snapshots \<\-chan \*Snapshot\) \<\-chan float64](<#SnapshotsAsOpenings>)
 - [func SnapshotsAsVolumes\(snapshots \<\-chan \*Snapshot\) \<\-chan int64](<#SnapshotsAsVolumes>)
-- [func Sync\(source, target Repository, defaultStartDate time.Time, workers int\) error](<#Sync>)
 - [type FileSystemRepository](<#FileSystemRepository>)
   - [func NewFileSystemRepository\(base string\) \*FileSystemRepository](<#NewFileSystemRepository>)
   - [func \(r \*FileSystemRepository\) Append\(name string, snapshots \<\-chan \*Snapshot\) error](<#FileSystemRepository.Append>)
@@ -47,6 +47,9 @@ The information provided on this project is strictly for informational purposes
   - [func \(r \*InMemoryRepository\) LastDate\(name string\) \(time.Time, error\)](<#InMemoryRepository.LastDate>)
 - [type Repository](<#Repository>)
 - [type Snapshot](<#Snapshot>)
+- [type Sync](<#Sync>)
+  - [func NewSync\(\) \*Sync](<#NewSync>)
+  - [func \(s \*Sync\) Run\(source, target Repository, defaultStartDate time.Time\) error](<#Sync.Run>)
 - [type TiingoEndOfDay](<#TiingoEndOfDay>)
   - [func \(e \*TiingoEndOfDay\) ToSnapshot\(\) \*Snapshot](<#TiingoEndOfDay.ToSnapshot>)
 - [type TiingoMeta](<#TiingoMeta>)
@@ -59,6 +62,20 @@ The information provided on this project is strictly for informational purposes
   - [func \(r \*TiingoRepository\) LastDate\(name string\) \(time.Time, error\)](<#TiingoRepository.LastDate>)
 
 
+## Constants
+
+<a name="DefaultSyncWorkers"></a>
+
+```go
+const (
+    // DefaultSyncWorkers is the default number of workers to use to synchronize.
+    DefaultSyncWorkers = 1
+
+    // DefaultSyncDelay is the default delay in seconds between each get request.
+    DefaultSyncDelay = 5
+)
+```
+
 <a name="SnapshotsAsClosings"></a>
 ## func [SnapshotsAsClosings](<https://github.com/cinar/indicator/blob/v2/asset/snapshot.go#L79>)
 
@@ -113,15 +130,6 @@ func SnapshotsAsVolumes(snapshots <-chan *Snapshot) <-chan int64
 
 SnapshotsAsVolumes extracts the volume field from each snapshot in the provided channel and returns a new channel containing only those volume values.The original snapshots channel can no longer be directly used afterwards.
 
-<a name="Sync"></a>
-## func [Sync](<https://github.com/cinar/indicator/blob/v2/asset/sync.go#L18>)
-
-```go
-func Sync(source, target Repository, defaultStartDate time.Time, workers int) error
-```
-
-Sync synchronizes assets between the source and target repositories using multi\-worker concurrency.
-
 <a name="FileSystemRepository"></a>
 ## type [FileSystemRepository](<https://github.com/cinar/indicator/blob/v2/asset/file_system_repository.go#L20-L25>)
 
@@ -314,6 +322,39 @@ type Snapshot struct {
 }
 ```
 
+<a name="Sync"></a>
+## type [Sync](<https://github.com/cinar/indicator/blob/v2/asset/sync.go#L25-L31>)
+
+Sync represents the configuration parameters for synchronizing assets between repositories.
+
+```go
+type Sync struct {
+    // Number of workers to use.
+    Workers int
+
+    // Delay between repository get requests to minimize the load to the remote server.
+    Delay int
+}
+```
+
+<a name="NewSync"></a>
+### func [NewSync](<https://github.com/cinar/indicator/blob/v2/asset/sync.go#L34>)
+
+```go
+func NewSync() *Sync
+```
+
+NewSync function initializes a new sync instance with the default parameters.
+
+<a name="Sync.Run"></a>
+### func \(\*Sync\) [Run](<https://github.com/cinar/indicator/blob/v2/asset/sync.go#L42>)
+
+```go
+func (s *Sync) Run(source, target Repository, defaultStartDate time.Time) error
+```
+
+Run synchronizes assets between the source and target repositories using multi\-worker concurrency.
+
 <a name="TiingoEndOfDay"></a>
 ## type [TiingoEndOfDay](<https://github.com/cinar/indicator/blob/v2/asset/tiingo_repository.go#L41-L80>)
 

asset/sync.go

@@ -13,9 +13,33 @@ import (
 	"github.com/cinar/indicator/helper"
 )
 
-// Sync synchronizes assets between the source and target repositories using
-// multi-worker concurrency.
-func Sync(source, target Repository, defaultStartDate time.Time, workers int) error {
+const (
+	// DefaultSyncWorkers is the default number of workers to use to synchronize.
+	DefaultSyncWorkers = 1
+
+	// DefaultSyncDelay is the default delay in seconds between each get request.
+	DefaultSyncDelay = 5
+)
+
+// Sync represents the configuration parameters for synchronizing assets between repositories.
+type Sync struct {
+	// Number of workers to use.
+	Workers int
+
+	// Delay between repository get requests to minimize the load to the remote server.
+	Delay int
+}
+
+// NewSync function initializes a new sync instance with the default parameters.
+func NewSync() *Sync {
+	return &Sync{
+		Workers: DefaultSyncWorkers,
+		Delay:   DefaultSyncDelay,
+	}
+}
+
+// Run synchronizes assets between the source and target repositories using multi-worker concurrency.
+func (s *Sync) Run(source, target Repository, defaultStartDate time.Time) error {
 	assets, err := target.Assets()
 	if err != nil {
 		return err
@@ -27,7 +51,7 @@ func Sync(source, target Repository, defaultStartDate time.Time, workers int) er
 	hasErrors := false
 	wg := &sync.WaitGroup{}
 
-	for i := 0; i < workers; i++ {
+	for i := 0; i < s.Workers; i++ {
 		wg.Add(1)
 
 		go func() {
@@ -54,6 +78,8 @@ func Sync(source, target Repository, defaultStartDate time.Time, workers int) er
 					hasErrors = true
 					continue
 				}
+
+				time.Sleep(time.Duration(s.Delay) * time.Second)
 			}
 		}()
 	}

asset/sync_test.go

@@ -63,7 +63,11 @@ func TestSync(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = asset.Sync(source, target, snapshots[0].Date, 1)
+	sync := asset.NewSync()
+	sync.Workers = 1
+	sync.Delay = 0
+
+	err = sync.Run(source, target, snapshots[0].Date)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -94,7 +98,11 @@ func TestSyncMissingOnSource(t *testing.T) {
 
 	defaultStartDate := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 
-	err = asset.Sync(source, target, defaultStartDate, 1)
+	sync := asset.NewSync()
+	sync.Workers = 1
+	sync.Delay = 0
+
+	err = sync.Run(source, target, defaultStartDate)
 	if err == nil {
 		t.Fatal("expected error")
 	}
@@ -110,7 +118,11 @@ func TestSyncFailingTargetAssets(t *testing.T) {
 
 	defaultStartDate := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 
-	err := asset.Sync(source, target, defaultStartDate, 1)
+	sync := asset.NewSync()
+	sync.Workers = 1
+	sync.Delay = 0
+
+	err := sync.Run(source, target, defaultStartDate)
 	if err == nil {
 		t.Fatal("expected error")
 	}
@@ -139,7 +151,11 @@ func TestSyncFailingTargetAppend(t *testing.T) {
 
 	defaultStartDate := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 
-	err := asset.Sync(source, target, defaultStartDate, 1)
+	sync := asset.NewSync()
+	sync.Workers = 1
+	sync.Delay = 0
+
+	err := sync.Run(source, target, defaultStartDate)
 	if err == nil {
 		t.Fatal("expected error")
 	}

cmd/indicator-sync/main.go

@@ -20,6 +20,7 @@ func main() {
 	var targetBase string
 	var minusDays int
 	var workers int
+	var delay int
 
 	fmt.Fprintln(os.Stderr, "Indicator Sync")
 	fmt.Fprintln(os.Stderr, "Copyright (c) 2021-2023 Onur Cinar.")
@@ -30,7 +31,8 @@ func main() {
 	flag.StringVar(&tiingoKey, "key", "", "tiingo service api key")
 	flag.StringVar(&targetBase, "target", ".", "target repository base directory")
 	flag.IntVar(&minusDays, "days", 0, "lookback period in days for the new assets")
-	flag.IntVar(&workers, "workers", 1, "number of concurrent workers")
+	flag.IntVar(&workers, "workers", asset.DefaultSyncWorkers, "number of concurrent workers")
+	flag.IntVar(&delay, "delay", asset.DefaultSyncDelay, "delay between each get")
 	flag.Parse()
 
 	if tiingoKey == "" {
@@ -42,7 +44,11 @@ func main() {
 	source := asset.NewTiingoRepository(tiingoKey)
 	target := asset.NewFileSystemRepository(targetBase)
 
-	err := asset.Sync(source, target, defaultStartDate, workers)
+	sync := asset.NewSync()
+	sync.Workers = workers
+	sync.Delay = delay
+
+	err := sync.Run(source, target, defaultStartDate)
 	if err != nil {
 		log.Fatal(err)
 	}

sp500.fish

@@ -0,0 +1,40 @@
+#!/usr/bin/env fish
+
+set SP_500 \
+A AAL AAPL  ABBV ABNB ABT ACGL ACN  ADBE ADI ADM ADP ADSK  AEE AEP AES\
+AFL AIG AIZ AJG AKAM ALB ALGN ALL  ALLE AMAT AMCR AMD AME AMGN AMP AMT\
+AMZN ANET ANSS AON AOS APA APD APH  APTV ARE ATO AVB AVGO AVY AWK AXON\
+AXP AZO BA BAC BALL BAX BBWI BBY  BDX BEN BF.B BG BIIB BIO BK BKNG BKR\
+BLDR BLK BMY  BR BRK.B BRO BSX BWA  BX BXP C CAG CAH CARR  CAT CB CBOE\
+CBRE CCI CCL CDAY CDNS CDW CE CEG  CF CFG CHD CHRW CHTR CI CINF CL CLX\
+CMA CMCSA CME  CMG CMI CMS CNC CNP  COF COO COP COR COST  CPB CPRT CPT\
+CRL CRM CSCO CSGP CSX CTAS CTLT CTRA CTSH CTVA CVS CVX CZR D DAL DD DE\
+DFS DG DGX DHI  DHR DIS DLR DLTR DOV DOW DPZ DRI  DTE DUK DVA DVN DXCM\
+EA EBAY ECL ED EFX EG EIX EL ELV EMN EMR ENPH EOG EPAM EQIX EQR EQT ES\
+ESS ETN ETR ETSY EVRG EW EXC EXPD  EXPE EXR F FANG FAST FCX FDS FDX FE\
+FFIV FI FICO  FIS FITB FLT FMC FOX  FOXA FRT FSLR FTNT FTV  GD GE GEHC\
+GEN GILD  GIS GL GLW GM  GNRC GOOG GOOGL GPC  GPN GRMN GS GWW  HAL HAS\
+HBAN HCA HD HES HIG HII HLT HOLX HON HPE HPQ HRL HSIC HST HSY HUBB HUM\
+HWM IBM ICE  IDXX IEX IFF ILMN INCY  INTC INTU INVH IP IPG  IQV IR IRM\
+ISRG IT ITW  IVZ J JBHT JBL JCI  JKHY JNJ JNPR JPM K KDP  KEY KEYS KHC\
+KIM KLAC KMB KMI KMX KO KR KVUE L  LDOS LEN LH LHX LIN LKQ LLY LMT LNT\
+LOW LRCX LULU LUV LVS LW LYB LYV  MA MAA MAR MAS MCD MCHP MCK MCO MDLZ\
+MDT MET META MGM MHK MKC MKTX MLM MMC MMM MNST MO MOH MOS MPC MPWR MRK\
+MRNA MRO MS MSCI MSFT MSI MTB MTCH  MTD MU NCLH NDAQ NDSN NEE NEM NFLX\
+NI NKE NOC NOW NRG NSC NTAP NTRS NUE NVDA NVR NWS NWSA NXPI O ODFL OKE\
+OMC ON ORCL  ORLY OTIS OXY PANW  PARA PAYC PAYX PCAR PCG  PEAK PEG PEP\
+PFE PFG PG PGR PH PHM PKG PLD PM PNC PNR PNW PODD POOL PPG PPL PRU PSA\
+PSX PTC PWR PXD PYPL QCOM QRVO RCL  REG REGN RF RHI RJF RL RMD ROK ROL\
+ROP ROST RSG RTX RVTY SBAC SBUX SCHW  SHW SJM SLB SNA SNPS SO SPG SPGI\
+SRE STE STLD STT  STX STZ SWK SWKS SYF SYK SYY T  TAP TDG TDY TECH TEL\
+TER TFC TFX TGT  TJX TMO TMUS TPR TRGP TRMB TROW TRV  TSCO TSLA TSN TT\
+TTWO TXN TXT TYL UAL UBER UDR UHS  ULTA UNH UNP UPS URI USB V VFC VICI\
+VLO VLTO VMC VRSK  VRSN VRTX VTR VTRS VZ WAB WAT WBA  WBD WDC WEC WELL\
+WFC WHR WM  WMB WMT WRB WRK WST  WTW WY WYNN XEL XOM XRAY  XYL YUM ZBH\
+ZBRA ZION ZTS
+
+for NAME in $SP_500
+	touch "$NAME.csv"
+end
+
+