From 436d6919ab8cf879e261a9c5bac347d19161ecd3 Mon Sep 17 00:00:00 2001 From: Onur Cinar Date: Sun, 31 Dec 2023 13:17:50 -0800 Subject: [PATCH] SP500 names and sync delay added. --- asset/README.md | 61 +++++++++++++++++++++++++++++++------- asset/sync.go | 34 ++++++++++++++++++--- asset/sync_test.go | 24 ++++++++++++--- cmd/indicator-sync/main.go | 10 +++++-- sp500.fish | 40 +++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 20 deletions(-) create mode 100755 sp500.fish diff --git a/asset/README.md b/asset/README.md index bcd5ad1..d6f39f1 100644 --- a/asset/README.md +++ b/asset/README.md @@ -24,13 +24,13 @@ The information provided on this project is strictly for informational purposes ## Index +- [Constants](<#constants>) - [func SnapshotsAsClosings\(snapshots \<\-chan \*Snapshot\) \<\-chan float64](<#SnapshotsAsClosings>) - [func SnapshotsAsDates\(snapshots \<\-chan \*Snapshot\) \<\-chan time.Time](<#SnapshotsAsDates>) - [func SnapshotsAsHighs\(snapshots \<\-chan \*Snapshot\) \<\-chan float64](<#SnapshotsAsHighs>) - [func SnapshotsAsLows\(snapshots \<\-chan \*Snapshot\) \<\-chan float64](<#SnapshotsAsLows>) - [func SnapshotsAsOpenings\(snapshots \<\-chan \*Snapshot\) \<\-chan float64](<#SnapshotsAsOpenings>) - [func SnapshotsAsVolumes\(snapshots \<\-chan \*Snapshot\) \<\-chan int64](<#SnapshotsAsVolumes>) -- [func Sync\(source, target Repository, defaultStartDate time.Time, workers int\) error](<#Sync>) - [type FileSystemRepository](<#FileSystemRepository>) - [func NewFileSystemRepository\(base string\) \*FileSystemRepository](<#NewFileSystemRepository>) - [func \(r \*FileSystemRepository\) Append\(name string, snapshots \<\-chan \*Snapshot\) error](<#FileSystemRepository.Append>) @@ -47,6 +47,9 @@ The information provided on this project is strictly for informational purposes - [func \(r \*InMemoryRepository\) LastDate\(name string\) \(time.Time, error\)](<#InMemoryRepository.LastDate>) - [type Repository](<#Repository>) - [type Snapshot](<#Snapshot>) +- [type Sync](<#Sync>) + - [func NewSync\(\) \*Sync](<#NewSync>) + - [func \(s \*Sync\) Run\(source, target Repository, defaultStartDate time.Time\) error](<#Sync.Run>) - [type TiingoEndOfDay](<#TiingoEndOfDay>) - [func \(e \*TiingoEndOfDay\) ToSnapshot\(\) \*Snapshot](<#TiingoEndOfDay.ToSnapshot>) - [type TiingoMeta](<#TiingoMeta>) @@ -59,6 +62,20 @@ The information provided on this project is strictly for informational purposes - [func \(r \*TiingoRepository\) LastDate\(name string\) \(time.Time, error\)](<#TiingoRepository.LastDate>) +## Constants + + + +```go +const ( + // DefaultSyncWorkers is the default number of workers to use to synchronize. + DefaultSyncWorkers = 1 + + // DefaultSyncDelay is the default delay in seconds between each get request. + DefaultSyncDelay = 5 +) +``` + ## func [SnapshotsAsClosings]() @@ -113,15 +130,6 @@ func SnapshotsAsVolumes(snapshots <-chan *Snapshot) <-chan int64 SnapshotsAsVolumes extracts the volume field from each snapshot in the provided channel and returns a new channel containing only those volume values.The original snapshots channel can no longer be directly used afterwards. - -## func [Sync]() - -```go -func Sync(source, target Repository, defaultStartDate time.Time, workers int) error -``` - -Sync synchronizes assets between the source and target repositories using multi\-worker concurrency. - ## type [FileSystemRepository]() @@ -314,6 +322,39 @@ type Snapshot struct { } ``` + +## type [Sync]() + +Sync represents the configuration parameters for synchronizing assets between repositories. + +```go +type Sync struct { + // Number of workers to use. + Workers int + + // Delay between repository get requests to minimize the load to the remote server. + Delay int +} +``` + + +### func [NewSync]() + +```go +func NewSync() *Sync +``` + +NewSync function initializes a new sync instance with the default parameters. + + +### func \(\*Sync\) [Run]() + +```go +func (s *Sync) Run(source, target Repository, defaultStartDate time.Time) error +``` + +Run synchronizes assets between the source and target repositories using multi\-worker concurrency. + ## type [TiingoEndOfDay]() diff --git a/asset/sync.go b/asset/sync.go index 5ab2deb..0e4244c 100644 --- a/asset/sync.go +++ b/asset/sync.go @@ -13,9 +13,33 @@ import ( "github.com/cinar/indicator/helper" ) -// Sync synchronizes assets between the source and target repositories using -// multi-worker concurrency. -func Sync(source, target Repository, defaultStartDate time.Time, workers int) error { +const ( + // DefaultSyncWorkers is the default number of workers to use to synchronize. + DefaultSyncWorkers = 1 + + // DefaultSyncDelay is the default delay in seconds between each get request. + DefaultSyncDelay = 5 +) + +// Sync represents the configuration parameters for synchronizing assets between repositories. +type Sync struct { + // Number of workers to use. + Workers int + + // Delay between repository get requests to minimize the load to the remote server. + Delay int +} + +// NewSync function initializes a new sync instance with the default parameters. +func NewSync() *Sync { + return &Sync{ + Workers: DefaultSyncWorkers, + Delay: DefaultSyncDelay, + } +} + +// Run synchronizes assets between the source and target repositories using multi-worker concurrency. +func (s *Sync) Run(source, target Repository, defaultStartDate time.Time) error { assets, err := target.Assets() if err != nil { return err @@ -27,7 +51,7 @@ func Sync(source, target Repository, defaultStartDate time.Time, workers int) er hasErrors := false wg := &sync.WaitGroup{} - for i := 0; i < workers; i++ { + for i := 0; i < s.Workers; i++ { wg.Add(1) go func() { @@ -54,6 +78,8 @@ func Sync(source, target Repository, defaultStartDate time.Time, workers int) er hasErrors = true continue } + + time.Sleep(time.Duration(s.Delay) * time.Second) } }() } diff --git a/asset/sync_test.go b/asset/sync_test.go index 3417ac8..4e64952 100644 --- a/asset/sync_test.go +++ b/asset/sync_test.go @@ -63,7 +63,11 @@ func TestSync(t *testing.T) { t.Fatal(err) } - err = asset.Sync(source, target, snapshots[0].Date, 1) + sync := asset.NewSync() + sync.Workers = 1 + sync.Delay = 0 + + err = sync.Run(source, target, snapshots[0].Date) if err != nil { t.Fatal(err) } @@ -94,7 +98,11 @@ func TestSyncMissingOnSource(t *testing.T) { defaultStartDate := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) - err = asset.Sync(source, target, defaultStartDate, 1) + sync := asset.NewSync() + sync.Workers = 1 + sync.Delay = 0 + + err = sync.Run(source, target, defaultStartDate) if err == nil { t.Fatal("expected error") } @@ -110,7 +118,11 @@ func TestSyncFailingTargetAssets(t *testing.T) { defaultStartDate := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) - err := asset.Sync(source, target, defaultStartDate, 1) + sync := asset.NewSync() + sync.Workers = 1 + sync.Delay = 0 + + err := sync.Run(source, target, defaultStartDate) if err == nil { t.Fatal("expected error") } @@ -139,7 +151,11 @@ func TestSyncFailingTargetAppend(t *testing.T) { defaultStartDate := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) - err := asset.Sync(source, target, defaultStartDate, 1) + sync := asset.NewSync() + sync.Workers = 1 + sync.Delay = 0 + + err := sync.Run(source, target, defaultStartDate) if err == nil { t.Fatal("expected error") } diff --git a/cmd/indicator-sync/main.go b/cmd/indicator-sync/main.go index 6270eeb..bf744ae 100644 --- a/cmd/indicator-sync/main.go +++ b/cmd/indicator-sync/main.go @@ -20,6 +20,7 @@ func main() { var targetBase string var minusDays int var workers int + var delay int fmt.Fprintln(os.Stderr, "Indicator Sync") fmt.Fprintln(os.Stderr, "Copyright (c) 2021-2023 Onur Cinar.") @@ -30,7 +31,8 @@ func main() { flag.StringVar(&tiingoKey, "key", "", "tiingo service api key") flag.StringVar(&targetBase, "target", ".", "target repository base directory") flag.IntVar(&minusDays, "days", 0, "lookback period in days for the new assets") - flag.IntVar(&workers, "workers", 1, "number of concurrent workers") + flag.IntVar(&workers, "workers", asset.DefaultSyncWorkers, "number of concurrent workers") + flag.IntVar(&delay, "delay", asset.DefaultSyncDelay, "delay between each get") flag.Parse() if tiingoKey == "" { @@ -42,7 +44,11 @@ func main() { source := asset.NewTiingoRepository(tiingoKey) target := asset.NewFileSystemRepository(targetBase) - err := asset.Sync(source, target, defaultStartDate, workers) + sync := asset.NewSync() + sync.Workers = workers + sync.Delay = delay + + err := sync.Run(source, target, defaultStartDate) if err != nil { log.Fatal(err) } diff --git a/sp500.fish b/sp500.fish new file mode 100755 index 0000000..da016b6 --- /dev/null +++ b/sp500.fish @@ -0,0 +1,40 @@ +#!/usr/bin/env fish + +set SP_500 \ +A AAL AAPL ABBV ABNB ABT ACGL ACN ADBE ADI ADM ADP ADSK AEE AEP AES\ +AFL AIG AIZ AJG AKAM ALB ALGN ALL ALLE AMAT AMCR AMD AME AMGN AMP AMT\ +AMZN ANET ANSS AON AOS APA APD APH APTV ARE ATO AVB AVGO AVY AWK AXON\ +AXP AZO BA BAC BALL BAX BBWI BBY BDX BEN BF.B BG BIIB BIO BK BKNG BKR\ +BLDR BLK BMY BR BRK.B BRO BSX BWA BX BXP C CAG CAH CARR CAT CB CBOE\ +CBRE CCI CCL CDAY CDNS CDW CE CEG CF CFG CHD CHRW CHTR CI CINF CL CLX\ +CMA CMCSA CME CMG CMI CMS CNC CNP COF COO COP COR COST CPB CPRT CPT\ +CRL CRM CSCO CSGP CSX CTAS CTLT CTRA CTSH CTVA CVS CVX CZR D DAL DD DE\ +DFS DG DGX DHI DHR DIS DLR DLTR DOV DOW DPZ DRI DTE DUK DVA DVN DXCM\ +EA EBAY ECL ED EFX EG EIX EL ELV EMN EMR ENPH EOG EPAM EQIX EQR EQT ES\ +ESS ETN ETR ETSY EVRG EW EXC EXPD EXPE EXR F FANG FAST FCX FDS FDX FE\ +FFIV FI FICO FIS FITB FLT FMC FOX FOXA FRT FSLR FTNT FTV GD GE GEHC\ +GEN GILD GIS GL GLW GM GNRC GOOG GOOGL GPC GPN GRMN GS GWW HAL HAS\ +HBAN HCA HD HES HIG HII HLT HOLX HON HPE HPQ HRL HSIC HST HSY HUBB HUM\ +HWM IBM ICE IDXX IEX IFF ILMN INCY INTC INTU INVH IP IPG IQV IR IRM\ +ISRG IT ITW IVZ J JBHT JBL JCI JKHY JNJ JNPR JPM K KDP KEY KEYS KHC\ +KIM KLAC KMB KMI KMX KO KR KVUE L LDOS LEN LH LHX LIN LKQ LLY LMT LNT\ +LOW LRCX LULU LUV LVS LW LYB LYV MA MAA MAR MAS MCD MCHP MCK MCO MDLZ\ +MDT MET META MGM MHK MKC MKTX MLM MMC MMM MNST MO MOH MOS MPC MPWR MRK\ +MRNA MRO MS MSCI MSFT MSI MTB MTCH MTD MU NCLH NDAQ NDSN NEE NEM NFLX\ +NI NKE NOC NOW NRG NSC NTAP NTRS NUE NVDA NVR NWS NWSA NXPI O ODFL OKE\ +OMC ON ORCL ORLY OTIS OXY PANW PARA PAYC PAYX PCAR PCG PEAK PEG PEP\ +PFE PFG PG PGR PH PHM PKG PLD PM PNC PNR PNW PODD POOL PPG PPL PRU PSA\ +PSX PTC PWR PXD PYPL QCOM QRVO RCL REG REGN RF RHI RJF RL RMD ROK ROL\ +ROP ROST RSG RTX RVTY SBAC SBUX SCHW SHW SJM SLB SNA SNPS SO SPG SPGI\ +SRE STE STLD STT STX STZ SWK SWKS SYF SYK SYY T TAP TDG TDY TECH TEL\ +TER TFC TFX TGT TJX TMO TMUS TPR TRGP TRMB TROW TRV TSCO TSLA TSN TT\ +TTWO TXN TXT TYL UAL UBER UDR UHS ULTA UNH UNP UPS URI USB V VFC VICI\ +VLO VLTO VMC VRSK VRSN VRTX VTR VTRS VZ WAB WAT WBA WBD WDC WEC WELL\ +WFC WHR WM WMB WMT WRB WRK WST WTW WY WYNN XEL XOM XRAY XYL YUM ZBH\ +ZBRA ZION ZTS + +for NAME in $SP_500 + touch "$NAME.csv" +end + +