Various changes to run scraper and init in headlessMode

src/apps/chifra/internal/init/handle_init.go

@@ -6,6 +6,7 @@ package initPkg
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
@@ -27,7 +28,12 @@ func (opts *InitOptions) HandleInit(rCtx *output.RenderCtx) error {
 	// TODO: BOGUS - IF THE SCRAPER IS RUNNING, THIS WILL CAUSE PROBLEMS
 	// Make sure that the temporary scraper folders are empty, so that, when the
 	// scraper starts, it starts on the correct block.
-	_ = file.CleanFolder(chain, config.PathToIndex(chain), []string{"ripe", "unripe", "maps", "staging"})
+	cleanList := []string{"ripe", "unripe", "maps", "staging"}
+	isHeadless := os.Getenv("TB_NODE_HEADLESS") == "true"
+	if isHeadless {
+		cleanList = []string{"ripe", "unripe"}
+	}
+	_ = file.CleanFolder(chain, config.PathToIndex(chain), cleanList)
 
 	existing, err := manifest.LoadManifest(chain, opts.PublisherAddr, manifest.LocalCache)
 	if err != nil {

src/apps/chifra/internal/init/handle_init_download.go

@@ -105,7 +105,7 @@ func (opts *InitOptions) downloadAndReportProgress(chunks []types.ChunkRecord, c
 			if sleep > .25 {
 				sleepStr = fmt.Sprintf(" [sleep: %0.2fms]", sleep*1000)
 			}
-			msg := fmt.Sprintf("Finished download of %s%s%s %s%s%s (% 4d of %4d %0.1f%%%s)", col, event.Message, colors.Off, col, rng, colors.Off, nProcessed, nTotal, pct, sleepStr)
+			msg := fmt.Sprintf("Finished download of %s%s%s %s%s%s (% 4d of %4d %0.1f%%%s) chain: %s", col, event.Message, colors.Off, col, rng, colors.Off, nProcessed, nTotal, pct, sleepStr, opts.Globals.Chain)
 			logger.Info(msg, spaces)
 			if successCount%10 == 0 {
 				sleep = base.Max(.0125, sleep/1.2)

src/apps/chifra/internal/scrape/handle_show.go

@@ -54,6 +54,8 @@ func (opts *ScrapeOptions) HandleScrape(rCtx *output.RenderCtx) error {
 		provider = "--rpc-provider--"
 	}
 
+	isHeadless := os.Getenv("TB_NODE_HEADLESS") == "true"
+
 	msg1 := fmt.Sprintf("Scraping %s", chain)
 	msg2 := fmt.Sprintf("  Rpc %s", provider)
 	msg3 := fmt.Sprintf("  Path %s", path)
@@ -67,10 +69,12 @@ func (opts *ScrapeOptions) HandleScrape(rCtx *output.RenderCtx) error {
 		}
 		return output.StreamMany(rCtx, fetchData, opts.Globals.OutputOpts())
 	} else {
-		logger.Info(msg1)
-		logger.Info(msg2)
-		logger.Info(msg3)
-		logger.Info(msg4)
+		if !isHeadless {
+			logger.Info(msg1)
+			logger.Info(msg2)
+			logger.Info(msg3)
+			logger.Info(msg4)
+		}
 	}
 
 	// Handle Ctr-C, docker stop and docker compose down (provided they
@@ -143,6 +147,7 @@ func (opts *ScrapeOptions) HandleScrape(rCtx *output.RenderCtx) error {
 			processedMap: make(map[base.Blknum]bool, opts.BlockCnt),
 			meta:         bm.meta,
 			nChannels:    int(opts.Settings.ChannelCount),
+			isHeadless:   isHeadless,
 		}
 
 		// Order dependant, be careful!
@@ -194,7 +199,9 @@ func (opts *ScrapeOptions) HandleScrape(rCtx *output.RenderCtx) error {
 		}
 
 		if bm.nRipe == 0 {
-			logger.Info(colors.Green+"no ripe files to consolidate", spaces, colors.Off)
+			if !bm.isHeadless {
+				logger.Info(colors.Green+"no ripe files to consolidate on chain", chain, strings.Repeat(" ", 40), colors.Off)
+			}
 			goto PAUSE
 
 		} else {
@@ -215,7 +222,9 @@ func (opts *ScrapeOptions) HandleScrape(rCtx *output.RenderCtx) error {
 		runCount++
 		if opts.RunCount != 0 && runCount >= opts.RunCount {
 			// No reason to clean up here. Next round will do so and user can use these files in the meantime.
-			logger.Info("run count reached")
+			if !isHeadless {
+				logger.Info("run count reached")
+			}
 			break
 		}
 

src/apps/chifra/internal/scrape/save_timestamps.go

@@ -81,16 +81,20 @@ func (bm *BlazeManager) WriteTimestamps(ctx context.Context, blocks []base.Blknu
 		}
 
 		ts := bm.timestamps[block]
-		logProgressTs("Updating timestamps ", block, blocks[len(blocks)-1])
+		if !bm.isHeadless {
+			logProgressTs("Updating timestamps ", block, blocks[len(blocks)-1])
+		}
 		if err := binary.Write(fp, binary.LittleEndian, &ts); err != nil {
 			return err
 		}
 
 	}
 
-	logger.Progress(true, fmt.Sprintf("Finished writing timestamps to block %-04d"+spaces,
-		blocks[len(blocks)-1],
-	))
+	if !bm.isHeadless {
+		logger.Progress(true, fmt.Sprintf("Finished writing timestamps to block %-04d"+spaces,
+			blocks[len(blocks)-1],
+		))
+	}
 
 	return nil
 }

src/apps/chifra/internal/scrape/scrape_blaze.go

@@ -186,9 +186,9 @@ func (bm *BlazeManager) WriteAppearances(bn base.Blknum, addrMap uniq.AddressBoo
 		sort.Strings(appearanceArray)
 
 		blockNumStr := utils.PadNum(int(bn), 9)
-		fileName := filepath.Join(ripePath, blockNumStr + ".txt")
+		fileName := filepath.Join(ripePath, blockNumStr+".txt")
 		if bn > bm.ripeBlock {
-			fileName = filepath.Join(unripePath, blockNumStr + ".txt")
+			fileName = filepath.Join(unripePath, blockNumStr+".txt")
 		}
 
 		toWrite := []byte(strings.Join(appearanceArray[:], "\n") + "\n")
@@ -243,12 +243,17 @@ func (bm *BlazeManager) syncedReporting(bn base.Blknum, force bool) {
 		if bm.ripeBlock > bn {
 			dist = (bm.ripeBlock - bn)
 		}
-		msg := fmt.Sprintf("Scraping %-04d of %-04d at block %d of %d (%d blocks from head)",
+		if bm.isHeadless && dist < 100 {
+			return
+		}
+		msg := fmt.Sprintf("Scraping %-04d of %-04d at block %d of %d (%d blocks from %s head)",
 			bm.nProcessed(),
 			bm.BlockCount(),
 			bn,
 			bm.ripeBlock,
-			dist)
+			dist,
+			bm.chain,
+		)
 		logger.Progress(true, msg)
 	}
 }

src/apps/chifra/internal/scrape/scrape_manager.go

@@ -26,6 +26,7 @@ type BlazeManager struct {
 	nTimestamps  int
 	nChannels    int
 	errors       []scrapeError
+	isHeadless   bool
 }
 
 type scrapeError struct {

src/apps/chifra/internal/scrape/scrape_manager_utils.go

@@ -17,18 +17,28 @@ func (bm *BlazeManager) report(nBlocks, perChunk, nChunks, nAppsNow, nAppsFound,
 	nNeeded := perChunk - base.Min(perChunk, nAppsNow)
 	appsPerAddr := float64(nAppsFound) / float64(nAddrsFound)
 	pctFull := float64(nAppsNow) / float64(perChunk)
+	pctStr := fmt.Sprintf("%0.1f%%", pctFull*100)
+	if len(pctStr) < 5 {
+		pctStr = " " + pctStr
+	}
+	chunksStr := ""
+	if nChunks > 0 {
+		chunksStr = fmt.Sprintf(" created {%d} chunk(s)", nChunks)
+	} else {
+		chunksStr = fmt.Sprintf(" %s", bm.chain)
+	}
 
-	report := `%s #{%d}, found {%6d} apps and {%5d} addrs ({%0.1f apps/addr}). created {%d} chunks, staged {%5d} apps (@%0.1f%%}), need {%5d}.`
+	report := `%7.7s @#% 9d}: {% 8d}/{% 8d} ({%0.1f apps/addr}) stage {% 8d} need {% 8d} (@%5.5s})%s`
 	msg := fmt.Sprintf(report,
 		bm.chain,
 		bm.EndBlock()-1,
 		nAppsFound,
 		nAddrsFound,
 		appsPerAddr,
-		nChunks,
 		nAppsNow,
-		pctFull*100,
 		nNeeded,
+		pctStr,
+		chunksStr,
 	)
 	logger.Info(colors.Colored(msg))
 }