From b453f81740ea64c21ca8602ad232b95ae49cda84 Mon Sep 17 00:00:00 2001 From: Hayden McAlister Date: Sun, 18 Jun 2023 04:07:40 +1200 Subject: [PATCH] Add more concurrent writers to data handlers --- hopfieldnetwork/datacollector/DefaultHandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hopfieldnetwork/datacollector/DefaultHandler.go b/hopfieldnetwork/datacollector/DefaultHandler.go index c70be90..116b556 100644 --- a/hopfieldnetwork/datacollector/DefaultHandler.go +++ b/hopfieldnetwork/datacollector/DefaultHandler.go @@ -63,7 +63,7 @@ func (handler *dataHandler) writeStop() error { func newParquetWriter[T interface{}](dataFilePath string, dataStruct T) (source.ParquetFile, *writer.ParquetWriter) { os.Remove(dataFilePath) dataFileWriter, _ := local.NewLocalFileWriter(dataFilePath) - parquetDataWriter, _ := writer.NewParquetWriter(dataFileWriter, dataStruct, 4) + parquetDataWriter, _ := writer.NewParquetWriter(dataFileWriter, dataStruct, 8) parquetDataWriter.RowGroupSize = 128 * 1024 * 1024 //128MB parquetDataWriter.PageSize = 8 * 1024 //8K parquetDataWriter.CompressionType = parquet.CompressionCodec_SNAPPY