diff --git a/decode_test.go b/decode_test.go index bd63c2e..0628c64 100644 --- a/decode_test.go +++ b/decode_test.go @@ -85,6 +85,21 @@ f,1,baz,,*string,*string`) } } +func Test_readTo_FromTo(t *testing.T) { + b := bytes.NewBufferString(`foo,BAR,Baz,Blah,SPtr,Omit +f,1,baz,,*string,*string +e,3,b,,,`) + var samples []Sample + xsvRead := NewXsvRead[Sample]() + xsvRead.To = 1 + if err := xsvRead.SetReader(csv.NewReader(b)).ReadTo(&samples); err != nil { + t.Fatal(err) + } + if len(samples) != 1 { + t.Fatalf("expected 1 sample instances, got %d", len(samples)) + } +} + func Test_readToNormalized(t *testing.T) { blah := 0 @@ -393,6 +408,62 @@ aa,bb,11,cc,dd,ee`) } } +func Test_readEach_FromTo(t *testing.T) { + b := bytes.NewBufferString(` +first,foo,BAR,Baz,last,abc +aa,bb,11,cc,dd,ee +ff,gg,22,hh,ii,jj +kk,ll,33,mm,nn,oo +`) + + c := make(chan SkipFieldSample) + var samples []SkipFieldSample + go func() { + xsvRead := NewXsvRead[SkipFieldSample]() + xsvRead.From = 1 + xsvRead.To = 2 + if err := xsvRead.SetReader(csv.NewReader(b)).ReadEach(c); err != nil { + t.Fatal(err) + } + }() + for v := range c { + samples = append(samples, v) + } + if len(samples) != 2 { + t.Fatalf("expected 2 sample instances, got %d", len(samples)) + } + expected := SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "aa", + Sample: Sample{ + Foo: "bb", + Bar: 11, + Baz: "cc", + }, + Quux: "dd", + }, + Corge: "ee", + } + if expected != samples[0] { + t.Fatalf("expected first sample %v, got %v", expected, samples[0]) + } + expected = SkipFieldSample{ + EmbedSample: EmbedSample{ + Qux: "ff", + Sample: Sample{ + Foo: "gg", + Bar: 22, + Baz: "hh", + }, + Quux: "ii", + }, + Corge: "jj", + } + if expected != samples[1] { + t.Fatalf("expected first sample %v, got %v", expected, samples[1]) + } +} + func Test_readEachWithoutHeaders(t *testing.T) { blah := 0 sptr := "" @@ -893,6 +964,43 @@ func TestCSVToMaps_OnRecord(t *testing.T) { } } +func TestCSVToMaps_FromTo(t *testing.T) { + b := bytes.NewBufferString(`foo,BAR,Baz +4,Jose,42 +2,Daniel,21 +5,Vincent,84`) + xsvRead := NewXsvRead[interface{}]() + xsvRead.From = 2 + xsvRead.To = 3 + m, err := xsvRead.SetReader(csv.NewReader(b)).ToMap() + if err != nil { + t.Fatal(err) + } + if len(m) != 2 { + t.Fatal("Expected 2 len, but", len(m)) + } + firstRecord := m[0] + if firstRecord["foo"] != "2" { + t.Fatal("Expected 2 got", firstRecord["foo"]) + } + if firstRecord["BAR"] != "Daniel" { + t.Fatal("Expected Daniel got", firstRecord["BAR"]) + } + if firstRecord["Baz"] != "21" { + t.Fatal("Expected 21 got", firstRecord["Baz"]) + } + secondRecord := m[1] + if secondRecord["foo"] != "5" { + t.Fatal("Expected 5 got", secondRecord["foo"]) + } + if secondRecord["BAR"] != "Vincent" { + t.Fatal("Expected Vincent got", secondRecord["BAR"]) + } + if secondRecord["Baz"] != "84" { + t.Fatal("Expected 84 got", secondRecord["Baz"]) + } +} + func TestUnmarshalToDecoder(t *testing.T) { blah := 0 sptr := "*string" diff --git a/examples/main.go b/examples/main.go index 3da1fa3..c54f811 100644 --- a/examples/main.go +++ b/examples/main.go @@ -2,10 +2,10 @@ package main import ( "fmt" + "github.com/shigetaichi/xsv" "os" "sync" "time" - "xsv" ) type NotUsed struct { @@ -120,4 +120,21 @@ func main() { if err != nil { panic(err) } + + // Read to struct slice + clientsFile2, err := os.Open("clients.csv") + if err != nil { + panic(err) + } + + var readClients []*Client + xsvRead := xsv.NewXsvRead[*Client]() + xsvRead.From = 2 + xsvRead.To = 4 + if err := xsvRead.SetFileReader(clientsFile2).ReadTo(&readClients); err != nil { + panic(err) + } + for _, readClient := range readClients { + fmt.Println(*readClient) + } } diff --git a/xsv_read.go b/xsv_read.go index b47be7b..b6f442b 100644 --- a/xsv_read.go +++ b/xsv_read.go @@ -3,17 +3,22 @@ package xsv import ( "bytes" "encoding/csv" + "errors" + "fmt" "os" + "strconv" "strings" ) // XsvRead manages configuration values related to the csv read process. type XsvRead[T any] struct { - TagName string //key in the struct field's tag to scan - TagSeparator string //separator string for multiple csv tags in struct fields - FailIfUnmatchedStructTags bool // indicates whether it is considered an error when there is an unmatched struct tag. - FailIfDoubleHeaderNames bool // indicates whether it is considered an error when a header name is repeated in the csv header. - ShouldAlignDuplicateHeadersWithStructFieldOrder bool // indicates whether we should align duplicate CSV headers per their alignment in the struct definition. + TagName string //key in the struct field's tag to scan + TagSeparator string //separator string for multiple csv tags in struct fields + FailIfUnmatchedStructTags bool // indicates whether it is considered an error when there is an unmatched struct tag. + FailIfDoubleHeaderNames bool // indicates whether it is considered an error when a header name is repeated in the csv header. + ShouldAlignDuplicateHeadersWithStructFieldOrder bool // indicates whether we should align duplicate CSV headers per their alignment in the struct definition. + From int // + To int // OnRecord func(T) T // callback function to be called on each record NameNormalizer Normalizer ErrorHandler ErrorHandler @@ -27,12 +32,44 @@ func NewXsvRead[T any]() *XsvRead[T] { FailIfUnmatchedStructTags: false, FailIfDoubleHeaderNames: false, ShouldAlignDuplicateHeadersWithStructFieldOrder: false, + From: 1, + To: -1, OnRecord: nil, NameNormalizer: func(s string) string { return s }, ErrorHandler: nil, } } +func (x *XsvRead[T]) checkFrom() (err error) { + if x.From >= 0 { + return nil + } + return errors.New(fmt.Sprintf("%s cannot be set to a negative value.", strconv.Quote("From"))) +} + +func (x *XsvRead[T]) checkTo() (err error) { + if x.To >= -1 { + return nil + } + return errors.New(fmt.Sprintf("%s cannot be set to a negative value other than -1.", strconv.Quote("To"))) +} + +func (x *XsvRead[T]) checkFromTo() (err error) { + if err := x.checkFrom(); err != nil { + return err + } + if err := x.checkTo(); err != nil { + return err + } + if x.To == -1 { + return nil + } + if x.From <= x.To { + return nil + } + return errors.New(fmt.Sprintf("%s cannot be set before %s", strconv.Quote("To"), strconv.Quote("From"))) +} + func (x *XsvRead[T]) SetReader(r *csv.Reader) (xr *XsvReader[T]) { xr = NewXsvReader(*x) xr.reader = r diff --git a/xsv_read_test.go b/xsv_read_test.go new file mode 100644 index 0000000..50e2878 --- /dev/null +++ b/xsv_read_test.go @@ -0,0 +1,54 @@ +package xsv + +import ( + "testing" +) + +func TestXsvRead_checkFromTo(t *testing.T) { + type Arg struct { + From int + To int + } + type testCase[T any] struct { + name string + arg Arg + wantErr bool + } + tests := []testCase[Arg]{ + { + name: "default", + arg: Arg{From: 1, To: -1}, + wantErr: false, + }, + { + name: "only 1 line", + arg: Arg{From: 1, To: 1}, + wantErr: false, + }, + { + name: "great and small reversals", + arg: Arg{From: 3, To: 2}, + wantErr: true, + }, + { + name: "unsigned int on From", + arg: Arg{From: -1, To: 2}, + wantErr: true, + }, + { + name: "unsigned int on To", + arg: Arg{From: 1, To: -2}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + xsvRead := NewXsvRead[any]() + xsvRead.From = tt.arg.From + xsvRead.To = tt.arg.To + if err := xsvRead.checkFromTo(); (err != nil) != tt.wantErr { + t.Errorf("checkFromTo() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/xsv_reader.go b/xsv_reader.go index b811eac..c9744ef 100644 --- a/xsv_reader.go +++ b/xsv_reader.go @@ -35,7 +35,19 @@ func (r *XsvReader[T]) ReadTo(out *[]T) error { if len(csvRows) == 0 { return ErrEmptyCSVFile } - if err := ensureOutCapacity(&outValue, len(csvRows)); err != nil { // Ensure the container is big enough to hold the CSV content + + if err := r.checkFromTo(); err != nil { + return err + } + to := r.To + if to >= 0 { + to++ + } else { + to = len(csvRows) + } + body := csvRows[r.From:to] + capacity := len(body) + 1 // Plus one for the header row. + if err := ensureOutCapacity(&outValue, capacity); err != nil { // Ensure the container is big enough to hold the CSV content return err } fieldInfos := getFieldInfos(outInnerType, []int{}, []string{}, r.TagName, r.TagSeparator, r.NameNormalizer) // Get the inner struct info to get CSV annotations @@ -48,7 +60,6 @@ func (r *XsvReader[T]) ReadTo(out *[]T) error { for i, h := range csvRows[0] { // apply normalizer func to headers headers[i] = r.NameNormalizer(h) } - body := csvRows[1:] csvHeadersLabels := make(map[int]*fieldInfo, len(outInnerStructInfo.Fields)) // Used to store the correspondance header <-> position in CSV @@ -173,7 +184,11 @@ func (r *XsvReader[T]) ReadEach(c chan T) error { return err } } - i := 0 + + if err := r.checkFromTo(); err != nil { + return err + } + i := 1 for { line, err := r.reader.Read() if err == io.EOF { @@ -181,22 +196,25 @@ func (r *XsvReader[T]) ReadEach(c chan T) error { } else if err != nil { return err } - outInner := createNewOutInner(outInnerWasPointer, outInnerType) - for j, csvColumnContent := range line { - if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name - if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct - return &csv.ParseError{ - Line: i + 2, //add 2 to account for the header & 0-indexing of arrays - Column: j + 1, - Err: err, + + if r.From <= i && i <= r.To { + outInner := createNewOutInner(outInnerWasPointer, outInnerType) + for j, csvColumnContent := range line { + if fieldInfo, ok := csvHeadersLabels[j]; ok { // Position found accordingly to header name + if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct + return &csv.ParseError{ + Line: i + 2, //add 2 to account for the header & 0-indexing of arrays + Column: j + 1, + Err: err, + } } } } + if r.OnRecord != nil { + outInner = reflect.ValueOf(r.OnRecord(outInner.Interface().(T))) + } + outValue.Send(outInner) } - if r.OnRecord != nil { - outInner = reflect.ValueOf(r.OnRecord(outInner.Interface().(T))) - } - outValue.Send(outInner) i++ } return nil @@ -216,7 +234,19 @@ func (r *XsvReader[T]) ReadToWithoutHeaders(out *[]T) error { if len(csvRows) == 0 { return ErrEmptyCSVFile } - if err := ensureOutCapacity(&outValue, len(csvRows)+1); err != nil { // Ensure the container is big enough to hold the CSV content + + if err := r.checkFromTo(); err != nil { + return err + } + to := r.To + if to >= 0 { + to++ + } else { + to = len(csvRows) + } + body := csvRows[r.From:to] + capacity := len(body) + 1 + if err := ensureOutCapacity(&outValue, capacity); err != nil { // Ensure the container is big enough to hold the CSV content return err } fieldInfos := getFieldInfos(outInnerType, []int{}, []string{}, r.TagName, r.TagSeparator, r.NameNormalizer) // Get the inner struct info to get CSV annotations @@ -260,7 +290,10 @@ func (r *XsvReader[T]) ReadEachWithoutHeaders(c chan T) error { return ErrNoStructTags } - i := 0 + if err := r.checkFromTo(); err != nil { + return err + } + i := 1 for { line, err := r.reader.Read() if err == io.EOF { @@ -268,21 +301,23 @@ func (r *XsvReader[T]) ReadEachWithoutHeaders(c chan T) error { } else if err != nil { return err } - outInner := createNewOutInner(outInnerWasPointer, outInnerType) - for j, csvColumnContent := range line { - fieldInfo := outInnerStructInfo.Fields[j] - if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct - return &csv.ParseError{ - Line: i + 2, //add 2 to account for the header & 0-indexing of arrays - Column: j + 1, - Err: err, + if r.From <= i && i <= r.To { + outInner := createNewOutInner(outInnerWasPointer, outInnerType) + for j, csvColumnContent := range line { + fieldInfo := outInnerStructInfo.Fields[j] + if err := setInnerField(&outInner, outInnerWasPointer, fieldInfo.IndexChain, csvColumnContent, fieldInfo.omitEmpty); err != nil { // Set field of struct + return &csv.ParseError{ + Line: i + 2, //add 2 to account for the header & 0-indexing of arrays + Column: j + 1, + Err: err, + } } } + if r.OnRecord != nil { + outInner = reflect.ValueOf(r.OnRecord(outInner.Interface().(T))) + } + outValue.Send(outInner) } - if r.OnRecord != nil { - outInner = reflect.ValueOf(r.OnRecord(outInner.Interface().(T))) - } - outValue.Send(outInner) i++ } return nil @@ -313,6 +348,11 @@ func (r *XsvReader[T]) ReadToCallback(f func(s T) error) error { func (r *XsvReader[T]) ToMap() ([]map[string]string, error) { var rows []map[string]string var header []string + var i = 0 + + if err := r.checkFromTo(); err != nil { + return nil, err + } for { record, err := r.reader.Read() if err == io.EOF { @@ -324,22 +364,30 @@ func (r *XsvReader[T]) ToMap() ([]map[string]string, error) { if header == nil { header = record } else { - dict := map[string]string{} - for i := range header { - dict[header[i]] = record[i] - } - if r.OnRecord != nil { - v := r.OnRecord(reflect.ValueOf(dict).Interface().(T)) - dict = reflect.ValueOf(v).Interface().(map[string]string) + if r.From <= i && i <= r.To { + dict := map[string]string{} + for i := range header { + dict[header[i]] = record[i] + } + if r.OnRecord != nil { + v := r.OnRecord(reflect.ValueOf(dict).Interface().(T)) + dict = reflect.ValueOf(v).Interface().(map[string]string) + } + rows = append(rows, dict) } - rows = append(rows, dict) } + i++ } return rows, nil } func (r *XsvReader[T]) ToChanMaps(c chan<- map[string]string) error { var header []string + var i = 0 + + if err := r.checkFromTo(); err != nil { + return err + } for { record, err := r.reader.Read() if err == io.EOF { @@ -351,16 +399,19 @@ func (r *XsvReader[T]) ToChanMaps(c chan<- map[string]string) error { if header == nil { header = record } else { - dict := map[string]string{} - for i := range header { - dict[header[i]] = record[i] - } - if r.OnRecord != nil { - v := r.OnRecord(reflect.ValueOf(dict).Interface().(T)) - dict = reflect.ValueOf(v).Interface().(map[string]string) + if r.From <= i && i <= r.To { + dict := map[string]string{} + for i := range header { + dict[header[i]] = record[i] + } + if r.OnRecord != nil { + v := r.OnRecord(reflect.ValueOf(dict).Interface().(T)) + dict = reflect.ValueOf(v).Interface().(map[string]string) + } + c <- dict } - c <- dict } + i++ } return nil }