Add mutate3 command using Expr language

csvtk/cmd/mutate3_test.go

@@ -0,0 +1,230 @@
+package cmd
+
+import (
+	"os"
+	"runtime"
+	"testing"
+)
+
+func TestMutate3(t *testing.T) {
+	cases := []struct {
+		expect   string
+		noHeader bool
+		opts     mutate3Opts
+		tabs     bool
+	}{
+		// Strings
+		{
+			opts: mutate3Opts{
+				ExprStr: ` $first_name + " " + $last_name `,
+				Files:   []string{"../../testdata/names.csv"},
+				Name:    "full_name",
+			},
+			expect: `id,first_name,last_name,username,full_name
+11,Rob,Pike,rob,Rob Pike
+2,Ken,Thompson,ken,Ken Thompson
+4,Robert,Griesemer,gri,Robert Griesemer
+1,Robert,Thompson,abc,Robert Thompson
+NA,Robert,Abel,123,Robert Abel
+`,
+		},
+
+		// Constants
+		{
+			tabs:     true,
+			noHeader: true,
+			opts: mutate3Opts{
+				ExprStr: ` "abc" `,
+				Files:   []string{"../../testdata/digitals.tsv"},
+			},
+			expect: `4	5	6	abc
+1	2	3	abc
+7	8	0	abc
+8	1,000	4	abc
+`,
+		},
+
+		// Math
+		{
+			tabs:     true,
+			noHeader: true,
+			opts: mutate3Opts{
+				ExprStr:      ` $1 + $3 `,
+				Files:        []string{"../../testdata/digitals.tsv"},
+				DecimalWidth: 0,
+			},
+			expect: `4	5	6	10
+1	2	3	4
+7	8	0	7
+8	1,000	4	12
+`,
+		},
+
+		// Bool
+		{
+			tabs:     true,
+			noHeader: true,
+			opts: mutate3Opts{
+				ExprStr: ` $1 > 5 `,
+				Files:   []string{"../../testdata/digitals.tsv"},
+			},
+			expect: `4	5	6	false
+1	2	3	false
+7	8	0	true
+8	1,000	4	true
+`,
+		},
+
+		// Ternary
+		{
+			tabs:     true,
+			noHeader: true,
+			opts: mutate3Opts{
+				ExprStr: `$1 > 5 ? "big" : "small"`,
+				Files:   []string{"../../testdata/digitals.tsv"},
+			},
+			expect: `4	5	6	small
+1	2	3	small
+7	8	0	big
+8	1,000	4	big
+`,
+		},
+
+		// Null coalescence
+		{
+			opts: mutate3Opts{
+				ExprStr: `$one ?? $two`,
+				Files:   []string{"../../testdata/null_coalescence.csv"},
+				Name:    "three",
+			},
+			expect: `one,two,three
+a1,a2,a1
+,b2,b2
+a2,,a2
+`,
+		},
+
+		// Position: --at 1
+		{
+			opts: mutate3Opts{
+				ExprStr:      `$a+$c`,
+				Files:        []string{"../../testdata/positions.csv"},
+				Name:         "x",
+				DecimalWidth: 0,
+				At:           1,
+			},
+			expect: `x,a,b,c
+4,1,2,3
+`,
+		},
+
+		// Position: --at 3
+		{
+			opts: mutate3Opts{
+				ExprStr:      `$a+$c`,
+				Files:        []string{"../../testdata/positions.csv"},
+				Name:         "x",
+				DecimalWidth: 0,
+				At:           3,
+			},
+			expect: `a,b,x,c
+1,2,4,3
+`,
+		},
+
+		// Position: --after a
+		{
+			opts: mutate3Opts{
+				ExprStr:      `$a+$c`,
+				Files:        []string{"../../testdata/positions.csv"},
+				Name:         "x",
+				DecimalWidth: 0,
+				After:        "a",
+			},
+			expect: `a,x,b,c
+1,4,2,3
+`,
+		},
+
+		// Position: --before c
+		{
+			opts: mutate3Opts{
+				ExprStr:      `$a+$c`,
+				Files:        []string{"../../testdata/positions.csv"},
+				Name:         "x",
+				DecimalWidth: 0,
+				Before:       "c",
+			},
+			expect: `a,b,x,c
+1,2,4,3
+`,
+		},
+
+		// Date math
+		{
+			opts: mutate3Opts{
+				ExprStr: `(date(${Out}) - date($In)).Hours() | int()`,
+				Files:   []string{"../../testdata/datesub.csv"},
+				Name:    "Hours",
+			},
+			expect: `ID,Name,In,Out,Hours
+1,Tom,2023-08-25 11:24:00,2023-08-27 08:33:02,45
+2,Sally,2023-08-25 11:28:00,2023-08-26 14:17:35,26
+3,Alf,2023-08-26 11:29:00,2023-08-29 20:43:00,81
+`,
+		},
+
+		// len
+		{
+			opts: mutate3Opts{
+				ExprStr: `len($SD)`,
+				Files:   []string{"../../testdata/mutate3len.csv"},
+				Name:    "Len",
+			},
+			expect: `SD,Len
+沈伟,6
+`,
+		},
+
+		// ulen
+		{
+			opts: mutate3Opts{
+				ExprStr: `ulen($SD)`,
+				Files:   []string{"../../testdata/mutate3len.csv"},
+				Name:    "Len",
+			},
+			expect: `SD,Len
+沈伟,4
+`,
+		},
+	}
+
+	for _, c := range cases {
+		f, err := os.CreateTemp("", "outfile")
+		if err != nil {
+			t.Fatalf("failed to open temp file: %s\n", err)
+		}
+		defer os.Remove(f.Name())
+
+		config := Config{
+			CommentChar:  '#',
+			Delimiter:    ',',
+			NoHeaderRow:  c.noHeader,
+			NumCPUs:      runtime.NumCPU(),
+			OutDelimiter: ',',
+			OutFile:      f.Name(),
+			Tabs:         c.tabs,
+		}
+
+		doMutate3(config, c.opts)
+
+		output, err := os.ReadFile(f.Name())
+		if err != nil {
+			t.Fatalf("failed to read temp file %q: %s\n", f.Name(), err)
+		}
+
+		if string(output) != c.expect {
+			t.Errorf("test failed:\noptions:\n\t%#v\nwant:\n\t%q\ngot:\n\t%q\n", c.opts, c.expect, output)
+		}
+	}
+}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/botond-sipos/thist v1.1.0
 	github.com/cheggaaa/pb/v3 v3.1.0
+	github.com/expr-lang/expr v1.16.3
 	github.com/fatih/color v1.13.0
 	github.com/mattn/go-colorable v0.1.13
 	github.com/mattn/go-runewidth v0.0.14

go.sum

@@ -51,6 +51,8 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/esiqveland/notify v0.11.0/go.mod h1:63UbVSaeJwF0LVJARHFuPgUAoM7o1BEvCZyknsuonBc=
+github.com/expr-lang/expr v1.16.3 h1:NLldf786GffptcXNxxJx5dQ+FzeWDKChBDqOOwyK8to=
+github.com/expr-lang/expr v1.16.3/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -175,11 +177,13 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939 h1:BhIUXV2ySTLrKgh/Hnts+QTQlIbWtomXt3LMdzME0A0=
 github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939/go.mod h1:omGxs4/6hNjxPKUTjmaNkPzehSnNJOJN6pMEbrlYIT4=
 github.com/twotwotwo/sorts v0.0.0-20160814051341-bf5c1f2b8553 h1:DRC1ubdb3ZmyyIeCSTxjZIQAnpLPfKVgYrLETQuOPjo=

testdata/datesub.csv

@@ -0,0 +1,4 @@
+ID,Name,In,Out
+1,Tom,2023-08-25 11:24:00,2023-08-27 08:33:02
+2,Sally,2023-08-25 11:28:00,2023-08-26 14:17:35
+3,Alf,2023-08-26 11:29:00,2023-08-29 20:43:00

testdata/mutate3len.csv

@@ -0,0 +1,2 @@
+SD
+沈伟

testdata/null_coalescence.csv

@@ -0,0 +1,4 @@
+one,two
+a1,a2
+,b2
+a2,

testdata/positions.csv

@@ -0,0 +1,2 @@
+a,b,c
+1,2,3