diff --git a/README.md b/README.md index bc72d2f1..e7f03d15 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ This project is used in production at an early stage and might undergo breaking We publish a [public Docker image `moov/ach-test-harness`](https://hub.docker.com/r/moov/ach-test-harness/) from Docker Hub or use this repository. No configuration is required to serve on `:2222` and metrics at `:3333/metrics` in Prometheus format. +### Docker image + Pull & start the Docker image: ``` $ docker-compose up @@ -46,6 +48,8 @@ harness_1 | ts=2021-03-24T20:36:10Z msg="listening on [::]:3333" level=info app You can then use an FTP client that connects to `localhost:2222` with a username of `admin` and password of `secret`. Upload files to the `outbound/` directory and watch for any responses. +### config.yml + After setup inspect the configuration file in `./examples/config.yml` and setup some scenarios to match uploaded files. ```yaml @@ -78,8 +82,18 @@ ACHTestHarness: action: return: code: "R03" + + - match: + amount: + value: 12357 # $123.57 + action: + delay: "12h" + return: + code: "R10" ``` +#### config schema + The full config for Responses is below: ```yaml @@ -95,8 +109,22 @@ match: routingNumber: # Exact match of ABA routing number (RDFIIdentification and CheckDigit) traceNumber: # Exact match of TraceNumber entryType: # Checks TransactionCode. Accepted values: credit, debit or prenote. +# Matching will find at most two Actions in the config file order. One Copy Action and one Return/Correction Action. +# Both actions will be executed if the Return/Correction Action has a delay. +# Valid combinations include: +# 1. Copy +# 2. Return/Correction with Delay +# 3. Return/Correction without Delay +# 4. Copy and Return/Correction with Delay +# 5. Nothing +# Invalid combinations are: +# 1. Copy and Return/Correction without Delay +# 2. Copy with Delay (validated when reading configuration) action: - # Copy the EntryDetail to another directory + # How long into the future should we wait before making the correction/return available? + delay: + + # Copy the EntryDetail to another directory (not valid with a delay) copy: path: # Filepath on the FTP server diff --git a/examples/config.yml b/examples/config.yml index fce62fd9..7cfe4236 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -25,6 +25,7 @@ ACHTestHarness: # This matches ./examples/utility-bill.ach accountNumber: "744-5678-99" action: + delay: "12h" correction: code: "C01" data: "744567899" diff --git a/pkg/filedrive/mtime_filter.go b/pkg/filedrive/mtime_filter.go new file mode 100644 index 00000000..ea906824 --- /dev/null +++ b/pkg/filedrive/mtime_filter.go @@ -0,0 +1,36 @@ +package filedrive + +import ( + "time" + + "goftp.io/server/core" +) + +type MTimeFilter struct { + core.Driver +} + +func (mtf MTimeFilter) ListDir(path string, callback func(core.FileInfo) error) error { + now := time.Now() + + return mtf.Driver.ListDir(path, func(info core.FileInfo) error { + if info.ModTime().Before(now) { + return callback(info) + } + return nil + }) +} + +type Factory struct { + DriverFactory core.DriverFactory +} + +func (f *Factory) NewDriver() (core.Driver, error) { + dd, err := f.DriverFactory.NewDriver() + if err != nil { + return nil, err + } + return MTimeFilter{ + Driver: dd, + }, nil +} diff --git a/pkg/response/batch_mirror.go b/pkg/response/batch_mirror.go index ad897ee3..a91e25eb 100644 --- a/pkg/response/batch_mirror.go +++ b/pkg/response/batch_mirror.go @@ -57,7 +57,7 @@ func (bm *batchMirror) saveFiles() error { if filename, err := bm.filename(); err != nil { return fmt.Errorf("unable to get filename: %v", err) } else { - bm.writer.Write(filepath.Join(path, filename), &buf) + bm.writer.Write(filepath.Join(path, filename), &buf, nil) } } return nil diff --git a/pkg/response/entry_transformer.go b/pkg/response/entry_transformer.go index df731419..4938abd5 100644 --- a/pkg/response/entry_transformer.go +++ b/pkg/response/entry_transformer.go @@ -9,12 +9,12 @@ import ( ) type EntryTransformer interface { - MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, action service.Action) (*ach.EntryDetail, error) + MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, action *service.Action) (*ach.EntryDetail, error) } type EntryTransformers []EntryTransformer -func (et EntryTransformers) MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, action service.Action) (*ach.EntryDetail, error) { +func (et EntryTransformers) MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, action *service.Action) (*ach.EntryDetail, error) { var err error for i := range et { ed, err = et[i].MorphEntry(fh, ed, action) @@ -27,7 +27,7 @@ func (et EntryTransformers) MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, a type CorrectionTransformer struct{} -func (t *CorrectionTransformer) MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, action service.Action) (*ach.EntryDetail, error) { +func (t *CorrectionTransformer) MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, action *service.Action) (*ach.EntryDetail, error) { if action.Correction == nil { return ed, nil } @@ -91,7 +91,7 @@ func generateCorrectedData(cor *service.Correction) string { type ReturnTransformer struct{} -func (t *ReturnTransformer) MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, action service.Action) (*ach.EntryDetail, error) { +func (t *ReturnTransformer) MorphEntry(fh ach.FileHeader, ed *ach.EntryDetail, action *service.Action) (*ach.EntryDetail, error) { if action.Return == nil { return ed, nil } diff --git a/pkg/response/entry_transformer_test.go b/pkg/response/entry_transformer_test.go index db173db0..10e70247 100644 --- a/pkg/response/entry_transformer_test.go +++ b/pkg/response/entry_transformer_test.go @@ -22,7 +22,7 @@ func TestMorphEntry__Correction(t *testing.T) { }, } ed := file.Batches[0].GetEntries()[0] - out, err := xform.MorphEntry(file.Header, ed, action) + out, err := xform.MorphEntry(file.Header, ed, &action) require.NoError(t, err) if out.Addenda98 == nil { @@ -49,7 +49,7 @@ func TestMorphEntry__Return(t *testing.T) { }, } ed := file.Batches[0].GetEntries()[0] - out, err := xform.MorphEntry(file.Header, ed, action) + out, err := xform.MorphEntry(file.Header, ed, &action) require.NoError(t, err) if out.Addenda98 != nil { diff --git a/pkg/response/file_transformer.go b/pkg/response/file_transformer.go index 18661fca..d001f732 100644 --- a/pkg/response/file_transformer.go +++ b/pkg/response/file_transformer.go @@ -40,22 +40,11 @@ func NewFileTransformer(logger log.Logger, cfg *service.Config, responses []serv func (ft *FileTransfomer) Transform(file *ach.File) error { out := ach.NewFile() out.SetValidation(ft.ValidateOpts) - - out.Header = ach.NewFileHeader() - out.Header.SetValidation(ft.ValidateOpts) - - out.Header.ImmediateDestination = file.Header.ImmediateOrigin - out.Header.ImmediateDestinationName = file.Header.ImmediateOriginName - out.Header.ImmediateOrigin = file.Header.ImmediateDestination - out.Header.ImmediateOriginName = file.Header.ImmediateDestinationName - out.Header.FileCreationDate = time.Now().Format("060102") - out.Header.FileCreationTime = time.Now().Format("1504") - out.Header.FileIDModifier = "A" - - if err := out.Header.Validate(); err != nil { - return fmt.Errorf("file transform: header validate: %v", err) + if err := createOutHeader(out, file, ft.ValidateOpts); err != nil { + return err } + var entrySaved = false for i := range file.Batches { mirror := newBatchMirror(ft.Writer, file.Batches[i]) batch, err := ach.NewBatch(file.Batches[i].GetHeader()) @@ -64,57 +53,136 @@ func (ft *FileTransfomer) Transform(file *ach.File) error { } entries := file.Batches[i].GetEntries() for j := range entries { - // Check if there's a matching Action and perform it - action := ft.Matcher.FindAction(entries[j]) - if action != nil { - entry, err := ft.Entry.MorphEntry(file.Header, entries[j], *action) + // Check if there's a matching Action and perform it. There may also be a future-dated action to execute. + copyAction, processAction := ft.Matcher.FindAction(entries[j]) + if copyAction != nil { + logger := ft.Matcher.Logger.With(copyAction) + logger.Log("Processing matched action") + + mirror.saveEntry(copyAction.Copy, entries[j]) + entrySaved = true + } + if processAction != nil { + logger := ft.Matcher.Logger.With(processAction) + logger.Log("Processing matched action") + + entry, err := ft.Entry.MorphEntry(file.Header, entries[j], processAction) if err != nil { return fmt.Errorf("transform batch[%d] morph entry[%d] error: %v", i, j, err) } - // When the entry is corrected we need to change the SEC code - if entry.Category == ach.CategoryNOC { - bh := batch.GetHeader() - bh.StandardEntryClassCode = ach.COR - if b, err := ach.NewBatch(bh); b != nil { - batch = b // replace entire Batch + if processAction.Delay != nil { + // need to save off the future-dated entry + futOut := ach.NewFile() + futOut.SetValidation(ft.ValidateOpts) + if futErr := createOutHeader(futOut, file, ft.ValidateOpts); futErr != nil { + return futErr + } + + futMirror := newBatchMirror(ft.Writer, file.Batches[i]) + var futBatch ach.Batcher + var futErr error + + // When the entry is corrected we need to change the SEC code + if entry.Category == ach.CategoryNOC { + bh := *file.Batches[i].GetHeader() + bh.StandardEntryClassCode = ach.COR + futBatch, futErr = ach.NewBatch(&bh) } else { - return fmt.Errorf("transform batch[%d] NOC entry[%d] error: %v", i, j, err) + futBatch, futErr = ach.NewBatch(file.Batches[i].GetHeader()) + } + if futErr != nil { + return fmt.Errorf("transform batch[%d] problem creating Batch: %v", i, futErr) + } + + // Add the transformed entry onto the batch + if entry != nil { + futBatch.AddEntry(entry) } - } - // Save this Entry - if action.Copy != nil { - mirror.saveEntry(action.Copy, entries[j]) + // Save off the entries as requested + if futErr = futMirror.saveFiles(); futErr != nil { + return fmt.Errorf("problem saving entries: %v", futErr) + } + // Create our Batch's Control and other fields + if futErr = futBatch.Create(); futErr != nil { + return fmt.Errorf("transform batch[%d] create error: %v", i, futErr) + } + futOut.AddBatch(futBatch) + + if futErr = writeOutFile(futOut, ft, processAction.Delay); futErr != nil { + return futErr + } } else { + // When the entry is corrected we need to change the SEC code + if entry.Category == ach.CategoryNOC { + bh := batch.GetHeader() + bh.StandardEntryClassCode = ach.COR + if b, err := ach.NewBatch(bh); b != nil { + batch = b // replace entire Batch + } else { + return fmt.Errorf("transform batch[%d] NOC entry[%d] error: %v", i, j, err) + } + } + // Add the transformed entry onto the batch if entry != nil { batch.AddEntry(entry) } + entrySaved = true } } } - // Save off the entries as requested - if err := mirror.saveFiles(); err != nil { - return fmt.Errorf("problem saving entries: %v", err) - } - // Create our Batch's Control and other fields - if entries := batch.GetEntries(); len(entries) > 0 { - if err := batch.Create(); err != nil { - return fmt.Errorf("transform batch[%d] create error: %v", i, err) + + if entrySaved { + // Save off the entries as requested + if err := mirror.saveFiles(); err != nil { + return fmt.Errorf("problem saving entries: %v", err) + } + // Create our Batch's Control and other fields + if entries := batch.GetEntries(); len(entries) > 0 { + if err := batch.Create(); err != nil { + return fmt.Errorf("transform batch[%d] create error: %v", i, err) + } + out.AddBatch(batch) } - out.AddBatch(batch) } } + if err := writeOutFile(out, ft, nil); err != nil { + return err + } + return nil +} + +func createOutHeader(out *ach.File, file *ach.File, opts *ach.ValidateOpts) error { + out.Header = ach.NewFileHeader() + out.Header.SetValidation(opts) + + out.Header.ImmediateDestination = file.Header.ImmediateOrigin + out.Header.ImmediateDestinationName = file.Header.ImmediateOriginName + out.Header.ImmediateOrigin = file.Header.ImmediateDestination + out.Header.ImmediateOriginName = file.Header.ImmediateDestinationName + out.Header.FileCreationDate = time.Now().Format("060102") + out.Header.FileCreationTime = time.Now().Format("1504") + out.Header.FileIDModifier = "A" + + if err := out.Header.Validate(); err != nil { + return fmt.Errorf("file transform: header validate: %v", err) + } + + return nil +} + +func writeOutFile(out *ach.File, ft *FileTransfomer, delay *time.Duration) error { if out != nil && len(out.Batches) > 0 { if err := out.Create(); err != nil { return fmt.Errorf("transform out create: %v", err) } if err := out.Validate(); err == nil { - filepath := filepath.Join(ft.returnPath, generateFilename(out)) // TODO(adam): need to determine return path - if err := ft.Writer.WriteFile(filepath, out); err != nil { - return fmt.Errorf("transform write %s: %v", filepath, err) + generatedFilePath := filepath.Join(ft.returnPath, generateFilename(out)) // TODO(adam): need to determine return path + if err := ft.Writer.WriteFile(generatedFilePath, out, delay); err != nil { + return fmt.Errorf("transform write %s: %v", generatedFilePath, err) } } else { return fmt.Errorf("transform validate out file: %v", err) diff --git a/pkg/response/file_transformer_test.go b/pkg/response/file_transformer_test.go index ee0e0b9f..68b86a94 100644 --- a/pkg/response/file_transformer_test.go +++ b/pkg/response/file_transformer_test.go @@ -1,110 +1,370 @@ package response import ( - "bytes" "os" "path/filepath" + "strings" "testing" + "time" "github.com/moov-io/ach" "github.com/moov-io/ach-test-harness/pkg/service" - "github.com/moov-io/ach/cmd/achcli/describe" "github.com/moov-io/base/log" "github.com/stretchr/testify/require" ) -func TestFileTransformer__CorrectedPrenote(t *testing.T) { - resp := service.Response{ - Match: service.Match{ - EntryType: service.EntryTypePrenote, - AccountNumber: "810044964044", +func TestFileTransformer(t *testing.T) { + var delay, err = time.ParseDuration("24h") + require.NoError(t, err) + + var matchPrenote = service.Match{ + EntryType: service.EntryTypePrenote, + AccountNumber: "810044964044", + } + var matchEntry1 = service.Match{ + IndividualName: "Incorrect Name", + } + var actionCopy = service.Action{ + Copy: &service.Copy{ + Path: "/reconciliation/", }, - Action: service.Action{ - Correction: &service.Correction{ - Code: "C01", - Data: "445566778", - }, + } + var actionReturn = service.Action{ + Return: &service.Return{ + Code: "R03", + }, + } + var actionCorrection = service.Action{ + Correction: &service.Correction{ + Code: "C01", + Data: "445566778", }, } - fileTransformer, dir := testFileTransformer(t, resp) + var actionDelayReturn = actionReturn + actionDelayReturn.Delay = &delay + var actionDelayCorrection = actionCorrection + actionDelayCorrection.Delay = &delay - prenote, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "prenote.ach")) - require.NoError(t, err) + t.Run("NoMatch", func(t *testing.T) { + resp := service.Response{ + Match: matchEntry1, + Action: actionCopy, + } + fileTransformer, dir := testFileTransformer(t, resp) - err = fileTransformer.Transform(prenote) - require.NoError(t, err) + // read the file + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "prenote.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) - retdir := filepath.Join(dir, "returned") + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) - fds, err := os.ReadDir(retdir) - require.NoError(t, err) - require.Len(t, fds, 1) + // verify no "returned" files created + retdir := filepath.Join(dir, "returned") + _, err = os.ReadDir(retdir) + require.Error(t, err) - found, err := ach.ReadFile(filepath.Join(retdir, fds[0].Name())) - require.NoError(t, err) + // verify no "reconciliation" files created + recondir := filepath.Join(dir, "reconciliation") + _, err = os.ReadDir(recondir) + require.Error(t, err) + }) - var out bytes.Buffer - describe.File(&out, found, nil) - require.Contains(t, out.String(), "26 (Checking Return NOC Debit)") - require.Contains(t, out.String(), "C01") -} + t.Run("CopyOnly", func(t *testing.T) { + resp := service.Response{ + Match: matchEntry1, + Action: actionCopy, + } + fileTransformer, dir := testFileTransformer(t, resp) -func TestFileTransformer__ReturnedPrenote(t *testing.T) { - resp := service.Response{ - Match: service.Match{ - EntryType: service.EntryTypePrenote, - AccountNumber: "810044964044", - }, - Action: service.Action{ - Return: &service.Return{ - Code: "R03", - }, - }, - } - fileTransformer, dir := testFileTransformer(t, resp) + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) - prenote, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "prenote.ach")) - require.NoError(t, err) + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) - err = fileTransformer.Transform(prenote) - require.NoError(t, err) + // verify no "returned" files created + retdir := filepath.Join(dir, "returned") + _, err = os.ReadDir(retdir) + require.Error(t, err) - retdir := filepath.Join(dir, "returned") + // verify the "reconciliation" file created + recondir := filepath.Join(dir, "reconciliation") + fds, err := os.ReadDir(recondir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, _ := ach.ReadFile(filepath.Join(recondir, fds[0].Name())) + require.Equal(t, matchEntry1.IndividualName, strings.Trim(found.Batches[0].GetEntries()[0].IndividualName, " ")) - fds, err := os.ReadDir(retdir) - require.NoError(t, err) - require.Len(t, fds, 1) + // verify the timestamp on the file is in the past + fInfo, err := fds[0].Info() + require.NoError(t, err) + require.Less(t, fInfo.ModTime(), time.Now()) + }) - found, err := ach.ReadFile(filepath.Join(retdir, fds[0].Name())) - require.NoError(t, err) + t.Run("ProcessOnly - Return", func(t *testing.T) { + resp := service.Response{ + Match: matchPrenote, + Action: actionReturn, + } + fileTransformer, dir := testFileTransformer(t, resp) + + // read the file + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "prenote.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) + + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) + + // verify the "returned" file created + retdir := filepath.Join(dir, "returned") + fds, err := os.ReadDir(retdir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, err := ach.ReadFile(filepath.Join(retdir, fds[0].Name())) + require.NoError(t, err) + require.Equal(t, "R03", found.Batches[0].GetEntries()[0].Addenda99.ReturnCode) + + // verify the timestamp on the file is in the past + fInfo, err := fds[0].Info() + require.NoError(t, err) + require.Less(t, fInfo.ModTime(), time.Now()) + + // verify no "reconciliation" files created + recondir := filepath.Join(dir, "reconciliation") + _, err = os.ReadDir(recondir) + require.Error(t, err) + }) + + t.Run("ProcessOnly - Correction", func(t *testing.T) { + resp := service.Response{ + Match: matchPrenote, + Action: actionCorrection, + } + fileTransformer, dir := testFileTransformer(t, resp) + + // read the file + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "prenote.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) + + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) + + // verify the "returned" file created + retdir := filepath.Join(dir, "returned") + fds, err := os.ReadDir(retdir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, err := ach.ReadFile(filepath.Join(retdir, fds[0].Name())) + require.NoError(t, err) + require.Equal(t, "C01", found.Batches[0].GetEntries()[0].Addenda98.ChangeCode) + + // verify the timestamp on the file is in the past + fInfo, err := fds[0].Info() + require.NoError(t, err) + require.Less(t, fInfo.ModTime(), time.Now()) + + // verify no "reconciliation" files created + recondir := filepath.Join(dir, "reconciliation") + _, err = os.ReadDir(recondir) + require.Error(t, err) + }) - var out bytes.Buffer - describe.File(&out, found, nil) - require.Contains(t, out.String(), "28 (Checking Prenote Debit)") - require.Contains(t, out.String(), "R03") + t.Run("DelayProcessOnly - Return", func(t *testing.T) { + resp := service.Response{ + Match: matchEntry1, + Action: actionDelayReturn, + } + fileTransformer, dir := testFileTransformer(t, resp) + + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) + + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) + + // verify the "returned" file created + retdir := filepath.Join(dir, "returned") + fds, err := os.ReadDir(retdir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, err := ach.ReadFile(filepath.Join(retdir, fds[0].Name())) + require.NoError(t, err) + require.Equal(t, "R03", found.Batches[0].GetEntries()[0].Addenda99.ReturnCode) + + // verify the timestamp on the file is in the future + fInfo, err := fds[0].Info() + require.NoError(t, err) + require.Greater(t, fInfo.ModTime(), time.Now()) + + // verify no "reconciliation" files created + recondir := filepath.Join(dir, "reconciliation") + _, err = os.ReadDir(recondir) + require.Error(t, err) + }) + + t.Run("DelayProcessOnly - Correction", func(t *testing.T) { + resp := service.Response{ + Match: matchEntry1, + Action: actionDelayCorrection, + } + fileTransformer, dir := testFileTransformer(t, resp) + + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) + + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) + + // verify the "returned" file created + retdir := filepath.Join(dir, "returned") + fds, err := os.ReadDir(retdir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, err := ach.ReadFile(filepath.Join(retdir, fds[0].Name())) + require.NoError(t, err) + require.Equal(t, "C01", found.Batches[0].GetEntries()[0].Addenda98.ChangeCode) + + // verify the timestamp on the file is in the future + fInfo, err := fds[0].Info() + require.NoError(t, err) + require.Greater(t, fInfo.ModTime(), time.Now()) + + // verify no "reconciliation" files created + recondir := filepath.Join(dir, "reconciliation") + _, err = os.ReadDir(recondir) + require.Error(t, err) + }) + + t.Run("CopyAndDelayProcess - Return", func(t *testing.T) { + resp1 := service.Response{ + Match: matchEntry1, + Action: actionCopy, + } + resp2 := service.Response{ + Match: matchEntry1, + Action: actionDelayReturn, + } + fileTransformer, dir := testFileTransformer(t, resp1, resp2) + + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) + + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) + + // verify the "returned" file created + retdir := filepath.Join(dir, "returned") + fds, err := os.ReadDir(retdir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, err := ach.ReadFile(filepath.Join(retdir, fds[0].Name())) + require.NoError(t, err) + require.Equal(t, "R03", found.Batches[0].GetEntries()[0].Addenda99.ReturnCode) + + // verify the timestamp on the file is in the future + fInfo, err := fds[0].Info() + require.NoError(t, err) + require.Greater(t, fInfo.ModTime(), time.Now()) + + // verify the "reconciliation" file created + recondir := filepath.Join(dir, "reconciliation") + fds, err = os.ReadDir(recondir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, _ = ach.ReadFile(filepath.Join(recondir, fds[0].Name())) + require.Equal(t, matchEntry1.IndividualName, strings.Trim(found.Batches[0].GetEntries()[0].IndividualName, " ")) + + // verify the timestamp on the file is in the past + fInfo, err = fds[0].Info() + require.NoError(t, err) + require.Less(t, fInfo.ModTime(), time.Now()) + }) + + t.Run("CopyAndDelayProcess - Correction", func(t *testing.T) { + resp1 := service.Response{ + Match: matchEntry1, + Action: actionCopy, + } + resp2 := service.Response{ + Match: matchEntry1, + Action: actionDelayCorrection, + } + fileTransformer, dir := testFileTransformer(t, resp1, resp2) + + achIn, err := ach.ReadFile(filepath.Join("..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, achIn) + + // transform the file + err = fileTransformer.Transform(achIn) + require.NoError(t, err) + + // verify the "returned" file created + retdir := filepath.Join(dir, "returned") + fds, err := os.ReadDir(retdir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, err := ach.ReadFile(filepath.Join(retdir, fds[0].Name())) + require.NoError(t, err) + require.Equal(t, "C01", found.Batches[0].GetEntries()[0].Addenda98.ChangeCode) + + // verify the timestamp on the file is in the future + fInfo, err := fds[0].Info() + require.NoError(t, err) + require.Greater(t, fInfo.ModTime(), time.Now()) + + // verify the "reconciliation" file created + recondir := filepath.Join(dir, "reconciliation") + fds, err = os.ReadDir(recondir) + require.NoError(t, err) + require.Len(t, fds, 1) + found, _ = ach.ReadFile(filepath.Join(recondir, fds[0].Name())) + require.Equal(t, matchEntry1.IndividualName, strings.Trim(found.Batches[0].GetEntries()[0].IndividualName, " ")) + + // verify the timestamp on the file is in the past + fInfo, err = fds[0].Info() + require.NoError(t, err) + require.Less(t, fInfo.ModTime(), time.Now()) + }) } -func testFileTransformer(t *testing.T, resp service.Response) (*FileTransfomer, string) { +func testFileTransformer(t *testing.T, resp ...service.Response) (*FileTransfomer, string) { t.Helper() - logger := log.NewTestLogger() + dir, ftpServer := fileBackedFtpServer(t) + cfg := &service.Config{ Matching: service.Matching{ Debug: true, }, Servers: service.ServerConfig{ FTP: &service.FTPConfig{ + RootPath: dir, Paths: service.Paths{ Return: "./returned/", }, }, }, } - responses := []service.Response{resp} - - dir, ftpServer := fileBackedFtpServer(t) + responses := resp + logger := log.NewTestLogger() w := NewFileWriter(logger, cfg.Servers, ftpServer) return NewFileTransformer(logger, cfg, responses, w), dir diff --git a/pkg/response/file_writer.go b/pkg/response/file_writer.go index 0cc163e1..00f65d1b 100644 --- a/pkg/response/file_writer.go +++ b/pkg/response/file_writer.go @@ -4,7 +4,9 @@ import ( "bytes" "fmt" "io" + "os" "path/filepath" + "time" "github.com/moov-io/ach" "github.com/moov-io/ach-test-harness/pkg/service" @@ -14,37 +16,39 @@ import ( ) type FileWriter interface { - Write(filepath string, r io.Reader) error - WriteFile(filename string, file *ach.File) error + Write(filepath string, r io.Reader, delay *time.Duration) error + WriteFile(filename string, file *ach.File, delay *time.Duration) error } func NewFileWriter(logger log.Logger, cfg service.ServerConfig, ftpServer *ftp.Server) FileWriter { if cfg.FTP != nil { return &FTPFileWriter{ - cfg: cfg.FTP.Paths, - logger: logger, - server: ftpServer, + cfg: cfg.FTP.Paths, + rootPath: cfg.FTP.RootPath, + logger: logger, + server: ftpServer, } } return nil } type FTPFileWriter struct { - cfg service.Paths - logger log.Logger - server *ftp.Server + cfg service.Paths + rootPath string + logger log.Logger + server *ftp.Server } -func (w *FTPFileWriter) WriteFile(filepath string, file *ach.File) error { +func (w *FTPFileWriter) WriteFile(filepath string, file *ach.File, futureDated *time.Duration) error { var buf bytes.Buffer if err := ach.NewWriter(&buf).Write(file); err != nil { return fmt.Errorf("write %s: %v", filepath, err) } w.logger.Info().Log(fmt.Sprintf("writing %s (%d bytes)", filepath, buf.Len())) - return w.Write(filepath, &buf) + return w.Write(filepath, &buf, futureDated) } -func (w *FTPFileWriter) Write(path string, r io.Reader) error { +func (w *FTPFileWriter) Write(path string, r io.Reader, futureDated *time.Duration) error { driver, err := w.server.Factory.NewDriver() if err != nil { return fmt.Errorf("get driver to write %s: %v", path, err) @@ -57,6 +61,13 @@ func (w *FTPFileWriter) Write(path string, r io.Reader) error { if _, err := driver.PutFile(path, r, false); err != nil { return fmt.Errorf("STOR %s: %v", path, err) } + + if futureDated != nil { + if err := os.Chtimes(filepath.Join(w.rootPath, path), time.Now(), time.Now().Add(*futureDated)); err != nil { + return fmt.Errorf("chtimes: %s: %v", path, err) + } + } + return nil } diff --git a/pkg/response/match/matcher.go b/pkg/response/match/matcher.go index fa4353d7..9230b97d 100644 --- a/pkg/response/match/matcher.go +++ b/pkg/response/match/matcher.go @@ -27,53 +27,29 @@ func New(logger log.Logger, cfg service.Matching, responses []service.Response) } } -func (m Matcher) FindAction(ed *ach.EntryDetail) *service.Action { - logger := m.Logger.With(log.Fields{ - "entry_trace_number": log.String(ed.TraceNumber), - }) - logger.Log("starting EntryDetail matching") - +func (m Matcher) FindAction(ed *ach.EntryDetail) (copyAction *service.Action, processAction *service.Action) { + /* + * See https://github.com/moov-io/ach-test-harness#config-schema for more details on how to configure. + */ for i := range m.Responses { + logger := m.Logger.With(log.Fields{ + "entry_trace_number": log.String(ed.TraceNumber), + }) + logger.Log("starting EntryDetail matching") + positive, negative := 0, 0 // Matchers are AND'd together matcher := m.Responses[i].Match action := m.Responses[i].Action - var copyPath string - var correctionCode string - var correctionData string - var returnCode string - var amount int - - // Safely retrieve several values that are needed for the debug log below - if action.Copy != nil { - copyPath = action.Copy.Path - logger = logger.With(log.Fields{ - "copy_path": log.String(copyPath), - }) + if copyAction != nil && action.Copy != nil { + continue // skip, we already have a copy action } - - if action.Correction != nil { - correctionCode = action.Correction.Code - correctionData = action.Correction.Data - logger = logger.With(log.Fields{ - "correction_code": log.String(correctionCode), - "correction_data": log.String(correctionData), - }) + if processAction != nil && action.Return != nil { + continue // skip, we already have a process action } - if action.Return != nil { - returnCode = action.Return.Code - logger = logger.With(log.Fields{ - "return_code": log.String(returnCode), - }) - } - - if matcher.Amount != nil { - amount = matcher.Amount.Value - logger = logger.With(log.Fields{ - "amount": log.Int(amount), - }) - } + logger = logger.With(action) + logger = logger.With(matcher) if m.Debug { logger = logger.With(log.Fields{ @@ -179,10 +155,19 @@ func (m Matcher) FindAction(ed *ach.EntryDetail) *service.Action { logger.Logf("FINAL matching score negative=%d positive=%d", negative, positive) if negative == 0 && positive > 0 { - return &m.Responses[i].Action + // Action is valid, figure out where it belongs + if m.Responses[i].Action.Copy != nil { + copyAction = &m.Responses[i].Action + } else { + processAction = &m.Responses[i].Action + // A non-Copy (process) Action with no Delay supersedes everything else + if processAction.Delay == nil { + return nil, processAction + } + } } } - return nil + return } func TraceNumber(m service.Match, ed *ach.EntryDetail) bool { diff --git a/pkg/response/match/matcher_test.go b/pkg/response/match/matcher_test.go index b691353c..f5b892e1 100644 --- a/pkg/response/match/matcher_test.go +++ b/pkg/response/match/matcher_test.go @@ -3,6 +3,7 @@ package match import ( "path/filepath" "testing" + "time" "github.com/moov-io/ach" "github.com/moov-io/ach-test-harness/pkg/service" @@ -176,46 +177,156 @@ func TestMatchTraceNumber(t *testing.T) { // TransactionCode RDFIIdentification AccountNumber Amount Name TraceNumber Category // 22 27397636 273976369 100 Incorrect Name 273976367520469 func TestMultiMatch(t *testing.T) { - matcher := Matcher{ - Logger: log.NewTestLogger(), - Responses: []service.Response{ + var delay, err = time.ParseDuration("12h") + require.NoError(t, err) + + var matchNone = service.Match{ + Amount: &service.Amount{ + Min: 500000, // $5,000.00 + Max: 1000000, // $10,000.00 + }, + EntryType: service.EntryTypeDebit, + } + var matchEntry1 = service.Match{ + IndividualName: "Incorrect Name", + } + var actionCopy = service.Action{ + Copy: &service.Copy{ + Path: "/reconciliation", + }, + } + var actionReturn = service.Action{ + Return: &service.Return{ + Code: "R01", + }, + } + var actionCorrection = service.Action{ + Correction: &service.Correction{ + Code: "C04", + Data: "Correct Name", + }, + } + var actionDelayReturn = actionReturn + actionDelayReturn.Delay = &delay + var actionDelayCorrection = actionCorrection + actionDelayCorrection.Delay = &delay + + t.Run("No Match", func(t *testing.T) { + var matcher Matcher + matcher.Logger = log.NewTestLogger() + matcher.Responses = []service.Response{} + + // Read our test file + file, err := ach.ReadFile(filepath.Join("..", "..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, file) + require.True(t, len(file.Batches) > 0) + entries := file.Batches[0].GetEntries() + + // Find our Action + copyAction, processAction := matcher.FindAction(entries[0]) + require.Nil(t, copyAction) + require.Nil(t, processAction) + + // Find our Action + copyAction, processAction = matcher.FindAction(entries[1]) + require.Nil(t, copyAction) + require.Nil(t, processAction) + }) + + t.Run("Match Copy only", func(t *testing.T) { + var matcher Matcher + matcher.Logger = log.NewTestLogger() + matcher.Responses = []service.Response{ { - Match: service.Match{ - Amount: &service.Amount{ - Min: 500000, // $5,000.00 - Max: 1000000, // $10,000.00 - }, - EntryType: service.EntryTypeDebit, - }, - Action: service.Action{ - Return: &service.Return{ - Code: "R01", - }, - }, + Match: matchEntry1, + Action: actionCopy, }, + } + + // Read our test file + file, err := ach.ReadFile(filepath.Join("..", "..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, file) + require.True(t, len(file.Batches) > 0) + entries := file.Batches[0].GetEntries() + + // Find our Action + copyAction, processAction := matcher.FindAction(entries[0]) + require.Nil(t, copyAction) + require.Nil(t, processAction) + + // Find our Action + copyAction, processAction = matcher.FindAction(entries[1]) + require.NotNil(t, copyAction) + require.Equal(t, actionCopy, *copyAction) + require.Nil(t, processAction) + }) + + t.Run("Match Process only", func(t *testing.T) { + var matcher Matcher + matcher.Logger = log.NewTestLogger() + matcher.Responses = []service.Response{ { - Match: service.Match{ - IndividualName: "Incorrect Name", - }, - Action: service.Action{ - Correction: &service.Correction{ - Code: "C04", - Data: "Correct Name", - }, - }, + Match: matchEntry1, + Action: actionReturn, }, - }, - } - - // Read our test file - file, err := ach.ReadFile(filepath.Join("..", "..", "..", "testdata", "20210308-1806-071000301.ach")) - require.NoError(t, err) - entries := file.Batches[0].GetEntries() + } - action := matcher.FindAction(entries[0]) - require.Nil(t, action) + // Read our test file + file, err := ach.ReadFile(filepath.Join("..", "..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, file) + require.True(t, len(file.Batches) > 0) + entries := file.Batches[0].GetEntries() + + // Find our Action + copyAction, processAction := matcher.FindAction(entries[0]) + require.Nil(t, copyAction) + require.Nil(t, processAction) + + // Find our Action + copyAction, processAction = matcher.FindAction(entries[1]) + require.Nil(t, copyAction) + require.NotNil(t, processAction) + require.Equal(t, actionReturn, *processAction) + }) + + t.Run("Match Copy + Process", func(t *testing.T) { + var matcher Matcher + matcher.Logger = log.NewTestLogger() + matcher.Responses = []service.Response{ + { + Match: matchEntry1, + Action: actionDelayCorrection, + }, + { + Match: matchNone, + Action: actionReturn, + }, + { + Match: matchEntry1, + Action: actionCopy, + }, + } - // Find our Action - action = matcher.FindAction(entries[1]) - require.Equal(t, action.Correction.Code, "C04") + // Read our test file + file, err := ach.ReadFile(filepath.Join("..", "..", "..", "testdata", "20210308-1806-071000301.ach")) + require.NoError(t, err) + require.NotNil(t, file) + require.True(t, len(file.Batches) > 0) + entries := file.Batches[0].GetEntries() + + // Find our Action + copyAction, processAction := matcher.FindAction(entries[0]) + require.Nil(t, copyAction) + require.Nil(t, processAction) + + // Find our Action + copyAction, processAction = matcher.FindAction(entries[1]) + require.NotNil(t, copyAction) + require.Equal(t, actionCopy, *copyAction) + require.NotNil(t, processAction) + require.Equal(t, actionDelayCorrection, *processAction) + }) } diff --git a/pkg/service/environment.go b/pkg/service/environment.go index 0b980c80..3984b4e0 100644 --- a/pkg/service/environment.go +++ b/pkg/service/environment.go @@ -60,6 +60,9 @@ func LoadConfig(logger log.Logger) (*Config, error) { if err := configService.Load(global); err != nil { return nil, err } + if err := global.Validate(); err != nil { + return nil, err + } cfg := &global.ACHTestHarness diff --git a/pkg/service/model_config.go b/pkg/service/model_config.go index 79637ff1..96ee6b54 100644 --- a/pkg/service/model_config.go +++ b/pkg/service/model_config.go @@ -3,15 +3,22 @@ package service import ( + "errors" "fmt" + "time" "github.com/moov-io/ach" + "github.com/moov-io/base/log" ) type GlobalConfig struct { ACHTestHarness Config } +func (gc *GlobalConfig) Validate() error { + return gc.ACHTestHarness.Validate() +} + // Config defines all the configuration for the app type Config struct { Servers ServerConfig @@ -20,6 +27,16 @@ type Config struct { Responses []Response } +func (cfg *Config) Validate() error { + for i := range cfg.Responses { + if err := cfg.Responses[i].Validate(); err != nil { + return err + } + } + + return nil +} + func (cfg *Config) responsePaths() []string { var out []string for i := range cfg.Responses { @@ -78,6 +95,13 @@ type Response struct { Action Action } +func (r *Response) Validate() error { + if r.Match.Empty() { + return errors.New("no Match configured") + } + return r.Action.Validate() +} + type Match struct { AccountNumber string Amount *Amount @@ -87,6 +111,17 @@ type Match struct { TraceNumber string } +func (m Match) Context() map[string]log.Valuer { + logFields := log.Fields{} + + if m.Amount != nil { + var amount = m.Amount.Value + logFields["amount"] = log.Int(amount) + } + + return logFields +} + func (m Match) Empty() bool { return m.AccountNumber == "" && m.Amount.Empty() && string(m.EntryType) == "" && m.IndividualName == "" && @@ -116,11 +151,68 @@ const ( ) type Action struct { + Delay *time.Duration // e.g. "12h" or "10s" Copy *Copy Correction *Correction Return *Return } +func (a Action) Context() map[string]log.Valuer { + logFields := log.Fields{} + + // Safely retrieve several values that are needed for the debug log below + if a.Delay != nil { + var delayTime = a.Delay.String() + logFields["delay"] = log.String(delayTime) + } + + if a.Copy != nil { + var copyPath = a.Copy.Path + logFields["copy_path"] = log.String(copyPath) + } + + if a.Correction != nil { + var correctionCode = a.Correction.Code + var correctionData = a.Correction.Data + logFields["correction_code"] = log.String(correctionCode) + logFields["correction_data"] = log.String(correctionData) + } + + if a.Return != nil { + var returnCode = a.Return.Code + logFields["return_code"] = log.String(returnCode) + } + + return logFields +} + +func (a *Action) Validate() error { + // Delay is only valid for Return and Correction + if a.Delay != nil && a.Copy != nil { + return errors.New("Delay and Copy are not valid together in an Action") + } + + // only allowed 1 of Copy, Return, Correction to be configured + var count = 0 + if a.Copy != nil { + count++ + } + if a.Return != nil { + count++ + } + if a.Correction != nil { + count++ + } + if count > 1 { + return errors.New("only 1 of Copy, Return, Correction can be configured in an Action") + } + if a.Delay != nil && count == 0 { + return errors.New("either Return or Correction is required if Delay is set") + } + + return nil +} + type Copy struct { Path string } diff --git a/pkg/service/model_config_test.go b/pkg/service/model_config_test.go index 9577119a..47ce744c 100644 --- a/pkg/service/model_config_test.go +++ b/pkg/service/model_config_test.go @@ -2,10 +2,26 @@ package service import ( "testing" + "time" "github.com/stretchr/testify/require" ) +func TestConfig__Config(t *testing.T) { + cfg := &Config{ + Responses: []Response{ + { + Action: Action{ + Copy: &Copy{Path: "/reconciliation/"}, + Return: &Return{Code: "R01"}, + }, + }, + }, + } + require.Error(t, cfg.Validate()) + +} + func TestConfig__Match(t *testing.T) { m := Match{} require.True(t, m.Empty()) @@ -14,6 +30,37 @@ func TestConfig__Match(t *testing.T) { require.False(t, m.Empty()) } +func TestConfig__Response(t *testing.T) { + r := &Response{ + Match: Match{ + IndividualName: "John Doe", + }, + Action: Action{ // invalid + Copy: &Copy{Path: "/reconciliation/"}, + Return: &Return{Code: "R01"}, + }, + } + require.Error(t, r.Validate()) + + r = &Response{ + Match: Match{ + IndividualName: "John Doe", + }, + Action: Action{ + Copy: &Copy{Path: "/reconciliation/"}, + }, + } + require.NoError(t, r.Validate()) + + r = &Response{ + // invalid - no Match + Action: Action{ + Return: &Return{Code: "R01"}, + }, + } + require.Error(t, r.Validate()) +} + func TestConfig__Amount(t *testing.T) { a := &Amount{} require.True(t, a.Empty()) @@ -31,3 +78,111 @@ func TestConfig__Return(t *testing.T) { r.Code = "R99" require.Error(t, r.Validate()) } + +func TestConfig__Action(t *testing.T) { + var delay, err = time.ParseDuration("12h") + require.NoError(t, err) + + var actionCopy = &Copy{Path: "/reconciliation/"} + var actionReturn = &Return{Code: "R01"} + var actionCorrection = &Correction{Code: "C01"} + + t.Run("Delay only", func(t *testing.T) { + var a Action + a.Delay = &delay + require.Error(t, a.Validate()) + }) + + t.Run("Delay + Copy", func(t *testing.T) { + var a Action + a.Delay = &delay + a.Copy = actionCopy + require.Error(t, a.Validate()) + }) + + t.Run("Delay + Return", func(t *testing.T) { + var a Action + a.Delay = &delay + a.Return = actionReturn + require.NoError(t, a.Validate()) + }) + + t.Run("Delay + Correction", func(t *testing.T) { + var a Action + a.Delay = &delay + a.Correction = actionCorrection + require.NoError(t, a.Validate()) + }) + + t.Run("Delay + Copy + Return", func(t *testing.T) { + var a Action + a.Delay = &delay + a.Copy = actionCopy + a.Return = actionReturn + require.Error(t, a.Validate()) + }) + + t.Run("Delay + Copy + Correction", func(t *testing.T) { + var a Action + a.Delay = &delay + a.Copy = actionCopy + a.Correction = actionCorrection + require.Error(t, a.Validate()) + }) + + t.Run("Delay + Copy + Return + Correction", func(t *testing.T) { + var a Action + a.Delay = &delay + a.Copy = actionCopy + a.Return = actionReturn + a.Correction = actionCorrection + require.Error(t, a.Validate()) + }) + + t.Run("Copy only", func(t *testing.T) { + var a Action + a.Copy = actionCopy + require.NoError(t, a.Validate()) + }) + + t.Run("Copy + Return", func(t *testing.T) { + var a Action + a.Copy = actionCopy + a.Return = actionReturn + require.Error(t, a.Validate()) + }) + + t.Run("Copy + Correction", func(t *testing.T) { + var a Action + a.Copy = actionCopy + a.Correction = actionCorrection + require.Error(t, a.Validate()) + }) + + t.Run("Copy + Return + Correction", func(t *testing.T) { + var a Action + a.Copy = actionCopy + a.Return = actionReturn + a.Correction = actionCorrection + require.Error(t, a.Validate()) + }) + + t.Run("Return only", func(t *testing.T) { + var a Action + a.Return = actionReturn + require.NoError(t, a.Validate()) + }) + + t.Run("Return + Correction", func(t *testing.T) { + var a Action + a.Return = actionReturn + a.Correction = actionCorrection + require.Error(t, a.Validate()) + }) + + t.Run("Correction only", func(t *testing.T) { + var a Action + a.Correction = actionCorrection + require.NoError(t, a.Validate()) + }) +} diff --git a/pkg/service/server.go b/pkg/service/server.go index 0f8412e0..06b27af8 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -10,6 +10,7 @@ import ( "path/filepath" _ "github.com/moov-io/ach-test-harness" + "github.com/moov-io/ach-test-harness/pkg/filedrive" "github.com/moov-io/base/admin" "github.com/moov-io/base/log" @@ -42,12 +43,15 @@ func bootFTPServer(errs chan<- error, logger log.Logger, cfg *FTPConfig, respons createDataDirectories(errs, logger, cfg) // Start the FTP server - fileDriver := &file.DriverFactory{ + fileDriverFactory := &file.DriverFactory{ RootPath: cfg.RootPath, Perm: ftp.NewSimplePerm("user", "group"), } + filteringDriver := &filedrive.Factory{ + DriverFactory: fileDriverFactory, + } opts := &ftp.ServerOpts{ - Factory: fileDriver, + Factory: filteringDriver, Port: cfg.Port, Hostname: cfg.Hostname, Auth: &ftp.SimpleAuth{