Merge pull request #123 from jenakan/parse-ascii-images

Parsing ICL with image data

bundle.go

@@ -118,8 +118,6 @@ func (b *Bundle) build() error {
 		}
 
 		itemCount = itemCount + 1
-		itemCount = itemCount + len(cd.CheckDetailAddendumA) + len(cd.CheckDetailAddendumB) + len(cd.CheckDetailAddendumC)
-		itemCount = itemCount + len(cd.ImageViewDetail) + len(cd.ImageViewData) + len(cd.ImageViewAnalysis)
 		bundleTotalAmount = bundleTotalAmount + cd.ItemAmount
 		if cd.MICRValidIndicator == 1 {
 			micrValidTotalAmount = micrValidTotalAmount + cd.ItemAmount
@@ -136,8 +134,6 @@ func (b *Bundle) build() error {
 			return err
 		}
 		itemCount = itemCount + 1
-		itemCount = itemCount + len(rd.ReturnDetailAddendumA) + len(rd.ReturnDetailAddendumB) + len(rd.ReturnDetailAddendumC) + len(rd.ReturnDetailAddendumD)
-		itemCount = itemCount + len(rd.ImageViewDetail) + len(rd.ImageViewData) + len(rd.ImageViewAnalysis)
 		bundleTotalAmount = bundleTotalAmount + rd.ItemAmount
 		bundleImagesCount = bundleImagesCount + len(rd.ImageViewDetail)
 	}

bundleHeader.go

@@ -6,7 +6,6 @@ package imagecashletter
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 	"time"
 	"unicode/utf8"
@@ -291,7 +290,6 @@ func (bh *BundleHeader) reservedField() string {
 
 // SetBundleSequenceNumber sets BundleSequenceNumber
 func (bh *BundleHeader) SetBundleSequenceNumber(seq int) string {
-	bundleSequence := strconv.Itoa(seq)
-	bh.BundleSequenceNumber = bundleSequence
+	bh.BundleSequenceNumber = bh.numericField(seq, 4)
 	return bh.BundleSequenceNumber
 }

cashLetter.go

@@ -4,7 +4,9 @@
 
 package imagecashletter
 
-import "fmt"
+import (
+	"fmt"
+)
 
 // CashLetterError is an Error that describes CashLetter validation issues
 type CashLetterError struct {
@@ -122,11 +124,17 @@ func (cl *CashLetter) build() error {
 		// Set Bundle Sequence Numbers
 		b.BundleHeader.SetBundleSequenceNumber(bundleSequenceNumber)
 
+		// Sequence  Number
+		cdSequenceNumber := 1
+
 		// Check Items
 		for _, cd := range b.Checks {
 
-			// Sequence  Number
-			cdSequenceNumber := 1
+			if cd.EceInstitutionItemSequenceNumber != "" {
+				i := cd.parseNumField(cd.EceInstitutionItemSequenceNumber)
+				cdSequenceNumber = i
+			}
+
 			// Record Numbers
 			cdAddendumARecordNumber := 1
 			cdAddendumCRecordNumber := 1
@@ -154,8 +162,6 @@ func (cl *CashLetter) build() error {
 			cdSequenceNumber++
 
 			cashLetterItemsCount = cashLetterItemsCount + 1
-			cashLetterItemsCount = cashLetterItemsCount + len(cd.CheckDetailAddendumA) + len(cd.CheckDetailAddendumB) + len(cd.CheckDetailAddendumC)
-			cashLetterItemsCount = cashLetterItemsCount + len(cd.ImageViewDetail) + len(cd.ImageViewData) + len(cd.ImageViewAnalysis)
 			cashLetterTotalAmount = cashLetterTotalAmount + cd.ItemAmount
 			cashLetterImagesCount = cashLetterImagesCount + len(cd.ImageViewDetail)
 		}
@@ -193,8 +199,6 @@ func (cl *CashLetter) build() error {
 			rdSequenceNumber++
 
 			cashLetterItemsCount = cashLetterItemsCount + 1
-			cashLetterItemsCount = cashLetterItemsCount + len(rd.ReturnDetailAddendumA) + len(rd.ReturnDetailAddendumB) + len(rd.ReturnDetailAddendumC) + len(rd.ReturnDetailAddendumD)
-			cashLetterItemsCount = cashLetterItemsCount + len(rd.ImageViewDetail) + len(rd.ImageViewData) + len(rd.ImageViewAnalysis)
 			cashLetterTotalAmount = cashLetterTotalAmount + rd.ItemAmount
 			cashLetterImagesCount = cashLetterImagesCount + len(rd.ImageViewDetail)
 		}
@@ -216,7 +220,11 @@ func (cl *CashLetter) build() error {
 	clc.CashLetterItemsCount = cashLetterItemsCount
 	clc.CashLetterTotalAmount = cashLetterTotalAmount
 	clc.CashLetterImagesCount = cashLetterImagesCount
-	clc.ECEInstitutionName = cl.GetHeader().ECEInstitutionRoutingNumber
+	if cl.CashLetterControl.ECEInstitutionName != "" {
+		clc.ECEInstitutionName = cl.CashLetterControl.ECEInstitutionName
+	} else {
+		clc.ECEInstitutionName = cl.GetHeader().ECEInstitutionRoutingNumber
+	}
 	clc.CreditTotalIndicator = creditIndicator
 	cl.CashLetterControl = clc
 	return nil

checkDetail.go

@@ -6,7 +6,6 @@ package imagecashletter
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 	"unicode/utf8"
 )
@@ -452,7 +451,6 @@ func (cd *CheckDetail) GetImageViewAnalysis() []ImageViewAnalysis {
 
 // SetEceInstitutionItemSequenceNumber sets EceInstitutionItemSequenceNumber
 func (cd *CheckDetail) SetEceInstitutionItemSequenceNumber(seq int) string {
-	itemSequence := strconv.Itoa(seq)
-	cd.EceInstitutionItemSequenceNumber = itemSequence
+	cd.EceInstitutionItemSequenceNumber = cd.numericField(seq, 15)
 	return cd.EceInstitutionItemSequenceNumber
 }

checkDetailAddendumA.go

@@ -6,7 +6,6 @@ package imagecashletter
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 	"time"
 	"unicode/utf8"
@@ -307,7 +306,6 @@ func (cdAddendumA *CheckDetailAddendumA) reservedField() string {
 
 // SetBOFDItemSequenceNumber sets BOFDItemSequenceNumber
 func (cdAddendumA *CheckDetailAddendumA) SetBOFDItemSequenceNumber(seq int) string {
-	itemSequence := strconv.Itoa(seq)
-	cdAddendumA.BOFDItemSequenceNumber = itemSequence
+	cdAddendumA.BOFDItemSequenceNumber = cdAddendumA.numericField(seq, 15)
 	return cdAddendumA.BOFDItemSequenceNumber
 }

file.go

@@ -77,6 +77,16 @@ func (e *FileError) Error() string {
 	return fmt.Sprintf("%s %s", e.FieldName, e.Msg)
 }
 
+// Format standard of X9.37 specification used to parse file
+type Format uint32
+
+const (
+	// Discover format
+	Discover Format = iota
+	//DSTU microformat as defined https://www.frbservices.org/assets/financial-services/check/setup/frb-x937-standards-reference.pdf
+	DSTU
+)
+
 // File is an imagecashletter file
 type File struct {
 	// ID is a client defined string used as a reference to this record
@@ -177,8 +187,6 @@ func (f *File) Create() error {
 	fileTotalRecordCount := 2
 	fileTotalItemCount := 0
 	fileTotalAmount := 0
-	cashLetterRecordCount := 0
-	bundleRecordCount := 0
 	creditIndicator := 0
 
 	// CashLetters
@@ -187,12 +195,11 @@ func (f *File) Create() error {
 		if err := cl.Validate(); err != nil {
 			return err
 		}
-		cashLetterRecordCount = cashLetterRecordCount + 2
-
-		fileTotalItemCount = fileTotalItemCount + len(cl.GetCreditItems())
+		// add 2 for each cashletter header/control
+		fileTotalRecordCount = fileTotalRecordCount + 2
 
 		if len(cl.GetCreditItems()) > 0 {
-			fileTotalItemCount = fileTotalItemCount + len(cl.GetCreditItems())
+			fileTotalRecordCount = fileTotalRecordCount + len(cl.GetCreditItems())
 			creditIndicator = 1
 		}
 
@@ -203,29 +210,32 @@ func (f *File) Create() error {
 				return err
 			}
 
-			bundleRecordCount = bundleRecordCount + 2
+			// add 2 for each bundle header/control
+			fileTotalRecordCount = fileTotalRecordCount + 2
 
 			// Check Items
 			for _, cd := range b.Checks {
-
 				fileTotalItemCount = fileTotalItemCount + 1
-				fileTotalItemCount = fileTotalItemCount + len(cd.CheckDetailAddendumA) + len(cd.CheckDetailAddendumB) + len(cd.CheckDetailAddendumC)
-				fileTotalItemCount = fileTotalItemCount + len(cd.ImageViewDetail) + len(cd.ImageViewData) + len(cd.ImageViewAnalysis)
+
+				fileTotalRecordCount = fileTotalRecordCount + 1
+				fileTotalRecordCount = fileTotalRecordCount + len(cd.CheckDetailAddendumA) + len(cd.CheckDetailAddendumB) + len(cd.CheckDetailAddendumC)
+				fileTotalRecordCount = fileTotalRecordCount + len(cd.ImageViewDetail) + len(cd.ImageViewData) + len(cd.ImageViewAnalysis)
+
 				fileTotalAmount = fileTotalAmount + cd.ItemAmount
 			}
 			// Returns Items
 			for _, rd := range b.Returns {
-
 				fileTotalItemCount = fileTotalItemCount + 1
-				fileTotalItemCount = fileTotalItemCount + len(rd.ReturnDetailAddendumA) + len(rd.ReturnDetailAddendumB) + len(rd.ReturnDetailAddendumC) + len(rd.ReturnDetailAddendumD)
-				fileTotalItemCount = fileTotalItemCount + len(rd.ImageViewDetail) + len(rd.ImageViewData) + len(rd.ImageViewAnalysis)
+
+				fileTotalRecordCount = fileTotalRecordCount + 1
+				fileTotalRecordCount = fileTotalRecordCount + len(rd.ReturnDetailAddendumA) + len(rd.ReturnDetailAddendumB) + len(rd.ReturnDetailAddendumC) + len(rd.ReturnDetailAddendumD)
+				fileTotalRecordCount = fileTotalRecordCount + len(rd.ImageViewDetail) + len(rd.ImageViewData) + len(rd.ImageViewAnalysis)
+
 				fileTotalAmount = fileTotalAmount + rd.ItemAmount
 			}
 		}
 	}
 
-	fileTotalRecordCount = fileTotalRecordCount + cashLetterRecordCount + bundleRecordCount + fileTotalItemCount
-
 	// create FileControl from calculated values
 	fc := NewFileControl()
 	fc.CashLetterCount = fileCashLetterCount

imageViewData.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"strings"
 	"time"
-	"unicode/utf8"
 )
 
 // Errors specific to a ImageViewData Record
@@ -159,7 +158,9 @@ func (ivData *ImageViewData) setRecordType() {
 
 // Parse takes the input record string and parses the ImageViewData values
 func (ivData *ImageViewData) Parse(record string) {
-	if utf8.RuneCountInString(record) < 105 {
+	// record contains binary data that may not map to UTF-8 charmap
+	recordLength := len(record)
+	if recordLength < 105 {
 		return // line too short
 	}
 	// Character position 1-2, Always "52"
@@ -192,7 +193,7 @@ func (ivData *ImageViewData) Parse(record string) {
 	ivData.LengthImageReferenceKey = ivData.parseStringField(record[101:105])
 
 	lirk := ivData.parseNumField(ivData.LengthImageReferenceKey)
-	if lirk < 0 || utf8.RuneCountInString(record) < 110+lirk {
+	if lirk < 0 || recordLength < 110+lirk {
 		return // line too short
 	}
 
@@ -202,7 +203,7 @@ func (ivData *ImageViewData) Parse(record string) {
 	ivData.LengthDigitalSignature = ivData.parseStringField(record[105+lirk : 110+lirk])
 
 	lds := ivData.parseNumField(ivData.LengthDigitalSignature)
-	if lds < 0 || utf8.RuneCountInString(record) < 117+lirk+lds {
+	if lds < 0 || recordLength < 117+lirk+lds {
 		return // line too short
 	}
 	// (111 + lirk) – (110 + lirk + lds)
@@ -211,7 +212,7 @@ func (ivData *ImageViewData) Parse(record string) {
 	ivData.LengthImageData = ivData.parseStringField(record[110+lirk+lds : 117+lirk+lds])
 
 	lid := ivData.parseNumField(ivData.LengthImageData)
-	if lid < 0 || utf8.RuneCountInString(record) < 117+lirk+lds+lid {
+	if lid < 0 || recordLength < 117+lirk+lds+lid {
 		return // line too short
 	}
 	// (118 + lirk + lds) – (117+lirk + lds + lid)

reader.go

@@ -6,6 +6,7 @@ package imagecashletter
 
 import (
 	"bufio"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"strconv"
@@ -32,6 +33,8 @@ type Reader struct {
 	scanner *bufio.Scanner
 	// file is ach.file model being built as r is parsed.
 	File File
+	// format in which to read file
+	format Format
 	// line is the current line being parsed from the input r
 	line string
 	// currentCashLetter is the current CashLetter being parsed
@@ -79,9 +82,48 @@ func NewReader(r io.Reader) *Reader {
 	}
 }
 
+// SetFormat of imagecashletter file
+func (r *Reader) SetFormat(format Format) {
+	switch format {
+	case Discover:
+		r.format = format
+	case DSTU:
+		r.format = format
+		r.scanner.Split(scanVariableLengthLines)
+	}
+}
+
+// ScanVariableLengthLines will return lines from imagecashletter file based on encoded line length
+// This implements bufio.SplitFunc
+func scanVariableLengthLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	// nothing to scan
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+
+	// line length can be variable
+	// use the 4 control bytes at the beginning of a line to determine its length
+	ctrl := data[0:4]
+	dataLen := int(binary.BigEndian.Uint32(ctrl))
+	lineLen := 4 + dataLen
+
+	// the last calculated line is expected to match the remaining bytes
+	if atEOF && lineLen != len(data) {
+		return len(data), data, io.ErrUnexpectedEOF
+	}
+
+	// return line while accounting for control bytes
+	if lineLen <= len(data) {
+		return lineLen, data[4:lineLen], nil
+	}
+
+	// request more data.
+	return 0, nil, nil
+}
+
 // Read reads each line of the imagecashletter file and defines which parser to use based
 // on the first character of each line. It also enforces imagecashletter formatting rules and returns
-// the appropriate error if issues are found.  It supports EBCDIC and ASCII
+// the appropriate error if issues are found.
 func (r *Reader) Read() (File, error) {
 	r.lineNum = 0
 	// read through the entire file

reader_test.go

@@ -6,6 +6,7 @@ package imagecashletter
 
 import (
 	"bytes"
+	"encoding/json"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -69,6 +70,39 @@ func TestICLFile(t *testing.T) {
 	}
 }
 
+func TestICLFile_DSTU(t *testing.T) {
+	fd, err := os.Open(filepath.Join("test", "testdata", "valid-dstu.x937"))
+	if err != nil {
+		t.Fatalf("Can not open local file: %s: \n", err)
+	}
+	defer fd.Close()
+
+	r := NewReader(fd)
+	r.SetFormat(DSTU)
+	ICLFile, err := r.Read()
+	if err != nil {
+		t.Errorf("Issue reading file: %+v \n", err)
+	}
+	t.Logf("r.File.Header=%#v", r.File.Header)
+	t.Logf("r.File.Control=%#v", r.File.Control)
+	// ensure we have a validated file structure
+	if ICLFile.Validate(); err != nil {
+		t.Errorf("Could not validate entire read file: %v", err)
+	}
+
+	actual, err := json.MarshalIndent(ICLFile, "", "    ")
+	if err != nil {
+		t.Errorf("Issue marshaling file: %+v \n", err)
+	}
+	expected, err := ioutil.ReadFile(filepath.Join("test", "testdata", "valid-dstu.json"))
+	if err != nil {
+		t.Errorf("Issue loading validation criteria: %+v \n", err)
+	}
+	if !bytes.Equal(actual, expected) {
+		t.Errorf("Read file does not match expected JSON")
+	}
+}
+
 // TestRecordTypeUnknown validates record type unknown
 func TestRecordTypeUnknown(t *testing.T) {
 	var line = "1735T231380104121042882201809051523NCitadel           Wells Fargo        US     "