Adding line_index to columns of rows based fixed-length envelope; JSON schema adjustment and validation code update. (#169)

Also some comment updates to flatfile.RecReader methods

Author: jf-tech

extensions/omniv21/fileformat/flatfile/fixedlength/decl.go

@@ -13,18 +13,22 @@ import (
 // ColumnDecl describes a column of an envelope.
 type ColumnDecl struct {
 	Name        string  `json:"name,omitempty"`
-	StartPos    int     `json:"start_pos,omitempty"` // 1-based. and rune-based.
-	Length      int     `json:"length,omitempty"`    // rune-based length.
+	StartPos    int     `json:"start_pos,omitempty"`  // 1-based. and rune-based.
+	Length      int     `json:"length,omitempty"`     // rune-based length.
+	LineIndex   *int    `json:"line_index,omitempty"` // 1-based.
 	LinePattern *string `json:"line_pattern,omitempty"`
 
 	linePatternRegexp *regexp.Regexp
 }
 
-func (c *ColumnDecl) lineMatch(line []byte) bool {
-	if c.linePatternRegexp == nil {
-		return true
+func (c *ColumnDecl) lineMatch(lineIndex int, line []byte) bool {
+	if c.LineIndex != nil {
+		return *c.LineIndex == lineIndex+1 // c.LineIndex is 1 based.
+	}
+	if c.linePatternRegexp != nil {
+		return c.linePatternRegexp.Match(line)
 	}
-	return c.linePatternRegexp.Match(line)
+	return true
 }
 
 func (c *ColumnDecl) lineToColumnValue(line []byte) string {
@@ -117,6 +121,9 @@ func (e *EnvelopeDecl) ChildDecls() []flatfile.RecDecl {
 }
 
 func (e *EnvelopeDecl) rowsBased() bool {
+	if e.Group() {
+		panic("envelope_group is neither rows based nor header/footer based")
+	}
 	// for header/footer based envelope, header must be specified; otherwise, it's rows based.
 	return e.Header == nil
 }

extensions/omniv21/fileformat/flatfile/fixedlength/decl_test.go

@@ -12,11 +12,13 @@ import (
 )
 
 func TestColumnDecl_LineMatch(t *testing.T) {
-	assert.True(t, (&ColumnDecl{}).lineMatch([]byte("test")))
+	assert.True(t, (&ColumnDecl{}).lineMatch(0, []byte("test")))
+	assert.False(t, (&ColumnDecl{LineIndex: testlib.IntPtr(2)}).lineMatch(0, []byte("test")))
+	assert.True(t, (&ColumnDecl{LineIndex: testlib.IntPtr(2)}).lineMatch(1, []byte("test")))
 	assert.False(t, (&ColumnDecl{linePatternRegexp: regexp.MustCompile("^ABC.*$")}).
-		lineMatch([]byte("test")))
+		lineMatch(0, []byte("test")))
 	assert.True(t, (&ColumnDecl{linePatternRegexp: regexp.MustCompile("^ABC.*$")}).
-		lineMatch([]byte("ABCDEFG")))
+		lineMatch(0, []byte("ABCDEFG")))
 }
 
 func TestColumnDecl_LineToColumnValue(t *testing.T) {
@@ -64,6 +66,9 @@ func TestEnvelopeDecl(t *testing.T) {
 	assert.Equal(t, e.childRecDecls, e.ChildDecls())
 
 	// rowsBased()
+	assert.PanicsWithValue(t, "envelope_group is neither rows based nor header/footer based",
+		func() { e.rowsBased() })
+	e.Type = strs.StrPtr(typeEnvelope)
 	assert.True(t, e.rowsBased())
 	e.Header = strs.StrPtr("^ABC$")
 	assert.False(t, e.rowsBased())

extensions/omniv21/fileformat/flatfile/fixedlength/validate.go

@@ -77,6 +77,11 @@ func (ctx *validateCtx) validateEnvelopeDecl(fqdn string, envelopeDecl *Envelope
 }
 
 func (ctx *validateCtx) validateColumnDecl(fqdn string, colDecl *ColumnDecl) (err error) {
+	if colDecl.LineIndex != nil && colDecl.LinePattern != nil {
+		return fmt.Errorf(
+			"envelope '%s' column '%s' cannot have both `line_index` and `line_pattern` specified at the same time",
+			fqdn, colDecl.Name)
+	}
 	if colDecl.LinePattern != nil {
 		if colDecl.linePatternRegexp, err = caches.GetRegex(*colDecl.LinePattern); err != nil {
 			return fmt.Errorf(

extensions/omniv21/fileformat/flatfile/fixedlength/validate_test.go

@@ -96,6 +96,19 @@ func TestValidateFileDecl_MinGreaterThanMax(t *testing.T) {
 	assert.Equal(t, `envelope/envelope_group 'A/B' has 'min' value 2 > 'max' value 1`, err.Error())
 }
 
+func TestValidateFileDecl_ColumnLineIndexAndLinePatternSameTime(t *testing.T) {
+	err := (&validateCtx{}).validateFileDecl(&FileDecl{
+		Envelopes: []*EnvelopeDecl{
+			{Name: "A", Columns: []*ColumnDecl{
+				{Name: "c", LineIndex: testlib.IntPtr(2), LinePattern: strs.StrPtr(".")}}},
+		},
+	})
+	assert.Error(t, err)
+	assert.Equal(t,
+		"envelope 'A' column 'c' cannot have both `line_index` and `line_pattern` specified at the same time",
+		err.Error())
+}
+
 func TestValidateFileDecl_InvalidColumnLinePattern(t *testing.T) {
 	err := (&validateCtx{}).validateFileDecl(&FileDecl{
 		Envelopes: []*EnvelopeDecl{
@@ -110,7 +123,7 @@ func TestValidateFileDecl_InvalidColumnLinePattern(t *testing.T) {
 }
 
 func TestValidateFileDecl_Success(t *testing.T) {
-	col1 := &ColumnDecl{Name: "c1"}
+	col1 := &ColumnDecl{Name: "c1", LineIndex: testlib.IntPtr(1)}
 	col2 := &ColumnDecl{Name: "c2"}
 	col3 := &ColumnDecl{Name: "c3", LinePattern: strs.StrPtr("^C$")}
 	fd := &FileDecl{
@@ -137,5 +150,6 @@ func TestValidateFileDecl_Success(t *testing.T) {
 	assert.Same(t, fd.Envelopes[0].Children[0], fd.Envelopes[0].childRecDecls[0].(*EnvelopeDecl))
 	assert.Equal(t, "A/B", fd.Envelopes[0].Children[0].fqdn)
 	assert.Equal(t, []*ColumnDecl{col1, col2, col3}, fd.Envelopes[0].Children[0].Columns)
-	assert.True(t, fd.Envelopes[0].Children[0].Columns[2].lineMatch([]byte("C")))
+	assert.True(t, fd.Envelopes[0].Children[0].Columns[0].lineMatch(0, []byte("C")))
+	assert.True(t, fd.Envelopes[0].Children[0].Columns[2].lineMatch(0, []byte("C")))
 }

extensions/omniv21/fileformat/flatfile/hierarchyReader.go

@@ -132,16 +132,18 @@ func (r *HierarchyReader) Release(n *idr.Node) {
 
 // readRec tries to read/match unprocessed data against the passed-in record decl.
 func (r *HierarchyReader) readRec(recDecl RecDecl) (*idr.Node, error) {
-	// If the decl is a solid non-group decl, then we will ask RecReader to match and create IDR.
-	// If the decl is a group, then we'll only ask RecReader to match but not creating IDR - instead
-	// we'll create an IDR node for the group decl here.
+	// If the decl is a Group(), the matching should be using the recursive algorithm
+	// to match the first-heir-in-line non-group descendent decl. If matched, the returned
+	// IDR node should be of this group node. This logic is similar to the EDI segment
+	// matching logic, essentially a greedy algo:
+	// https://github.com/jf-tech/omniparser/blob/6802ed98d0e5325a6908ebbc6d2da0e4655ed125/extensions/omniv21/fileformat/edi/seg.go#L87
 	nonGroupDecl := recDecl
 	for nonGroupDecl.Group() && len(nonGroupDecl.ChildDecls()) > 0 {
 		nonGroupDecl = nonGroupDecl.ChildDecls()[0]
 	}
 	if nonGroupDecl.Group() {
-		// We must have a non-group solid record to perform the match; if not, it's a no
-		// match, thus returning nil for IDR and nil for error.
+		// We must have a non-group solid record to perform the match; if not, return
+		// nil for IDR, indicating no match, and nil for error.
 		return nil, nil
 	}
 	// Now we have a solid record to perform match, let's call RecReader to do that.

extensions/omniv21/fileformat/flatfile/recreader.go

@@ -20,17 +20,11 @@ type RecReader interface {
 	//   return (false, nil).
 	MoreUnprocessedData() (more bool, err error)
 
-	// ReadAndMatch matches the passed-in RecDecl to unprocessed data and creates a
+	// ReadAndMatch matches the passed-in *non-group* RecDecl to unprocessed data and creates a
 	// corresponding IDR node if data matches and createIDR flag is turned on.
 	// Implementation notes:
 	// - If io.EOF is encountered while there is still unmatched thus unprocessed data,
 	//   io.EOF shouldn't be returned.
 	// - If a non io.EOF error encountered during IO, return (false, nil, err).
-	// - If the decl is a Group(), the matching should be using the recursive algorithm
-	//   to match the first-heir-in-line non-group descendent decl. If matched, the returned
-	//   IDR node should be of this group node, and the actual matched record data should be
-	//   internally cached for the next call(s). This logic is similar to the EDI segment
-	//   matching logic:
-	//   https://github.com/jf-tech/omniparser/blob/6802ed98d0e5325a6908ebbc6d2da0e4655ed125/extensions/omniv21/fileformat/edi/seg.go#L87
 	ReadAndMatch(decl RecDecl, createIDR bool) (matched bool, node *idr.Node, err error)
 }