Adding flatfile.fixedlength.EnvelopeDecl and its ColumnDecl (#166)

ColumnDecl is mostly similar to the existing fixed-length implementation
(https://github.com/jf-tech/omniparser/blob/d8e230aa07673b9ce7b933419c4d528b1796e56f/extensions/omniv21/fileformat/fixedlength/decl.go#L17)

EnvelopeDecl is mostly different, going after the new flatfile.RecDecl model with Type/Target/Min/Max/etc introduced.

However, no panic. With the new decl, the final resulting schema (to be coming soon) will by and large be (very very) similar
to existing fixed length schemas, with only a couple of minor/trivial tweaks for existing complex use cases and with no
changes at all for simple/common use cases.

Author: jf-tech

extensions/omniv21/fileformat/flatfile/fixedlength/decl.go

@@ -0,0 +1,166 @@
+package fixedlength
+
+import (
+	"fmt"
+	"regexp"
+	"unicode/utf8"
+
+	"github.com/jf-tech/go-corelib/maths"
+
+	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/flatfile"
+)
+
+// ColumnDecl describes a column of an envelope.
+type ColumnDecl struct {
+	Name        string  `json:"name,omitempty"`
+	StartPos    int     `json:"start_pos,omitempty"` // 1-based. and rune-based.
+	Length      int     `json:"length,omitempty"`    // rune-based length.
+	LinePattern *string `json:"line_pattern,omitempty"`
+
+	linePatternRegexp *regexp.Regexp
+}
+
+func (c *ColumnDecl) lineMatch(line []byte) bool {
+	if c.linePatternRegexp == nil {
+		return true
+	}
+	return c.linePatternRegexp.Match(line)
+}
+
+func (c *ColumnDecl) lineToColumnValue(line []byte) string {
+	// StartPos is 1-based and its value >= 1 guaranteed by json schema validation done earlier.
+	start := c.StartPos - 1
+	// First chop off the prefix prior to c.StartPos
+	for start > 0 && len(line) > 0 {
+		_, adv := utf8.DecodeRune(line)
+		line = line[adv:]
+		start--
+	}
+	// Then from that position, count c.Length runes and that's the string value we need.
+	// Note if c.Length is longer than what's left in the line, we'll simply take all of
+	// the remaining line (and no error here, since we haven't yet seen a useful case where
+	// we need to be excessively strict.)
+	lenCount := c.Length
+	i := 0
+	for lenCount > 0 && i < len(line) {
+		_, adv := utf8.DecodeRune(line[i:])
+		i += adv
+		lenCount--
+	}
+	return string(line[:i])
+}
+
+const (
+	typeEnvelope = "envelope"
+	typeGroup    = "envelope_group"
+)
+
+// EnvelopeDecl describes an envelope of a fixed-length input.
+// If Rows/Header/Footer are all nil, then it defaults to Rows = 1.
+// If Rows specified, then Header/Footer must be nil. (JSON schema validation will ensure this.)
+// If Header is specified, Rows must be nil. (JSON schema validation will ensure this.)
+// Footer is optional; If not specified, Header will be used for a single-line envelope matching.
+type EnvelopeDecl struct {
+	Name     string          `json:"name,omitempty"`
+	Rows     *int            `json:"rows,omitempty"`
+	Header   *string         `json:"header,omitempty"`
+	Footer   *string         `json:"footer,omitempty"`
+	Type     *string         `json:"type,omitempty"`
+	IsTarget bool            `json:"is_target,omitempty"`
+	Min      *int            `json:"min,omitempty"`
+	Max      *int            `json:"max,omitempty"`
+	Columns  []*ColumnDecl   `json:"columns,omitempty"`
+	Children []*EnvelopeDecl `json:"child_envelopes,omitempty"`
+
+	fqdn          string // fullly hierarchical name to the envelope.
+	childRecDecls []flatfile.RecDecl
+	headerRegexp  *regexp.Regexp
+	footerRegexp  *regexp.Regexp
+}
+
+func (e *EnvelopeDecl) DeclName() string {
+	return e.Name
+}
+
+func (e *EnvelopeDecl) Target() bool {
+	return e.IsTarget
+}
+
+func (e *EnvelopeDecl) Group() bool {
+	return e.Type != nil && *e.Type == typeGroup
+}
+
+// MinOccurs defaults to 0. Fixed-length input most common scenario is min=0/max=unbounded.
+func (e *EnvelopeDecl) MinOccurs() int {
+	switch e.Min {
+	case nil:
+		return 0
+	default:
+		return *e.Min
+	}
+}
+
+// MaxOccurs defaults to unbounded. Fixed-length input most common scenario is min=0/max=unbounded.
+func (e *EnvelopeDecl) MaxOccurs() int {
+	switch {
+	case e.Max == nil:
+		fallthrough
+	case *e.Max < 0:
+		return maths.MaxIntValue
+	default:
+		return *e.Max
+	}
+}
+
+func (e *EnvelopeDecl) ChildDecls() []flatfile.RecDecl {
+	return e.childRecDecls
+}
+
+func (e *EnvelopeDecl) rowsBased() bool {
+	// for header/footer based envelope, header must be specified; otherwise, it's rows based.
+	return e.Header == nil
+}
+
+// rows() defaults to 1. Fixed-length input most common scenario is rows-based single line envelope.
+func (e *EnvelopeDecl) rows() int {
+	if !e.rowsBased() {
+		panic(fmt.Sprintf("envelope '%s' is not rows based", e.fqdn))
+	}
+	if e.Rows == nil {
+		return 1
+	}
+	return *e.Rows
+}
+
+func (e *EnvelopeDecl) matchHeader(line []byte) bool {
+	if e.headerRegexp == nil {
+		panic(fmt.Sprintf("envelope '%s' is not header/footer based", e.fqdn))
+	}
+	return e.headerRegexp.Match(line)
+}
+
+// Footer is optional. If not specified, it always matches. Thus for a header/footer envelope,
+// if the footer isn't specified, it effectively becomes a single-row envelope matched by header,
+// given that after the header matches a line, matchFooter is called on the same line.
+func (e *EnvelopeDecl) matchFooter(line []byte) bool {
+	if e.footerRegexp == nil {
+		return true
+	}
+	return e.footerRegexp.Match(line)
+}
+
+func toFlatFileRecDecls(es []*EnvelopeDecl) []flatfile.RecDecl {
+	if len(es) == 0 {
+		return nil
+	}
+	ret := make([]flatfile.RecDecl, len(es))
+	for i, d := range es {
+		ret[i] = d
+	}
+	return ret
+}
+
+// FileDecl describes fixed-length schema `file_declaration` setting.
+type FileDecl struct {
+	Envelopes []*EnvelopeDecl `json:"envelopes,omitempty"`
+}

extensions/omniv21/fileformat/flatfile/fixedlength/decl_test.go

@@ -0,0 +1,103 @@
+package fixedlength
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/jf-tech/go-corelib/maths"
+	"github.com/jf-tech/go-corelib/strs"
+	"github.com/jf-tech/go-corelib/testlib"
+	"github.com/jf-tech/omniparser/extensions/omniv21/fileformat/flatfile"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestColumnDecl_LineMatch(t *testing.T) {
+	assert.True(t, (&ColumnDecl{}).lineMatch([]byte("test")))
+	assert.False(t, (&ColumnDecl{linePatternRegexp: regexp.MustCompile("^ABC.*$")}).
+		lineMatch([]byte("test")))
+	assert.True(t, (&ColumnDecl{linePatternRegexp: regexp.MustCompile("^ABC.*$")}).
+		lineMatch([]byte("ABCDEFG")))
+}
+
+func TestColumnDecl_LineToColumnValue(t *testing.T) {
+	decl := func(start, length int) *ColumnDecl {
+		return &ColumnDecl{StartPos: start, Length: length}
+	}
+	assert.Equal(t, "", decl(10, 4).lineToColumnValue([]byte("test")))   // fully out of range
+	assert.Equal(t, "st", decl(3, 4).lineToColumnValue([]byte("test")))  // partially out of range
+	assert.Equal(t, "tes", decl(1, 3).lineToColumnValue([]byte("test"))) // fully in range
+}
+
+func TestEnvelopeDecl(t *testing.T) {
+	// DeclName()
+	e := &EnvelopeDecl{Name: "e1"}
+	assert.Equal(t, "e1", e.DeclName())
+	e.fqdn = e.DeclName()
+
+	// Target()
+	assert.False(t, e.Target())
+	e.IsTarget = true
+	assert.True(t, e.Target())
+
+	// Group()
+	assert.False(t, e.Group())
+	e.Type = strs.StrPtr(typeEnvelope)
+	assert.False(t, e.Group())
+	e.Type = strs.StrPtr(typeGroup)
+	assert.True(t, e.Group())
+
+	// MinOccurs()
+	assert.Equal(t, 0, e.MinOccurs())
+	e.Min = testlib.IntPtr(42)
+	assert.Equal(t, 42, e.MinOccurs())
+
+	// MaxOccurs()
+	assert.Equal(t, maths.MaxIntValue, e.MaxOccurs())
+	e.Max = testlib.IntPtr(-1)
+	assert.Equal(t, maths.MaxIntValue, e.MaxOccurs())
+	e.Max = testlib.IntPtr(42)
+	assert.Equal(t, 42, e.MaxOccurs())
+
+	// ChildDecls()
+	assert.Nil(t, e.ChildDecls())
+	e.childRecDecls = []flatfile.RecDecl{}
+	assert.Equal(t, e.childRecDecls, e.ChildDecls())
+
+	// rowsBased()
+	assert.True(t, e.rowsBased())
+	e.Header = strs.StrPtr("^ABC$")
+	assert.False(t, e.rowsBased())
+
+	// rows()
+	assert.PanicsWithValue(t, "envelope 'e1' is not rows based", func() { e.rows() })
+	e.Header = nil
+	assert.Equal(t, 1, e.rows())
+	e.Rows = testlib.IntPtr(42)
+	assert.Equal(t, 42, e.rows())
+
+	// matchHeader()
+	assert.PanicsWithValue(
+		t, "envelope 'e1' is not header/footer based", func() { e.matchHeader(nil) })
+	e.headerRegexp = regexp.MustCompile("^ABC$")
+	assert.False(t, e.matchHeader([]byte("ABCD")))
+	assert.True(t, e.matchHeader([]byte("ABC")))
+
+	// matchFooter()
+	assert.True(t, e.matchFooter([]byte("ABCD")))
+	e.footerRegexp = regexp.MustCompile("^ABC$")
+	assert.False(t, e.matchFooter([]byte("ABCD")))
+	assert.True(t, e.matchFooter([]byte("ABC")))
+}
+
+func TestToFlatFileRecDecls(t *testing.T) {
+	assert.Nil(t, toFlatFileRecDecls(nil))
+	assert.Nil(t, toFlatFileRecDecls([]*EnvelopeDecl{}))
+	es := []*EnvelopeDecl{
+		{},
+		{},
+	}
+	ds := toFlatFileRecDecls(es)
+	for i := range ds {
+		assert.Same(t, es[i], ds[i].(*EnvelopeDecl))
+	}
+}