Skip to content

Commit 881ab61

Browse files
jf-techjf-tech
authored
Adding flatfile.fixedlength validation routines. (#168)
Most of the heavy-lifting validation is done through JSON schema, but there are some validations have to be done in code, namely those cross property validations (such as min>max, etc) as well as some decl preparation during the validation phase, such as initializing/compiling regexp.
1 parent 7046f31 commit 881ab61

File tree

4 files changed

+233
-4
lines changed

4 files changed

+233
-4
lines changed

‎extensions/omniv21/fileformat/flatfile/fixedlength/validate.go‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package fixedlength
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/jf-tech/go-corelib/caches"
7+
"github.com/jf-tech/go-corelib/strs"
8+
)
9+
10+
type validateCtx struct {
11+
seenTarget bool
12+
}
13+
14+
func (ctx *validateCtx) validateFileDecl(fileDecl *FileDecl) error {
15+
for _, envelopeDecl := range fileDecl.Envelopes {
16+
if err := ctx.validateEnvelopeDecl(envelopeDecl.Name, envelopeDecl); err != nil {
17+
return err
18+
}
19+
}
20+
if !ctx.seenTarget && len(fileDecl.Envelopes) > 0 {
21+
// for easy of use and convenience, if no is_target=true envelope is specified, then
22+
// the first one will be automatically designated as target envelope.
23+
fileDecl.Envelopes[0].IsTarget = true
24+
}
25+
return nil
26+
}
27+
28+
func (ctx *validateCtx) validateEnvelopeDecl(fqdn string, envelopeDecl *EnvelopeDecl) (err error) {
29+
envelopeDecl.fqdn = fqdn
30+
if envelopeDecl.Header != nil {
31+
if envelopeDecl.headerRegexp, err = caches.GetRegex(*envelopeDecl.Header); err != nil {
32+
return fmt.Errorf(
33+
"envelope/envelope_group '%s' has an invalid 'header' regexp '%s': %s",
34+
fqdn, *envelopeDecl.Header, err.Error())
35+
}
36+
}
37+
if envelopeDecl.Footer != nil {
38+
if envelopeDecl.footerRegexp, err = caches.GetRegex(*envelopeDecl.Footer); err != nil {
39+
return fmt.Errorf(
40+
"envelope/envelope_group '%s' has an invalid 'footer' regexp '%s': %s",
41+
fqdn, *envelopeDecl.Footer, err.Error())
42+
}
43+
}
44+
if envelopeDecl.Group() {
45+
if len(envelopeDecl.Columns) > 0 {
46+
return fmt.Errorf("envelope_group '%s' must not have any columns", fqdn)
47+
}
48+
if len(envelopeDecl.Children) <= 0 {
49+
return fmt.Errorf(
50+
"envelope_group '%s' must have at least one child envelope/envelope_group", fqdn)
51+
}
52+
}
53+
if envelopeDecl.Target() {
54+
if ctx.seenTarget {
55+
return fmt.Errorf(
56+
"a second envelope/envelope_group ('%s') with 'is_target' = true is not allowed",
57+
fqdn)
58+
}
59+
ctx.seenTarget = true
60+
}
61+
if envelopeDecl.MinOccurs() > envelopeDecl.MaxOccurs() {
62+
return fmt.Errorf("envelope/envelope_group '%s' has 'min' value %d > 'max' value %d",
63+
fqdn, envelopeDecl.MinOccurs(), envelopeDecl.MaxOccurs())
64+
}
65+
for _, colDecl := range envelopeDecl.Columns {
66+
if err = ctx.validateColumnDecl(fqdn, colDecl); err != nil {
67+
return err
68+
}
69+
}
70+
for _, c := range envelopeDecl.Children {
71+
if err = ctx.validateEnvelopeDecl(strs.BuildFQDN2("/", fqdn, c.Name), c); err != nil {
72+
return err
73+
}
74+
}
75+
envelopeDecl.childRecDecls = toFlatFileRecDecls(envelopeDecl.Children)
76+
return nil
77+
}
78+
79+
func (ctx *validateCtx) validateColumnDecl(fqdn string, colDecl *ColumnDecl) (err error) {
80+
if colDecl.LinePattern != nil {
81+
if colDecl.linePatternRegexp, err = caches.GetRegex(*colDecl.LinePattern); err != nil {
82+
return fmt.Errorf(
83+
"envelope '%s' column '%s' has an invalid 'line_pattern' regexp '%s': %s",
84+
fqdn, colDecl.Name, *colDecl.LinePattern, err.Error())
85+
}
86+
}
87+
return nil
88+
}

‎extensions/omniv21/fileformat/flatfile/fixedlength/validate_test.go‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package fixedlength
2+
3+
import (
4+
"testing"
5+
6+
"github.com/jf-tech/go-corelib/strs"
7+
"github.com/jf-tech/go-corelib/testlib"
8+
"github.com/stretchr/testify/assert"
9+
)
10+
11+
func TestValidateFileDecl_AutoTargetFirstEnvelope(t *testing.T) {
12+
decl := &FileDecl{
13+
Envelopes: []*EnvelopeDecl{
14+
{Name: "A"},
15+
},
16+
}
17+
assert.False(t, decl.Envelopes[0].Target())
18+
err := (&validateCtx{}).validateFileDecl(decl)
19+
assert.NoError(t, err)
20+
assert.True(t, decl.Envelopes[0].Target())
21+
}
22+
23+
func TestValidateFileDecl_InvalidHeaderRegexp(t *testing.T) {
24+
err := (&validateCtx{}).validateFileDecl(&FileDecl{
25+
Envelopes: []*EnvelopeDecl{
26+
{Name: "A", Header: strs.StrPtr("[invalid")},
27+
},
28+
})
29+
assert.Error(t, err)
30+
assert.Equal(t,
31+
"envelope/envelope_group 'A' has an invalid 'header' regexp '[invalid': error parsing regexp: missing closing ]: `[invalid`",
32+
err.Error())
33+
}
34+
35+
func TestValidateFileDecl_InvalidFooterRegexp(t *testing.T) {
36+
err := (&validateCtx{}).validateFileDecl(&FileDecl{
37+
Envelopes: []*EnvelopeDecl{
38+
{Name: "A", Footer: strs.StrPtr("[invalid")},
39+
},
40+
})
41+
assert.Error(t, err)
42+
assert.Equal(t,
43+
"envelope/envelope_group 'A' has an invalid 'footer' regexp '[invalid': error parsing regexp: missing closing ]: `[invalid`",
44+
err.Error())
45+
}
46+
47+
func TestValidateFileDecl_GroupHasColumns(t *testing.T) {
48+
err := (&validateCtx{}).validateFileDecl(&FileDecl{
49+
Envelopes: []*EnvelopeDecl{
50+
{
51+
Name: "A",
52+
Type: strs.StrPtr(typeGroup),
53+
Columns: []*ColumnDecl{{}},
54+
Children: []*EnvelopeDecl{{}},
55+
},
56+
},
57+
})
58+
assert.Error(t, err)
59+
assert.Equal(t, `envelope_group 'A' must not have any columns`, err.Error())
60+
}
61+
62+
func TestValidateFileDecl_GroupHasNoChildren(t *testing.T) {
63+
err := (&validateCtx{}).validateFileDecl(&FileDecl{
64+
Envelopes: []*EnvelopeDecl{
65+
{Name: "A", Type: strs.StrPtr(typeGroup), IsTarget: true},
66+
},
67+
})
68+
assert.Error(t, err)
69+
assert.Equal(t,
70+
`envelope_group 'A' must have at least one child envelope/envelope_group`, err.Error())
71+
}
72+
73+
func TestValidateFileDecl_TwoIsTarget(t *testing.T) {
74+
err := (&validateCtx{}).validateFileDecl(&FileDecl{
75+
Envelopes: []*EnvelopeDecl{
76+
{Name: "A", IsTarget: true},
77+
{Name: "B", Type: strs.StrPtr(typeGroup), Children: []*EnvelopeDecl{
78+
{Name: "C", IsTarget: true},
79+
}},
80+
},
81+
})
82+
assert.Error(t, err)
83+
assert.Equal(t,
84+
`a second envelope/envelope_group ('B/C') with 'is_target' = true is not allowed`,
85+
err.Error())
86+
}
87+
88+
func TestValidateFileDecl_MinGreaterThanMax(t *testing.T) {
89+
err := (&validateCtx{}).validateFileDecl(&FileDecl{
90+
Envelopes: []*EnvelopeDecl{
91+
{Name: "A", Children: []*EnvelopeDecl{
92+
{Name: "B", Min: testlib.IntPtr(2), Max: testlib.IntPtr(1)}}},
93+
},
94+
})
95+
assert.Error(t, err)
96+
assert.Equal(t, `envelope/envelope_group 'A/B' has 'min' value 2 > 'max' value 1`, err.Error())
97+
}
98+
99+
func TestValidateFileDecl_InvalidColumnLinePattern(t *testing.T) {
100+
err := (&validateCtx{}).validateFileDecl(&FileDecl{
101+
Envelopes: []*EnvelopeDecl{
102+
{Name: "A", Columns: []*ColumnDecl{
103+
{Name: "c", LinePattern: strs.StrPtr("[invalid")}}},
104+
},
105+
})
106+
assert.Error(t, err)
107+
assert.Equal(t,
108+
"envelope 'A' column 'c' has an invalid 'line_pattern' regexp '[invalid': error parsing regexp: missing closing ]: `[invalid`",
109+
err.Error())
110+
}
111+
112+
func TestValidateFileDecl_Success(t *testing.T) {
113+
col1 := &ColumnDecl{Name: "c1"}
114+
col2 := &ColumnDecl{Name: "c2"}
115+
col3 := &ColumnDecl{Name: "c3", LinePattern: strs.StrPtr("^C$")}
116+
fd := &FileDecl{
117+
Envelopes: []*EnvelopeDecl{
118+
{
119+
Name: "A",
120+
Header: strs.StrPtr("^A_BEGIN$"),
121+
Footer: strs.StrPtr("^A_END$"),
122+
Children: []*EnvelopeDecl{
123+
{
124+
Name: "B", IsTarget: true,
125+
Columns: []*ColumnDecl{col1, col2, col3},
126+
},
127+
},
128+
},
129+
},
130+
}
131+
err := (&validateCtx{}).validateFileDecl(fd)
132+
assert.NoError(t, err)
133+
assert.Equal(t, "A", fd.Envelopes[0].fqdn)
134+
assert.True(t, fd.Envelopes[0].matchHeader([]byte("A_BEGIN")))
135+
assert.True(t, fd.Envelopes[0].matchFooter([]byte("A_END")))
136+
assert.Equal(t, 1, len(fd.Envelopes[0].childRecDecls))
137+
assert.Same(t, fd.Envelopes[0].Children[0], fd.Envelopes[0].childRecDecls[0].(*EnvelopeDecl))
138+
assert.Equal(t, "A/B", fd.Envelopes[0].Children[0].fqdn)
139+
assert.Equal(t, []*ColumnDecl{col1, col2, col3}, fd.Envelopes[0].Children[0].Columns)
140+
assert.True(t, fd.Envelopes[0].Children[0].Columns[2].lineMatch([]byte("C")))
141+
}

‎extensions/omniv21/validation/fixedlength2FileDeclaration.go‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎extensions/omniv21/validation/fixedlength2FileDeclaration.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
}
4949
}
5050
},
51-
"required": [],
51+
"required": [], "$comment": "yes, 'name' is actually optional",
5252
"additionalProperties": false
5353
},
5454
"envelope_header_footer_based": {
@@ -77,7 +77,7 @@
7777
}
7878
}
7979
},
80-
"required": [ "header" ],
80+
"required": [ "header" ], "$comment": "yes, 'name' is actually optional",
8181
"additionalProperties": false
8282
},
8383
"column_type": {

0 commit comments

Comments
 (0)