From 150788956633f8e0ddf9bb310b3a11e441f90421 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Mon, 12 Jul 2021 17:14:31 +0530 Subject: [PATCH 1/5] Add yamlgen Signed-off-by: Saswata Mukherjee --- go.mod | 3 +- go.sum | 2 + pkg/mdformatter/mdgen/mdgen.go | 41 ++++++ pkg/yamlgen/yamlgen.go | 221 +++++++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 pkg/yamlgen/yamlgen.go diff --git a/go.mod b/go.mod index cea5fdd..5f1c563 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Kunde21/markdownfmt/v3 v3.1.0 github.com/alecthomas/kingpin/v2 v2.3.2 github.com/charmbracelet/glamour v0.6.0 + github.com/dave/jennifer v1.4.1 github.com/efficientgo/core v1.0.0-rc.2 github.com/efficientgo/tools/extkingpin v0.0.0-20230505153745-6b7392939a60 github.com/fatih/structtag v1.2.0 @@ -18,6 +19,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 github.com/mattn/go-sqlite3 v1.14.17 github.com/oklog/run v1.1.0 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 github.com/prometheus/common v0.42.0 github.com/sergi/go-diff v1.0.0 @@ -64,7 +66,6 @@ require ( github.com/niklasfasching/go-org v1.6.5 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect diff --git a/go.sum b/go.sum index f434a4c..ebc2ea6 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= +github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/pkg/mdformatter/mdgen/mdgen.go b/pkg/mdformatter/mdgen/mdgen.go index 38676d5..47b84d7 100644 --- a/pkg/mdformatter/mdgen/mdgen.go +++ b/pkg/mdformatter/mdgen/mdgen.go @@ -6,17 +6,20 @@ package mdgen import ( "bytes" "fmt" + "io/ioutil" "os/exec" "strconv" "strings" "github.com/bwplotka/mdox/pkg/mdformatter" + "github.com/bwplotka/mdox/pkg/yamlgen" "github.com/mattn/go-shellwords" ) const ( infoStringKeyExec = "mdox-exec" infoStringKeyExitCode = "mdox-expect-exit-code" + infoStringKeyGoStruct = "mdox-gen-go-struct" ) var ( @@ -58,6 +61,11 @@ func (t *genCodeBlockTransformer) TransformCodeBlock(ctx mdformatter.SourceConte return nil, fmt.Errorf("got %q without variable. Expected format is e.g ```yaml %s=\"\" but got %s", val[0], infoStringKeyExitCode, string(infoString)) } infoStringAttr[val[0]] = val[1] + case infoStringKeyGoStruct: + if len(val) != 2 { + return nil, fmt.Errorf("got %q without variable. Expected format is e.g ```yaml %s=\"\" but got %s", val[0], infoStringKeyGoStruct, string(infoString)) + } + infoStringAttr[val[0]] = val[1] } } @@ -98,6 +106,39 @@ func (t *genCodeBlockTransformer) TransformCodeBlock(ctx mdformatter.SourceConte return output, nil } + if fileWithStruct, ok := infoStringAttr[infoStringKeyGoStruct]; ok { + // This is like mdox-gen-go-struct=:structname for now. + fs := strings.Split(fileWithStruct, ":") + src, err := ioutil.ReadFile(fs[0]) + if err != nil { + return nil, fmt.Errorf("read file for yaml gen %v: %w", fs[0], err) + } + + generatedCode, err := yamlgen.GenGoCode(src) + if err != nil { + return nil, fmt.Errorf("generate code for yaml gen %v: %w", fs[0], err) + } + + b, err := yamlgen.ExecGoCode(ctx, generatedCode) + if err != nil { + return nil, fmt.Errorf("execute generated code for yaml gen %v: %w", fs[0], err) + } + + // TODO(saswatamcode): This feels sort of hacky, need better way of printing. + // Remove `---` and check struct name. + yamls := bytes.Split(b, []byte("---")) + for _, yaml := range yamls { + lines := bytes.Split(yaml, []byte("\n")) + if len(lines) > 1 { + if string(lines[1]) == fs[1] { + ret := bytes.Join(lines[2:len(lines)-1], []byte("\n")) + ret = append(ret, []byte("\n")...) + return ret, nil + } + } + } + } + panic("should never get here") } diff --git a/pkg/yamlgen/yamlgen.go b/pkg/yamlgen/yamlgen.go new file mode 100644 index 0000000..93f8d5e --- /dev/null +++ b/pkg/yamlgen/yamlgen.go @@ -0,0 +1,221 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +package yamlgen + +import ( + "bytes" + "context" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + "github.com/dave/jennifer/jen" + "github.com/pkg/errors" +) + +// TODO(saswatamcode): Add tests. +// TODO(saswatamcode): Check jennifer code for some safety. +// TODO(saswatamcode): Add mechanism for caching output from generated code. +// TODO(saswatamcode): Currently takes file names, need to make it module based(something such as https://golang.org/pkg/cmd/go/internal/list/). + +// GenGoCode generates Go code for yaml gen from structs in src file. +func GenGoCode(src []byte) (string, error) { + // Create new main file. + fset := token.NewFileSet() + generatedCode := jen.NewFile("main") + + // Parse source file. + f, err := parser.ParseFile(fset, "", src, parser.AllErrors) + if err != nil { + return "", err + } + + // Add imports if needed(will not be used if not required in code). + for _, s := range f.Imports { + generatedCode.ImportName(s.Path.Value[1:len(s.Path.Value)-1], "") + } + + // Init statements for structs. + var init []jen.Code + // Declare config map, i.e, `configs := map[string]interface{}{}`. + init = append(init, jen.Id("configs").Op(":=").Map(jen.String()).Interface().Values()) + + // Loop through declarations in file. + for _, decl := range f.Decls { + // Cast to generic declaration node. + if genericDecl, ok := decl.(*ast.GenDecl); ok { + // Check if declaration spec is `type`. + if typeDecl, ok := genericDecl.Specs[0].(*ast.TypeSpec); ok { + var structFields []jen.Code + // Cast to `type struct`. + structDecl := typeDecl.Type.(*ast.StructType) + fields := structDecl.Fields.List + // Loop and generate fields for each field. + for _, field := range fields { + // Each field might have multiple names. + names := field.Names + for _, n := range names { + pos := n.Obj.Decl.(*ast.Field) + structFields = append(structFields, jen.Id(n.Name).Id(string(src[pos.Type.Pos()-1:pos.Type.End()-1])).Id(string(src[pos.Tag.Pos()-1:pos.Tag.End()-1]))) + } + } + + // Add initialize statements for struct. + init = append(init, jen.Id("configs").Index(jen.Lit(typeDecl.Name.Name)).Op("=").Id(typeDecl.Name.Name+"{}")) + // Finally put struct inside generated code. + generatedCode.Type().Id(typeDecl.Name.Name).Struct(structFields...) + } + } + } + + // Add for loop to iterate through map and return config YAML. + init = append(init, jen.For( + jen.List(jen.Id("k"), jen.Id("config")).Op(":=").Range().Id("configs"), + ).Block( + // We import the cfggen Generate method directly to generate output. + jen.Qual("fmt", "Println").Call(jen.Lit("---")), + jen.Qual("fmt", "Println").Call(jen.Id("k")), + // TODO(saswatamcode): Replace with import from mdox itself once merged. + // jen.Qual("github.com/bwplotka/mdox/yamlgen", "Generate").Call(jen.Id("config"), jen.Qual("os", "Stderr")), + jen.Qual("structgen/cfggen", "Generate").Call(jen.Id("config"), jen.Qual("os", "Stderr")), + )) + + // Generate main function in new module. + generatedCode.Func().Id("main").Params().Block( + init..., + ) + return fmt.Sprintf("%#v", generatedCode), nil +} + +// execGoCode executes and returns output from generated Go code. +func ExecGoCode(ctx context.Context, mainGo string) ([]byte, error) { + tmpDir, err := ioutil.TempDir("", "structgen") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) + + // TODO(saswatamcode): Remove once merged. + // This is weird but need it for getting Generate function inside tmpDir in PR. + // Once merged this can be removed and can be replaced with just an import in tmpDir/main.go. + err = os.Mkdir(filepath.Join(tmpDir, "cfggen"), 0700) + if err != nil { + return nil, err + } + code, err := os.Create(filepath.Join(tmpDir, "cfggen/cfggen.go")) + if err != nil { + return nil, err + } + defer code.Close() + _, err = code.Write([]byte(cfggenFile)) + if err != nil { + return nil, err + } + + // Copy generated code to main.go. + main, err := os.Create(filepath.Join(tmpDir, "main.go")) + if err != nil { + return nil, err + } + defer main.Close() + + _, err = main.Write([]byte(mainGo)) + if err != nil { + return nil, err + } + + // Create go.mod in temp dir. + cmd := exec.CommandContext(ctx, "go", "mod", "init", "structgen") + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "run %v", cmd) + } + + // Import required packages(generate go.sum). + cmd = exec.CommandContext(ctx, "go", "mod", "tidy") + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "run %v", cmd) + } + + // Execute generate code and return output. + b := bytes.Buffer{} + cmd = exec.CommandContext(ctx, "go", "run", "main.go") + cmd.Dir = tmpDir + cmd.Stderr = &b + cmd.Stdout = &b + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "run %v out %v", cmd, b.String()) + } + + return b.Bytes(), nil +} + +// TODO(saswatamcode): Remove once merged. +// This is weird but need it for getting Generate function inside tmpDir in PR. +// Could also do with commit hash and go.mod, but it would change on each commit in PR). +// Once merged this can be removed and can be replaced with just an import in tmpDir/main.go. +const cfggenFile = `package cfggen + +import ( + "io" + "reflect" + + "github.com/fatih/structtag" + "github.com/pkg/errors" + "gopkg.in/yaml.v3" +) + +func Generate(obj interface{}, w io.Writer) error { + // We forbid omitempty option. This is for simplification for doc generation. + if err := checkForOmitEmptyTagOption(obj); err != nil { + return errors.Wrap(err, "invalid type") + } + return yaml.NewEncoder(w).Encode(obj) +} + +func checkForOmitEmptyTagOption(obj interface{}) error { + return checkForOmitEmptyTagOptionRec(reflect.ValueOf(obj)) +} + +func checkForOmitEmptyTagOptionRec(v reflect.Value) error { + switch v.Kind() { + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + tags, err := structtag.Parse(string(v.Type().Field(i).Tag)) + if err != nil { + return errors.Wrapf(err, "%s: failed to parse tag %q", v.Type().Field(i).Name, v.Type().Field(i).Tag) + } + + tag, err := tags.Get("yaml") + if err != nil { + return errors.Wrapf(err, "%s: failed to get tag %q", v.Type().Field(i).Name, v.Type().Field(i).Tag) + } + + for _, opts := range tag.Options { + if opts == "omitempty" { + return errors.Errorf("omitempty is forbidden for config, but spotted on field '%s'", v.Type().Field(i).Name) + } + } + + if err := checkForOmitEmptyTagOptionRec(v.Field(i)); err != nil { + return errors.Wrapf(err, "%s", v.Type().Field(i).Name) + } + } + + case reflect.Ptr: + return errors.New("nil pointers are not allowed in configuration") + + case reflect.Interface: + return checkForOmitEmptyTagOptionRec(v.Elem()) + } + + return nil +} +` From 10983db4acddbb5f36ebdb1873f84d675fcbb0c0 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Tue, 20 Jul 2021 10:27:31 +0530 Subject: [PATCH 2/5] Add support for array in struct Signed-off-by: Saswata Mukherjee --- pkg/yamlgen/yamlgen.go | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/pkg/yamlgen/yamlgen.go b/pkg/yamlgen/yamlgen.go index 93f8d5e..0787ae0 100644 --- a/pkg/yamlgen/yamlgen.go +++ b/pkg/yamlgen/yamlgen.go @@ -8,8 +8,10 @@ import ( "context" "fmt" "go/ast" + "go/importer" "go/parser" "go/token" + "go/types" "io/ioutil" "os" "os/exec" @@ -56,18 +58,42 @@ func GenGoCode(src []byte) (string, error) { // Cast to `type struct`. structDecl := typeDecl.Type.(*ast.StructType) fields := structDecl.Fields.List + arrayInit := make(jen.Dict) + // Loop and generate fields for each field. for _, field := range fields { // Each field might have multiple names. names := field.Names for _, n := range names { - pos := n.Obj.Decl.(*ast.Field) - structFields = append(structFields, jen.Id(n.Name).Id(string(src[pos.Type.Pos()-1:pos.Type.End()-1])).Id(string(src[pos.Tag.Pos()-1:pos.Tag.End()-1]))) + if n.IsExported() { + pos := n.Obj.Decl.(*ast.Field) + + // Make type map to check if field is array. + info := types.Info{Types: make(map[ast.Expr]types.TypeAndValue)} + _, err = (&types.Config{Importer: importer.ForCompiler(fset, "source", nil)}).Check("mypkg", fset, []*ast.File{f}, &info) + if err != nil { + return "", err + } + typ := info.Types[field.Type].Type + + switch typ.(type) { + case *types.Slice: + // Field is of type array so initialize it using code like `[]Type{Type{}}`. + arrayInit[jen.Id(n.Name)] = jen.Id(string(src[pos.Type.Pos()-1 : pos.Type.End()-1])).Values(jen.Id(string(src[pos.Type.Pos()+1 : pos.Type.End()-1])).Values()) + default: + } + // Copy struct field to generated code. + if pos.Tag != nil { + structFields = append(structFields, jen.Id(n.Name).Id(string(src[pos.Type.Pos()-1:pos.Type.End()-1])).Id(pos.Tag.Value)) + } + } } } - // Add initialize statements for struct. - init = append(init, jen.Id("configs").Index(jen.Lit(typeDecl.Name.Name)).Op("=").Id(typeDecl.Name.Name+"{}")) + // Add initialize statements for struct via code like `configs["Type"] = Type{}`. + // If struct has array members, use array initializer via code like `configs["Config"] = Config{ArrayMember: []Type{Type{}}}`. + init = append(init, jen.Id("configs").Index(jen.Lit(typeDecl.Name.Name)).Op("=").Id(typeDecl.Name.Name).Values(arrayInit)) + // Finally put struct inside generated code. generatedCode.Type().Id(typeDecl.Name.Name).Struct(structFields...) } From f6511ac279ea14e76413043d537b20dbba1aefc9 Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Sun, 1 Aug 2021 12:02:58 +0530 Subject: [PATCH 3/5] Add support for string types Signed-off-by: Saswata Mukherjee --- pkg/yamlgen/yamlgen.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pkg/yamlgen/yamlgen.go b/pkg/yamlgen/yamlgen.go index 0787ae0..db4dd1a 100644 --- a/pkg/yamlgen/yamlgen.go +++ b/pkg/yamlgen/yamlgen.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "go/ast" - "go/importer" "go/parser" "go/token" "go/types" @@ -16,6 +15,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "github.com/dave/jennifer/jen" "github.com/pkg/errors" @@ -56,7 +56,11 @@ func GenGoCode(src []byte) (string, error) { if typeDecl, ok := genericDecl.Specs[0].(*ast.TypeSpec); ok { var structFields []jen.Code // Cast to `type struct`. - structDecl := typeDecl.Type.(*ast.StructType) + structDecl, ok := typeDecl.Type.(*ast.StructType) + if !ok { + generatedCode.Type().Id(typeDecl.Name.Name).Id(string(src[typeDecl.Type.Pos()-1 : typeDecl.Type.End()-1])) + continue + } fields := structDecl.Fields.List arrayInit := make(jen.Dict) @@ -68,20 +72,12 @@ func GenGoCode(src []byte) (string, error) { if n.IsExported() { pos := n.Obj.Decl.(*ast.Field) - // Make type map to check if field is array. - info := types.Info{Types: make(map[ast.Expr]types.TypeAndValue)} - _, err = (&types.Config{Importer: importer.ForCompiler(fset, "source", nil)}).Check("mypkg", fset, []*ast.File{f}, &info) - if err != nil { - return "", err - } - typ := info.Types[field.Type].Type - - switch typ.(type) { - case *types.Slice: - // Field is of type array so initialize it using code like `[]Type{Type{}}`. + // Check if field is a slice type. + sliceRe := regexp.MustCompile(`.*\[.*\].*`) + if sliceRe.MatchString(types.ExprString(field.Type)) { arrayInit[jen.Id(n.Name)] = jen.Id(string(src[pos.Type.Pos()-1 : pos.Type.End()-1])).Values(jen.Id(string(src[pos.Type.Pos()+1 : pos.Type.End()-1])).Values()) - default: } + // Copy struct field to generated code. if pos.Tag != nil { structFields = append(structFields, jen.Id(n.Name).Id(string(src[pos.Type.Pos()-1:pos.Type.End()-1])).Id(pos.Tag.Value)) From aa0018eb4f3fd08299cc19630144562a44b4d52c Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Mon, 23 Aug 2021 13:35:41 +0530 Subject: [PATCH 4/5] Implement suggestions; Change cfggen Signed-off-by: Saswata Mukherjee --- pkg/mdformatter/linktransformer/config.go | 2 +- pkg/yamlgen/yamlgen.go | 99 +++-------------------- 2 files changed, 13 insertions(+), 88 deletions(-) diff --git a/pkg/mdformatter/linktransformer/config.go b/pkg/mdformatter/linktransformer/config.go index b98c664..da15c94 100644 --- a/pkg/mdformatter/linktransformer/config.go +++ b/pkg/mdformatter/linktransformer/config.go @@ -17,7 +17,7 @@ import ( ) type Config struct { - Version int + Version int `yaml:"version"` Cache CacheConfig `yaml:"cache"` diff --git a/pkg/yamlgen/yamlgen.go b/pkg/yamlgen/yamlgen.go index db4dd1a..08a8637 100644 --- a/pkg/yamlgen/yamlgen.go +++ b/pkg/yamlgen/yamlgen.go @@ -104,14 +104,11 @@ func GenGoCode(src []byte) (string, error) { jen.Qual("fmt", "Println").Call(jen.Lit("---")), jen.Qual("fmt", "Println").Call(jen.Id("k")), // TODO(saswatamcode): Replace with import from mdox itself once merged. - // jen.Qual("github.com/bwplotka/mdox/yamlgen", "Generate").Call(jen.Id("config"), jen.Qual("os", "Stderr")), - jen.Qual("structgen/cfggen", "Generate").Call(jen.Id("config"), jen.Qual("os", "Stderr")), + jen.Qual("github.com/bwplotka/mdox/pkg/yamlgen", "Generate").Call(jen.Id("config"), jen.Qual("os", "Stderr")), )) // Generate main function in new module. - generatedCode.Func().Id("main").Params().Block( - init..., - ) + generatedCode.Func().Id("main").Params().Block(init...) return fmt.Sprintf("%#v", generatedCode), nil } @@ -123,23 +120,6 @@ func ExecGoCode(ctx context.Context, mainGo string) ([]byte, error) { } defer os.RemoveAll(tmpDir) - // TODO(saswatamcode): Remove once merged. - // This is weird but need it for getting Generate function inside tmpDir in PR. - // Once merged this can be removed and can be replaced with just an import in tmpDir/main.go. - err = os.Mkdir(filepath.Join(tmpDir, "cfggen"), 0700) - if err != nil { - return nil, err - } - code, err := os.Create(filepath.Join(tmpDir, "cfggen/cfggen.go")) - if err != nil { - return nil, err - } - defer code.Close() - _, err = code.Write([]byte(cfggenFile)) - if err != nil { - return nil, err - } - // Copy generated code to main.go. main, err := os.Create(filepath.Join(tmpDir, "main.go")) if err != nil { @@ -156,14 +136,22 @@ func ExecGoCode(ctx context.Context, mainGo string) ([]byte, error) { cmd := exec.CommandContext(ctx, "go", "mod", "init", "structgen") cmd.Dir = tmpDir if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "run %v", cmd) + return nil, errors.Wrapf(err, "mod init %v", cmd) + } + + // Replace for unreleased mdox yamlgen so don't need to copy cfggen code to new dir and compile. + // Currently in github.com/saswatamcode/mdox@v0.2.2-0.20210823074517-0245f9afb0a8. Replace once #79 is merged. + cmd = exec.CommandContext(ctx, "go", "mod", "edit", "-replace", "github.com/bwplotka/mdox=github.com/saswatamcode/mdox@v0.2.2-0.20210823074517-0245f9afb0a8") + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "mod edit %v", cmd) } // Import required packages(generate go.sum). cmd = exec.CommandContext(ctx, "go", "mod", "tidy") cmd.Dir = tmpDir if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "run %v", cmd) + return nil, errors.Wrapf(err, "mod tidy %v", cmd) } // Execute generate code and return output. @@ -178,66 +166,3 @@ func ExecGoCode(ctx context.Context, mainGo string) ([]byte, error) { return b.Bytes(), nil } - -// TODO(saswatamcode): Remove once merged. -// This is weird but need it for getting Generate function inside tmpDir in PR. -// Could also do with commit hash and go.mod, but it would change on each commit in PR). -// Once merged this can be removed and can be replaced with just an import in tmpDir/main.go. -const cfggenFile = `package cfggen - -import ( - "io" - "reflect" - - "github.com/fatih/structtag" - "github.com/pkg/errors" - "gopkg.in/yaml.v3" -) - -func Generate(obj interface{}, w io.Writer) error { - // We forbid omitempty option. This is for simplification for doc generation. - if err := checkForOmitEmptyTagOption(obj); err != nil { - return errors.Wrap(err, "invalid type") - } - return yaml.NewEncoder(w).Encode(obj) -} - -func checkForOmitEmptyTagOption(obj interface{}) error { - return checkForOmitEmptyTagOptionRec(reflect.ValueOf(obj)) -} - -func checkForOmitEmptyTagOptionRec(v reflect.Value) error { - switch v.Kind() { - case reflect.Struct: - for i := 0; i < v.NumField(); i++ { - tags, err := structtag.Parse(string(v.Type().Field(i).Tag)) - if err != nil { - return errors.Wrapf(err, "%s: failed to parse tag %q", v.Type().Field(i).Name, v.Type().Field(i).Tag) - } - - tag, err := tags.Get("yaml") - if err != nil { - return errors.Wrapf(err, "%s: failed to get tag %q", v.Type().Field(i).Name, v.Type().Field(i).Tag) - } - - for _, opts := range tag.Options { - if opts == "omitempty" { - return errors.Errorf("omitempty is forbidden for config, but spotted on field '%s'", v.Type().Field(i).Name) - } - } - - if err := checkForOmitEmptyTagOptionRec(v.Field(i)); err != nil { - return errors.Wrapf(err, "%s", v.Type().Field(i).Name) - } - } - - case reflect.Ptr: - return errors.New("nil pointers are not allowed in configuration") - - case reflect.Interface: - return checkForOmitEmptyTagOptionRec(v.Elem()) - } - - return nil -} -` From 021aec70e287b8ce3cafbaaff788801bf078011f Mon Sep 17 00:00:00 2001 From: Saswata Mukherjee Date: Sat, 10 Jun 2023 13:21:31 +0530 Subject: [PATCH 5/5] Add basic tests and fix pkg import and struct initialization Signed-off-by: Saswata Mukherjee --- go.mod | 2 +- go.sum | 2 ++ pkg/yamlgen/yamlgen.go | 69 +++++++++++++++++++++++++++---------- pkg/yamlgen/yamlgen_test.go | 60 ++++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 19 deletions(-) create mode 100644 pkg/yamlgen/yamlgen_test.go diff --git a/go.mod b/go.mod index 5f1c563..bd32631 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Kunde21/markdownfmt/v3 v3.1.0 github.com/alecthomas/kingpin/v2 v2.3.2 github.com/charmbracelet/glamour v0.6.0 - github.com/dave/jennifer v1.4.1 + github.com/dave/jennifer v1.6.1 github.com/efficientgo/core v1.0.0-rc.2 github.com/efficientgo/tools/extkingpin v0.0.0-20230505153745-6b7392939a60 github.com/fatih/structtag v1.2.0 diff --git a/go.sum b/go.sum index ebc2ea6..ff6c373 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA= +github.com/dave/jennifer v1.6.1 h1:T4T/67t6RAA5AIV6+NP8Uk/BIsXgDoqEowgycdQQLuk= +github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/pkg/yamlgen/yamlgen.go b/pkg/yamlgen/yamlgen.go index 08a8637..3efd96d 100644 --- a/pkg/yamlgen/yamlgen.go +++ b/pkg/yamlgen/yamlgen.go @@ -6,16 +6,15 @@ package yamlgen import ( "bytes" "context" - "fmt" "go/ast" "go/parser" "go/token" "go/types" - "io/ioutil" "os" "os/exec" "path/filepath" "regexp" + "strings" "github.com/dave/jennifer/jen" "github.com/pkg/errors" @@ -31,6 +30,8 @@ func GenGoCode(src []byte) (string, error) { // Create new main file. fset := token.NewFileSet() generatedCode := jen.NewFile("main") + // Don't really need to format here, saves time. + // generatedCode.NoFormat = true // Parse source file. f, err := parser.ParseFile(fset, "", src, parser.AllErrors) @@ -38,11 +39,6 @@ func GenGoCode(src []byte) (string, error) { return "", err } - // Add imports if needed(will not be used if not required in code). - for _, s := range f.Imports { - generatedCode.ImportName(s.Path.Value[1:len(s.Path.Value)-1], "") - } - // Init statements for structs. var init []jen.Code // Declare config map, i.e, `configs := map[string]interface{}{}`. @@ -54,17 +50,18 @@ func GenGoCode(src []byte) (string, error) { if genericDecl, ok := decl.(*ast.GenDecl); ok { // Check if declaration spec is `type`. if typeDecl, ok := genericDecl.Specs[0].(*ast.TypeSpec); ok { - var structFields []jen.Code // Cast to `type struct`. structDecl, ok := typeDecl.Type.(*ast.StructType) if !ok { generatedCode.Type().Id(typeDecl.Name.Name).Id(string(src[typeDecl.Type.Pos()-1 : typeDecl.Type.End()-1])) continue } - fields := structDecl.Fields.List + + var structFields []jen.Code arrayInit := make(jen.Dict) // Loop and generate fields for each field. + fields := structDecl.Fields.List for _, field := range fields { // Each field might have multiple names. names := field.Names @@ -72,15 +69,51 @@ func GenGoCode(src []byte) (string, error) { if n.IsExported() { pos := n.Obj.Decl.(*ast.Field) - // Check if field is a slice type. - sliceRe := regexp.MustCompile(`.*\[.*\].*`) - if sliceRe.MatchString(types.ExprString(field.Type)) { - arrayInit[jen.Id(n.Name)] = jen.Id(string(src[pos.Type.Pos()-1 : pos.Type.End()-1])).Values(jen.Id(string(src[pos.Type.Pos()+1 : pos.Type.End()-1])).Values()) + // Copy struct field to generated code, with imports in case of other package imported field. + if pos.Tag != nil { + typeStr := string(src[pos.Type.Pos()-1 : pos.Type.End()-1]) + // Check if field is imported from other package. + if strings.Contains(typeStr, ".") { + typeArr := strings.SplitN(typeStr, ".", 2) + // Match the import name with the import statement. + for _, s := range f.Imports { + moduleName := "" + // Choose to copy same alias as in source file or use package name. + if s.Name == nil { + _, moduleName = filepath.Split(s.Path.Value[1 : len(s.Path.Value)-1]) + generatedCode.ImportName(s.Path.Value[1:len(s.Path.Value)-1], moduleName) + } else { + moduleName = s.Name.String() + generatedCode.ImportAlias(s.Path.Value[1:len(s.Path.Value)-1], moduleName) + } + // Add field to struct only if import name matches. + if moduleName == typeArr[0] { + structFields = append(structFields, jen.Id(n.Name).Qual(s.Path.Value[1:len(s.Path.Value)-1], typeArr[1]).Id(pos.Tag.Value)) + } + } + } else { + structFields = append(structFields, jen.Id(n.Name).Id(string(src[pos.Type.Pos()-1:pos.Type.End()-1])).Id(pos.Tag.Value)) + } } - // Copy struct field to generated code. - if pos.Tag != nil { - structFields = append(structFields, jen.Id(n.Name).Id(string(src[pos.Type.Pos()-1:pos.Type.End()-1])).Id(pos.Tag.Value)) + // Check if field is a slice type. + sliceRe := regexp.MustCompile(`^\[.*\].*$`) + typeStr := types.ExprString(field.Type) + if sliceRe.MatchString(typeStr) { + iArr := "[]int" + fArr := "[]float" + cArr := "[]complex" + uArr := "[]uint" + switch typeStr { + case "[]bool", "[]string", "[]byte", "[]rune", + iArr, iArr + "8", iArr + "16", iArr + "32", iArr + "64", + fArr + "32", fArr + "64", + cArr + "64", cArr + "128", + uArr, uArr + "8", uArr + "16", uArr + "32", uArr + "64", uArr + "ptr": + arrayInit[jen.Id(n.Name)] = jen.Id(typeStr + "{}") + default: + arrayInit[jen.Id(n.Name)] = jen.Id(string(src[pos.Type.Pos()-1 : pos.Type.End()-1])).Values(jen.Id(string(src[pos.Type.Pos()+1 : pos.Type.End()-1])).Values()) + } } } } @@ -109,12 +142,12 @@ func GenGoCode(src []byte) (string, error) { // Generate main function in new module. generatedCode.Func().Id("main").Params().Block(init...) - return fmt.Sprintf("%#v", generatedCode), nil + return generatedCode.GoString(), nil } // execGoCode executes and returns output from generated Go code. func ExecGoCode(ctx context.Context, mainGo string) ([]byte, error) { - tmpDir, err := ioutil.TempDir("", "structgen") + tmpDir, err := os.MkdirTemp("", "structgen") if err != nil { return nil, err } diff --git a/pkg/yamlgen/yamlgen_test.go b/pkg/yamlgen/yamlgen_test.go new file mode 100644 index 0000000..f8f2d92 --- /dev/null +++ b/pkg/yamlgen/yamlgen_test.go @@ -0,0 +1,60 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +package yamlgen + +import ( + "testing" + + "github.com/efficientgo/core/testutil" + "golang.org/x/net/context" +) + +func TestYAMLGen_GenGoCode(t *testing.T) { + t.Run("normal struct", func(t *testing.T) { + source := []byte("package main\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n") + generatedCode, err := GenGoCode(source) + testutil.Ok(t, err) + + expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"TestConfig\"] = TestConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + testutil.Equals(t, expected, generatedCode) + }) + + t.Run("struct with unexported field", func(t *testing.T) { + source := []byte("package main\n\nimport \"regexp\"\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n\n\tr *regexp.Regexp\n}\n") + generatedCode, err := GenGoCode(source) + testutil.Ok(t, err) + + expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + testutil.Equals(t, expected, generatedCode) + }) + + t.Run("struct with array fields", func(t *testing.T) { + source := []byte("package main\n\nimport \"regexp\"\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n\n\tr *regexp.Regexp\n}\n\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n") + generatedCode, err := GenGoCode(source) + testutil.Ok(t, err) + + expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"Config\"] = Config{\n\t\tIgnore: []IgnoreConfig{IgnoreConfig{}},\n\t\tValidator: []ValidatorConfig{ValidatorConfig{}},\n\t}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tconfigs[\"IgnoreConfig\"] = IgnoreConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + testutil.Equals(t, expected, generatedCode) + }) +} + +func TestYAMLGen_ExecGoCode(t *testing.T) { + t.Run("normal struct", func(t *testing.T) { + generatedCode := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"TestConfig\"] = TestConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + output, err := ExecGoCode(context.TODO(), generatedCode) + testutil.Ok(t, err) + + expected := "---\nTestConfig\nurl: \"\"\nid: 0\ntoken: \"\"\n" + testutil.Equals(t, expected, string(output)) + }) + + t.Run("struct with array fields", func(t *testing.T) { + generatedCode := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"Config\"] = Config{\n\t\tIgnore: []IgnoreConfig{IgnoreConfig{}},\n\t\tValidator: []ValidatorConfig{ValidatorConfig{}},\n\t}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tconfigs[\"IgnoreConfig\"] = IgnoreConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + output, err := ExecGoCode(context.TODO(), generatedCode) + testutil.Ok(t, err) + + expected := "---\nConfig\nversion: 0\nvalidators:\n - type: \"\"\n regex: \"\"\n token: \"\"\nignore:\n - url: \"\"\n id: 0\n token: \"\"\n---\nValidatorConfig\ntype: \"\"\nregex: \"\"\ntoken: \"\"\n---\nIgnoreConfig\nurl: \"\"\nid: 0\ntoken: \"\"\n" + testutil.Equals(t, expected, string(output)) + }) +}