diff --git a/go.mod b/go.mod index cea5fdd..bd32631 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Kunde21/markdownfmt/v3 v3.1.0 github.com/alecthomas/kingpin/v2 v2.3.2 github.com/charmbracelet/glamour v0.6.0 + github.com/dave/jennifer v1.6.1 github.com/efficientgo/core v1.0.0-rc.2 github.com/efficientgo/tools/extkingpin v0.0.0-20230505153745-6b7392939a60 github.com/fatih/structtag v1.2.0 @@ -18,6 +19,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 github.com/mattn/go-sqlite3 v1.14.17 github.com/oklog/run v1.1.0 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.15.1 github.com/prometheus/common v0.42.0 github.com/sergi/go-diff v1.0.0 @@ -64,7 +66,6 @@ require ( github.com/niklasfasching/go-org v1.6.5 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect diff --git a/go.sum b/go.sum index f434a4c..ff6c373 100644 --- a/go.sum +++ b/go.sum @@ -87,6 +87,10 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= +github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA= +github.com/dave/jennifer v1.6.1 h1:T4T/67t6RAA5AIV6+NP8Uk/BIsXgDoqEowgycdQQLuk= +github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/pkg/mdformatter/linktransformer/config.go b/pkg/mdformatter/linktransformer/config.go index b98c664..da15c94 100644 --- a/pkg/mdformatter/linktransformer/config.go +++ b/pkg/mdformatter/linktransformer/config.go @@ -17,7 +17,7 @@ import ( ) type Config struct { - Version int + Version int `yaml:"version"` Cache CacheConfig `yaml:"cache"` diff --git a/pkg/mdformatter/mdgen/mdgen.go b/pkg/mdformatter/mdgen/mdgen.go index 38676d5..47b84d7 100644 --- a/pkg/mdformatter/mdgen/mdgen.go +++ b/pkg/mdformatter/mdgen/mdgen.go @@ -6,17 +6,20 @@ package mdgen import ( "bytes" "fmt" + "io/ioutil" "os/exec" "strconv" "strings" "github.com/bwplotka/mdox/pkg/mdformatter" + "github.com/bwplotka/mdox/pkg/yamlgen" "github.com/mattn/go-shellwords" ) const ( infoStringKeyExec = "mdox-exec" infoStringKeyExitCode = "mdox-expect-exit-code" + infoStringKeyGoStruct = "mdox-gen-go-struct" ) var ( @@ -58,6 +61,11 @@ func (t *genCodeBlockTransformer) TransformCodeBlock(ctx mdformatter.SourceConte return nil, fmt.Errorf("got %q without variable. Expected format is e.g ```yaml %s=\"\" but got %s", val[0], infoStringKeyExitCode, string(infoString)) } infoStringAttr[val[0]] = val[1] + case infoStringKeyGoStruct: + if len(val) != 2 { + return nil, fmt.Errorf("got %q without variable. Expected format is e.g ```yaml %s=\"\" but got %s", val[0], infoStringKeyGoStruct, string(infoString)) + } + infoStringAttr[val[0]] = val[1] } } @@ -98,6 +106,39 @@ func (t *genCodeBlockTransformer) TransformCodeBlock(ctx mdformatter.SourceConte return output, nil } + if fileWithStruct, ok := infoStringAttr[infoStringKeyGoStruct]; ok { + // This is like mdox-gen-go-struct=:structname for now. + fs := strings.Split(fileWithStruct, ":") + src, err := ioutil.ReadFile(fs[0]) + if err != nil { + return nil, fmt.Errorf("read file for yaml gen %v: %w", fs[0], err) + } + + generatedCode, err := yamlgen.GenGoCode(src) + if err != nil { + return nil, fmt.Errorf("generate code for yaml gen %v: %w", fs[0], err) + } + + b, err := yamlgen.ExecGoCode(ctx, generatedCode) + if err != nil { + return nil, fmt.Errorf("execute generated code for yaml gen %v: %w", fs[0], err) + } + + // TODO(saswatamcode): This feels sort of hacky, need better way of printing. + // Remove `---` and check struct name. + yamls := bytes.Split(b, []byte("---")) + for _, yaml := range yamls { + lines := bytes.Split(yaml, []byte("\n")) + if len(lines) > 1 { + if string(lines[1]) == fs[1] { + ret := bytes.Join(lines[2:len(lines)-1], []byte("\n")) + ret = append(ret, []byte("\n")...) + return ret, nil + } + } + } + } + panic("should never get here") } diff --git a/pkg/yamlgen/yamlgen.go b/pkg/yamlgen/yamlgen.go new file mode 100644 index 0000000..3efd96d --- /dev/null +++ b/pkg/yamlgen/yamlgen.go @@ -0,0 +1,201 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +package yamlgen + +import ( + "bytes" + "context" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/dave/jennifer/jen" + "github.com/pkg/errors" +) + +// TODO(saswatamcode): Add tests. +// TODO(saswatamcode): Check jennifer code for some safety. +// TODO(saswatamcode): Add mechanism for caching output from generated code. +// TODO(saswatamcode): Currently takes file names, need to make it module based(something such as https://golang.org/pkg/cmd/go/internal/list/). + +// GenGoCode generates Go code for yaml gen from structs in src file. +func GenGoCode(src []byte) (string, error) { + // Create new main file. + fset := token.NewFileSet() + generatedCode := jen.NewFile("main") + // Don't really need to format here, saves time. + // generatedCode.NoFormat = true + + // Parse source file. + f, err := parser.ParseFile(fset, "", src, parser.AllErrors) + if err != nil { + return "", err + } + + // Init statements for structs. + var init []jen.Code + // Declare config map, i.e, `configs := map[string]interface{}{}`. + init = append(init, jen.Id("configs").Op(":=").Map(jen.String()).Interface().Values()) + + // Loop through declarations in file. + for _, decl := range f.Decls { + // Cast to generic declaration node. + if genericDecl, ok := decl.(*ast.GenDecl); ok { + // Check if declaration spec is `type`. + if typeDecl, ok := genericDecl.Specs[0].(*ast.TypeSpec); ok { + // Cast to `type struct`. + structDecl, ok := typeDecl.Type.(*ast.StructType) + if !ok { + generatedCode.Type().Id(typeDecl.Name.Name).Id(string(src[typeDecl.Type.Pos()-1 : typeDecl.Type.End()-1])) + continue + } + + var structFields []jen.Code + arrayInit := make(jen.Dict) + + // Loop and generate fields for each field. + fields := structDecl.Fields.List + for _, field := range fields { + // Each field might have multiple names. + names := field.Names + for _, n := range names { + if n.IsExported() { + pos := n.Obj.Decl.(*ast.Field) + + // Copy struct field to generated code, with imports in case of other package imported field. + if pos.Tag != nil { + typeStr := string(src[pos.Type.Pos()-1 : pos.Type.End()-1]) + // Check if field is imported from other package. + if strings.Contains(typeStr, ".") { + typeArr := strings.SplitN(typeStr, ".", 2) + // Match the import name with the import statement. + for _, s := range f.Imports { + moduleName := "" + // Choose to copy same alias as in source file or use package name. + if s.Name == nil { + _, moduleName = filepath.Split(s.Path.Value[1 : len(s.Path.Value)-1]) + generatedCode.ImportName(s.Path.Value[1:len(s.Path.Value)-1], moduleName) + } else { + moduleName = s.Name.String() + generatedCode.ImportAlias(s.Path.Value[1:len(s.Path.Value)-1], moduleName) + } + // Add field to struct only if import name matches. + if moduleName == typeArr[0] { + structFields = append(structFields, jen.Id(n.Name).Qual(s.Path.Value[1:len(s.Path.Value)-1], typeArr[1]).Id(pos.Tag.Value)) + } + } + } else { + structFields = append(structFields, jen.Id(n.Name).Id(string(src[pos.Type.Pos()-1:pos.Type.End()-1])).Id(pos.Tag.Value)) + } + } + + // Check if field is a slice type. + sliceRe := regexp.MustCompile(`^\[.*\].*$`) + typeStr := types.ExprString(field.Type) + if sliceRe.MatchString(typeStr) { + iArr := "[]int" + fArr := "[]float" + cArr := "[]complex" + uArr := "[]uint" + switch typeStr { + case "[]bool", "[]string", "[]byte", "[]rune", + iArr, iArr + "8", iArr + "16", iArr + "32", iArr + "64", + fArr + "32", fArr + "64", + cArr + "64", cArr + "128", + uArr, uArr + "8", uArr + "16", uArr + "32", uArr + "64", uArr + "ptr": + arrayInit[jen.Id(n.Name)] = jen.Id(typeStr + "{}") + default: + arrayInit[jen.Id(n.Name)] = jen.Id(string(src[pos.Type.Pos()-1 : pos.Type.End()-1])).Values(jen.Id(string(src[pos.Type.Pos()+1 : pos.Type.End()-1])).Values()) + } + } + } + } + } + + // Add initialize statements for struct via code like `configs["Type"] = Type{}`. + // If struct has array members, use array initializer via code like `configs["Config"] = Config{ArrayMember: []Type{Type{}}}`. + init = append(init, jen.Id("configs").Index(jen.Lit(typeDecl.Name.Name)).Op("=").Id(typeDecl.Name.Name).Values(arrayInit)) + + // Finally put struct inside generated code. + generatedCode.Type().Id(typeDecl.Name.Name).Struct(structFields...) + } + } + } + + // Add for loop to iterate through map and return config YAML. + init = append(init, jen.For( + jen.List(jen.Id("k"), jen.Id("config")).Op(":=").Range().Id("configs"), + ).Block( + // We import the cfggen Generate method directly to generate output. + jen.Qual("fmt", "Println").Call(jen.Lit("---")), + jen.Qual("fmt", "Println").Call(jen.Id("k")), + // TODO(saswatamcode): Replace with import from mdox itself once merged. + jen.Qual("github.com/bwplotka/mdox/pkg/yamlgen", "Generate").Call(jen.Id("config"), jen.Qual("os", "Stderr")), + )) + + // Generate main function in new module. + generatedCode.Func().Id("main").Params().Block(init...) + return generatedCode.GoString(), nil +} + +// execGoCode executes and returns output from generated Go code. +func ExecGoCode(ctx context.Context, mainGo string) ([]byte, error) { + tmpDir, err := os.MkdirTemp("", "structgen") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) + + // Copy generated code to main.go. + main, err := os.Create(filepath.Join(tmpDir, "main.go")) + if err != nil { + return nil, err + } + defer main.Close() + + _, err = main.Write([]byte(mainGo)) + if err != nil { + return nil, err + } + + // Create go.mod in temp dir. + cmd := exec.CommandContext(ctx, "go", "mod", "init", "structgen") + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "mod init %v", cmd) + } + + // Replace for unreleased mdox yamlgen so don't need to copy cfggen code to new dir and compile. + // Currently in github.com/saswatamcode/mdox@v0.2.2-0.20210823074517-0245f9afb0a8. Replace once #79 is merged. + cmd = exec.CommandContext(ctx, "go", "mod", "edit", "-replace", "github.com/bwplotka/mdox=github.com/saswatamcode/mdox@v0.2.2-0.20210823074517-0245f9afb0a8") + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "mod edit %v", cmd) + } + + // Import required packages(generate go.sum). + cmd = exec.CommandContext(ctx, "go", "mod", "tidy") + cmd.Dir = tmpDir + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "mod tidy %v", cmd) + } + + // Execute generate code and return output. + b := bytes.Buffer{} + cmd = exec.CommandContext(ctx, "go", "run", "main.go") + cmd.Dir = tmpDir + cmd.Stderr = &b + cmd.Stdout = &b + if err := cmd.Run(); err != nil { + return nil, errors.Wrapf(err, "run %v out %v", cmd, b.String()) + } + + return b.Bytes(), nil +} diff --git a/pkg/yamlgen/yamlgen_test.go b/pkg/yamlgen/yamlgen_test.go new file mode 100644 index 0000000..f8f2d92 --- /dev/null +++ b/pkg/yamlgen/yamlgen_test.go @@ -0,0 +1,60 @@ +// Copyright (c) Bartłomiej Płotka @bwplotka +// Licensed under the Apache License 2.0. + +package yamlgen + +import ( + "testing" + + "github.com/efficientgo/core/testutil" + "golang.org/x/net/context" +) + +func TestYAMLGen_GenGoCode(t *testing.T) { + t.Run("normal struct", func(t *testing.T) { + source := []byte("package main\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n") + generatedCode, err := GenGoCode(source) + testutil.Ok(t, err) + + expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"TestConfig\"] = TestConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + testutil.Equals(t, expected, generatedCode) + }) + + t.Run("struct with unexported field", func(t *testing.T) { + source := []byte("package main\n\nimport \"regexp\"\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n\n\tr *regexp.Regexp\n}\n") + generatedCode, err := GenGoCode(source) + testutil.Ok(t, err) + + expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + testutil.Equals(t, expected, generatedCode) + }) + + t.Run("struct with array fields", func(t *testing.T) { + source := []byte("package main\n\nimport \"regexp\"\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\n\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n\n\tr *regexp.Regexp\n}\n\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n") + generatedCode, err := GenGoCode(source) + testutil.Ok(t, err) + + expected := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"Config\"] = Config{\n\t\tIgnore: []IgnoreConfig{IgnoreConfig{}},\n\t\tValidator: []ValidatorConfig{ValidatorConfig{}},\n\t}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tconfigs[\"IgnoreConfig\"] = IgnoreConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + testutil.Equals(t, expected, generatedCode) + }) +} + +func TestYAMLGen_ExecGoCode(t *testing.T) { + t.Run("normal struct", func(t *testing.T) { + generatedCode := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype TestConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"TestConfig\"] = TestConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + output, err := ExecGoCode(context.TODO(), generatedCode) + testutil.Ok(t, err) + + expected := "---\nTestConfig\nurl: \"\"\nid: 0\ntoken: \"\"\n" + testutil.Equals(t, expected, string(output)) + }) + + t.Run("struct with array fields", func(t *testing.T) { + generatedCode := "package main\n\nimport (\n\t\"fmt\"\n\tyamlgen \"github.com/bwplotka/mdox/pkg/yamlgen\"\n\t\"os\"\n)\n\ntype Config struct {\n\tVersion int `yaml:\"version\"`\n\tValidator []ValidatorConfig `yaml:\"validators\"`\n\tIgnore []IgnoreConfig `yaml:\"ignore\"`\n}\ntype ValidatorConfig struct {\n\tType string `yaml:\"type\"`\n\tRegex string `yaml:\"regex\"`\n\tToken string `yaml:\"token\"`\n}\ntype IgnoreConfig struct {\n\tUrl string `yaml:\"url\"`\n\tID int `yaml:\"id\"`\n\tToken string `yaml:\"token\"`\n}\n\nfunc main() {\n\tconfigs := map[string]interface{}{}\n\tconfigs[\"Config\"] = Config{\n\t\tIgnore: []IgnoreConfig{IgnoreConfig{}},\n\t\tValidator: []ValidatorConfig{ValidatorConfig{}},\n\t}\n\tconfigs[\"ValidatorConfig\"] = ValidatorConfig{}\n\tconfigs[\"IgnoreConfig\"] = IgnoreConfig{}\n\tfor k, config := range configs {\n\t\tfmt.Println(\"---\")\n\t\tfmt.Println(k)\n\t\tyamlgen.Generate(config, os.Stderr)\n\t}\n}\n" + output, err := ExecGoCode(context.TODO(), generatedCode) + testutil.Ok(t, err) + + expected := "---\nConfig\nversion: 0\nvalidators:\n - type: \"\"\n regex: \"\"\n token: \"\"\nignore:\n - url: \"\"\n id: 0\n token: \"\"\n---\nValidatorConfig\ntype: \"\"\nregex: \"\"\ntoken: \"\"\n---\nIgnoreConfig\nurl: \"\"\nid: 0\ntoken: \"\"\n" + testutil.Equals(t, expected, string(output)) + }) +}