diff --git a/Makefile b/Makefile index 88c53390..567c4e02 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,7 @@ GIT_COMMIT:=$(shell git rev-parse --short HEAD) IMG ?= perconalab/version-service:$(GIT_BRANCH)-$(GIT_COMMIT) init: - go build -modfile=tools/go.mod -o bin/yq github.com/mikefarah/yq/v3 - go build -modfile=tools/go.mod -o bin/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go - go build -modfile=tools/go.mod -o bin/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc - go build -modfile=tools/go.mod -o bin/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway - go build -modfile=tools/go.mod -o bin/protoc-gen-openapiv2 github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 + cd tools && go generate -x -tags=tools curl -L "https://github.com/bufbuild/buf/releases/download/v1.34.0/buf-$(shell uname -s)-$(shell uname -m)" -o "./bin/buf" chmod +x ./bin/buf diff --git a/tools/operator-tool/README.md b/tools/operator-tool/README.md new file mode 100644 index 00000000..ae898d42 --- /dev/null +++ b/tools/operator-tool/README.md @@ -0,0 +1,75 @@ +# operator-tool + +`operator-tool` is designed to generate a source file for a version service. It retrieves a list of product versions from the [Percona Downloads](https://www.percona.com/downloads) API (`https://www.percona.com/products-api.php`) and searches for the corresponding images in the [Docker Hub repository](https://hub.docker.com/u/percona). If an image is not specified in the API, the latest tag of that image will be used. + +Build it using `make init`. + +## Usage + +### Help + +```sh +$ ./bin/operator-tool --help +Usage of ./bin/operator-tool: + -cap int + Sets a limit on the number of versions allowed for each major version of a product + -file string + Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file + -include-arch-images + Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file + -only-latest + Add only latest major version images to the specified "-file". If "-file" is not specified, returns a file with latest major versions. + -operator string + Operator name. Available values: [psmdb pxc ps pg] + -patch string + Provide a path to a patch file to add additional images. Must be used together with the --file option + -verbose + Show logs + -version string + Operator version +``` + +### Generating source file from zero + +```sh +$ ./bin/operator-tool --operator "psmdb" --version "1.17.0" # outputs source file for psmdb-operator +... +$ ./bin/operator-tool --operator "pg" --version "2.5.0" # outputs source file for pg-operator +... +$ ./bin/operator-tool --operator "ps" --version "0.8.0" # outputs source file for ps-operator +... +$ ./bin/operator-tool --operator "pxc" --version "1.15.1" # outputs source file for pxc-operator +... +``` + +### Generating source file based on older file + +```sh +$ ./bin/operator-tool --file ./sources/operator.2.5.0.pg-operator.json --version "1.17.0" # outputs source file for pg-operator, excluding older versions specified in the file +... +``` + +### Patching existing source file with a patch file + +```sh +$ ./bin/operator-tool --file ./sources/operator.2.5.0.pg-operator.json --patch ./tools/operator-tool/patch-file.json.example +... +``` + +*Patch file example:* +The example patch file can be found [here](./patch-file.json.example). + +```json +{ + "operator": { + "2.4.28": { + "image_path": "some-path:tag" + } + }, + "pmm": { + "2.50.1": { + "image_path": "some-path:tag" + } + } +} +``` diff --git a/tools/operator-tool/cmd/edit.go b/tools/operator-tool/cmd/edit.go new file mode 100644 index 00000000..70d517df --- /dev/null +++ b/tools/operator-tool/cmd/edit.go @@ -0,0 +1,163 @@ +package main + +import ( + "fmt" + "reflect" + "slices" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + gover "github.com/hashicorp/go-version" + + "operator-tool/internal/matrix" + "operator-tool/internal/util" +) + +// deleteOldVersions removes versions from the matrix that are older than those specified in the file. +func deleteOldVersions(file string, m *vsAPI.VersionMatrix) error { + oldestVersions, err := getOldestVersions(file) + if err != nil { + return fmt.Errorf("failed to get oldest versions from base file: %w", err) + } + + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + oldestVersion, ok := oldestVersions[fieldName] + if !ok { + return nil + } + + m := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(m) == 0 { + return nil + } + + for k := range m { + if util.Goversion(k).Compare(oldestVersion) < 0 { + fieldValue.SetMapIndex(reflect.ValueOf(k), reflect.Value{}) // delete old version from map + } + } + return nil + }) +} + +func keepOnlyLatestVersions(m *vsAPI.VersionMatrix) error { + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + return nil + } + + latestByMajorVer := make(map[int]string) + for v := range versionMap { + majorVer := util.Goversion(v).Segments()[0] + + curLatest, ok := latestByMajorVer[majorVer] + if !ok || util.Goversion(v).Compare(util.Goversion(curLatest)) > 0 { + latestByMajorVer[majorVer] = v + continue + } + } + + for v := range versionMap { + majorVer := util.Goversion(v).Segments()[0] + if latestByMajorVer[majorVer] != v { + fieldValue.SetMapIndex(reflect.ValueOf(v), reflect.Value{}) // delete non-latest version from map + } + } + return nil + }) +} + +// getOldestVersions returns a map where each key is a struct field name from the VersionMatrix +// of the specified file, and each value is the corresponding oldest version for that field. +func getOldestVersions(filePath string) (map[string]*gover.Version, error) { + prod, err := util.ReadBaseFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read base file: %w", err) + } + + versions := make(map[string]*gover.Version) + if err := matrix.Iterate(prod.Versions[0].Matrix, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + return nil + } + oldestVersion := "" + for k := range versionMap { + if oldestVersion == "" { + oldestVersion = k + continue + } + if util.Goversion(oldestVersion).Compare(util.Goversion(k)) > 0 { + oldestVersion = k + } + } + versions[fieldName] = util.Goversion(oldestVersion) + return nil + }); err != nil { + return nil, err + } + + return versions, nil +} + +func limitMajorVersions(m *vsAPI.VersionMatrix, capacity int) error { + if capacity <= 0 { + return nil + } + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + versionsByMajorVer := make(map[int][]string) + for v := range versionMap { + majorVer := util.Goversion(v).Segments()[0] + versionsByMajorVer[majorVer] = append(versionsByMajorVer[majorVer], v) + } + for _, versions := range versionsByMajorVer { + if len(versions) <= capacity { + return nil + } + slices.SortFunc(versions, func(a, b string) int { + return util.Goversion(b).Compare(util.Goversion(a)) + }) + + versionsToDelete := versions[capacity:] + for _, v := range versionsToDelete { + fieldValue.SetMapIndex(reflect.ValueOf(v), reflect.Value{}) + } + } + + return nil + }) +} + +// updateMatrixStatuses updates the statuses of version maps. +// For each major version, it sets the highest version as "recommended" +// and all other versions as "available". +func updateMatrixStatuses(m *vsAPI.VersionMatrix) error { + setStatus := func(vm map[string]*vsAPI.Version) { + highestVersions := make(map[int]string) + for version := range vm { + vm[version].Status = vsAPI.Status_available + + majorVersion := util.Goversion(version).Segments()[0] + + currentHighestVersion, ok := highestVersions[majorVersion] + + if !ok || util.Goversion(version).Compare(util.Goversion(currentHighestVersion)) > 0 { + highestVersions[majorVersion] = version + } + } + + for _, version := range highestVersions { + vm[version].Status = vsAPI.Status_recommended + } + } + + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + return nil + } + setStatus(versionMap) + return nil + }) +} diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go new file mode 100644 index 00000000..49a30dd0 --- /dev/null +++ b/tools/operator-tool/cmd/main.go @@ -0,0 +1,172 @@ +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "slices" + "strings" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + + "operator-tool/internal/filler" + "operator-tool/internal/matrix" + "operator-tool/internal/util" + "operator-tool/pkg/registry" +) + +const ( + operatorNameSuffix = "-operator" + + operatorNamePSMDB = "psmdb" + operatorNamePXC = "pxc" + operatorNamePS = "ps" + operatorNamePG = "pg" +) + +var validOperatorNames = []string{ + operatorNamePSMDB, + operatorNamePXC, + operatorNamePS, + operatorNamePG, +} + +var ( + operatorName = flag.String("operator", "", fmt.Sprintf("Operator name. Available values: %v", validOperatorNames)) + version = flag.String("version", "", "Operator version") + filePath = flag.String("file", "", "Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file") + patch = flag.String("patch", "", "Provide a path to a patch file to add additional images. Must be used together with the --file option") + verbose = flag.Bool("verbose", false, "Show logs") + includeMultiImages = flag.Bool("include-arch-images", false, `Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file`) + versionCap = flag.Int("cap", 0, `Sets a limit on the number of versions allowed for each major version of a product`) + onlyLatest = flag.Bool("only-latest", false, `Add only latest major version images to the specified "-file". If "-file" is not specified, returns a file with latest major versions.`) +) + +func main() { + flag.Parse() + + if *version == "" && *patch == "" { + log.Fatalln("ERROR: --version should be provided") + } + + if *filePath != "" { + product, err := util.ReadBaseFile(*filePath) + if err != nil { + log.Fatalln("ERROR: failed to read base file:", err.Error()) + } + *operatorName = strings.TrimSuffix(product.Versions[0].Product, operatorNameSuffix) + } else { + if *operatorName == "" { + log.Fatalln("ERROR: --operator or --file should be provided") + } + if *patch != "" { + log.Fatalln("ERROR: --file should be provided when --patch is used") + } + } + + switch { + case slices.Contains(validOperatorNames, *operatorName): + if !*verbose { + log.SetOutput(io.Discard) + } + + if err := printSourceFile(*operatorName, *version, *filePath, *patch, *includeMultiImages, *versionCap, *onlyLatest); err != nil { + log.SetOutput(os.Stderr) + log.Fatalln("ERROR: failed to generate source file:", err.Error()) + } + default: + log.Fatalf("ERROR: Unknown operator name: %s. Available values: %v\n", *operatorName, validOperatorNames) + } +} + +func printSourceFile(operatorName, version, file, patchFile string, includeArchSuffixes bool, capacity int, onlyLatest bool) error { + var productResponse *vsAPI.ProductResponse + var err error + + registryClient := registry.NewClient() + + if file != "" && patchFile != "" { + patchMatrix, err := util.ReadPatchFile(patchFile) + if err != nil { + return fmt.Errorf("failed to read patch file: %w", err) + } + productResponse, err = patchProductResponse(registryClient, file, patchMatrix, version) + if err != nil { + return fmt.Errorf("failed to patch product response: %w", err) + } + } else { + productResponse, err = getProductResponse(registryClient, operatorName, version, includeArchSuffixes) + if err != nil { + return fmt.Errorf("failed to get product response: %w", err) + } + if onlyLatest { + if err := keepOnlyLatestVersions(productResponse.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) + } + } + + if file != "" { + if onlyLatest { + productResponse, err = patchProductResponse(registryClient, file, productResponse.Versions[0].Matrix, version) + if err != nil { + return fmt.Errorf("failed to patch product response: %w", err) + } + } else { + if err := deleteOldVersions(file, productResponse.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) + } + } + } + } + + if err := updateMatrixStatuses(productResponse.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to update matrix statuses: %w", err) + } + if err := limitMajorVersions(productResponse.Versions[0].Matrix, capacity); err != nil { + return fmt.Errorf("failed to delete versions exceeding capacity: %w", err) + } + + content, err := util.Marshal(productResponse) + if err != nil { + return fmt.Errorf("failed to marshal product response: %w", err) + } + + fmt.Println(string(content)) + return nil +} + +func getProductResponse(rc *registry.RegistryClient, operatorName, version string, includeArchSuffixes bool) (*vsAPI.ProductResponse, error) { + var versionMatrix *vsAPI.VersionMatrix + var err error + + f := &filler.VersionFiller{ + RegistryClient: rc, + IncludeArchSuffixes: includeArchSuffixes, + } + switch operatorName { + case operatorNamePG: + versionMatrix, err = matrix.PG(f, version) + case operatorNamePS: + versionMatrix, err = matrix.PS(f, version) + case operatorNamePSMDB: + versionMatrix, err = matrix.PSMDB(f, version) + case operatorNamePXC: + versionMatrix, err = matrix.PXC(f, version) + default: + panic("problems with validation. unknown operator name " + operatorName) + } + if err != nil { + return nil, fmt.Errorf("failed to get version matrix: %w", err) + } + return &vsAPI.ProductResponse{ + Versions: []*vsAPI.OperatorVersion{ + { + Product: operatorName + operatorNameSuffix, + Operator: version, + Matrix: versionMatrix, + }, + }, + }, nil +} diff --git a/tools/operator-tool/cmd/patch.go b/tools/operator-tool/cmd/patch.go new file mode 100644 index 00000000..7eb2b9c1 --- /dev/null +++ b/tools/operator-tool/cmd/patch.go @@ -0,0 +1,104 @@ +package main + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + "google.golang.org/protobuf/encoding/protojson" + + "operator-tool/internal/matrix" + "operator-tool/internal/util" + "operator-tool/pkg/registry" +) + +func patchProductResponse(rc *registry.RegistryClient, baseFilepath string, patchMatrix *vsAPI.VersionMatrix, operatorVersion string) (*vsAPI.ProductResponse, error) { + baseFile, err := util.ReadBaseFile(baseFilepath) + if err != nil { + return nil, fmt.Errorf("failed to read base file: %w", err) + } + if err := updateMatrixData(rc, patchMatrix); err != nil { + return nil, fmt.Errorf("failed to update patch matrix hashes: %w", err) + } + + matrixToMap := func(matrix *vsAPI.VersionMatrix) (map[string]map[string]map[string]any, error) { + data, err := protojson.Marshal(matrix) + if err != nil { + return nil, fmt.Errorf("failed to marshal: %w", err) + } + + m := make(map[string]map[string]map[string]any) + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return m, nil + } + + baseMatrix, err := matrixToMap(baseFile.Versions[0].Matrix) + if err != nil { + return nil, fmt.Errorf("failed to convert base matrix to map: %w", err) + } + patchMatrixMap, err := matrixToMap(patchMatrix) + if err != nil { + return nil, fmt.Errorf("failed to convert patch matrix to map: %w", err) + } + + for product, versions := range patchMatrixMap { + for version, verInfo := range versions { + if _, ok := baseMatrix[product]; !ok { + baseMatrix[product] = make(map[string]map[string]any) + } + baseMatrix[product][version] = verInfo + } + } + + mapToMatrix := func(m map[string]map[string]map[string]any) (*vsAPI.VersionMatrix, error) { + data, err := json.Marshal(m) + if err != nil { + return nil, fmt.Errorf("failed to marshal: %w", err) + } + + matrix := new(vsAPI.VersionMatrix) + if err := protojson.Unmarshal(data, matrix); err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + + return matrix, nil + } + + baseFile.Versions[0].Matrix, err = mapToMatrix(baseMatrix) + if err != nil { + return nil, fmt.Errorf("failed to convert patched map to matrix: %w", err) + } + if operatorVersion == "" { + baseFile.Versions[0].Operator = operatorVersion + } + return baseFile, nil +} + +func updateMatrixData(rc *registry.RegistryClient, m *vsAPI.VersionMatrix) error { + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + return nil + } + + for k, v := range versionMap { + imageSpl := strings.Split(v.ImagePath, ":") + if len(imageSpl) == 1 { + return fmt.Errorf("image %s doesn't have tag", v.ImagePath) + } + tag := imageSpl[len(imageSpl)-1] + imageName := strings.TrimSuffix(v.ImagePath, ":"+tag) + image, err := rc.GetTag(imageName, tag) + if err != nil { + return fmt.Errorf("failed to get tag %s for image %s: %w", tag, imageName, err) + } + versionMap[k].ImageHash = image.DigestAMD64 + versionMap[k].ImageHashArm64 = image.DigestARM64 + } + return nil + }) +} diff --git a/tools/operator-tool/go.mod b/tools/operator-tool/go.mod new file mode 100644 index 00000000..6830fc9c --- /dev/null +++ b/tools/operator-tool/go.mod @@ -0,0 +1,23 @@ +module operator-tool + +go 1.23.4 + +toolchain go1.23.5 + +require ( + github.com/Percona-Lab/percona-version-service v0.0.0-20241013113618-2966a16cabb1 + github.com/hashicorp/go-version v1.7.0 + google.golang.org/protobuf v1.36.0 +) + +replace github.com/Percona-Lab/percona-version-service => ../../ + +require ( + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241219192143-6b3ec007d9bb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb // indirect + google.golang.org/grpc v1.69.2 // indirect +) diff --git a/tools/operator-tool/go.sum b/tools/operator-tool/go.sum new file mode 100644 index 00000000..7149f859 --- /dev/null +++ b/tools/operator-tool/go.sum @@ -0,0 +1,40 @@ +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +google.golang.org/genproto/googleapis/api v0.0.0-20241219192143-6b3ec007d9bb h1:B7GIB7sr443wZ/EAEl7VZjmh1V6qzkt5V+RYcUYtS1U= +google.golang.org/genproto/googleapis/api v0.0.0-20241219192143-6b3ec007d9bb/go.mod h1:E5//3O5ZIG2l71Xnt+P/CYUY8Bxs8E7WMoZ9tlcMbAY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb h1:3oy2tynMOP1QbTC0MsNNAV+Se8M2Bd0A5+x1QHyw+pI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= +google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= diff --git a/tools/operator-tool/internal/filler/filler.go b/tools/operator-tool/internal/filler/filler.go new file mode 100644 index 00000000..a3b1b7cc --- /dev/null +++ b/tools/operator-tool/internal/filler/filler.go @@ -0,0 +1,338 @@ +package filler + +import ( + "errors" + "fmt" + "log" + "regexp" + "slices" + "strings" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + gover "github.com/hashicorp/go-version" + + "operator-tool/internal/util" + "operator-tool/pkg/registry" +) + +// VersionFiller is a helper type for creating a map[string]*vsAPI.Version +// using information retrieved from Docker Hub. +type VersionFiller struct { + RegistryClient *registry.RegistryClient + IncludeArchSuffixes bool + + errs []error +} + +func (f *VersionFiller) addErr(err error) { + f.errs = append(f.errs, err) +} + +func (f *VersionFiller) exec(vm map[string]*vsAPI.Version, err error) map[string]*vsAPI.Version { + if err != nil { + f.addErr(err) + return nil + } + return vm +} + +// addVersionsFromRegistry searches Docker Hub for all tags associated with the specified image +// and appends any missing tags that match the MAJOR.MINOR.PATCH version format to the returned versions slice. +// +// Tags with a "-debug" suffix are excluded. +func (f *VersionFiller) addVersionsFromRegistry(image string, versions []string) []string { + wantedVerisons := make(map[string]struct{}, len(versions)) + coreVersions := make(map[string]struct{}) + for _, v := range versions { + wantedVerisons[v] = struct{}{} + coreVersions[util.Goversion(v).Core().String()] = struct{}{} + } + + tags, err := f.RegistryClient.GetTags(image) + if err != nil { + f.addErr(err) + return nil + } + + // getVersionMap will search for images with these suffixes. We don't need them in this function + ignoredSuffixes := append(util.GetArchSuffixes(), "-debug") + + hasIgnoredSuffix := func(tag string) bool { + for _, s := range ignoredSuffixes { + if strings.HasSuffix(tag, s) { + return true + } + } + return false + } + + for _, tag := range tags { + if hasIgnoredSuffix(tag) { + continue + } + if _, err := gover.NewVersion(tag); err != nil { + continue + } + if _, ok := coreVersions[util.Goversion(tag).Core().String()]; !ok { + continue + } + if _, ok := wantedVerisons[tag]; ok { + continue + } + versions = append(versions, tag) + } + return versions +} + +// Normal returns a map[string]*Version for the specified image by filtering tags +// with the given list of versions. +// +// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". +// Prerelease versions are preferred for each core version when available. See preferPrereleaseVersionsFilter function. +func (f *VersionFiller) Normal(image string, versions []string, addVersionsFromRegistry bool) map[string]*vsAPI.Version { + if addVersionsFromRegistry { + versions = f.addVersionsFromRegistry(image, versions) + } + + versions = preferPrereleaseVersionsFilter(versions) + + return f.exec(getVersionMap(f.RegistryClient, image, versions, f.IncludeArchSuffixes)) +} + +// preferPrereleaseVersionsFilter filters a slice of version strings to prioritize prerelease versions +// for each unique core version. For example, if the input is []string{"4.0.4-40", "4.0.4"}, the output +// will be []string{"4.0.4-40"}, as the prerelease version is preferred. +// +// If no prerelease versions are found for a core version, the function returns the non-prerelease versions +// for that core version instead. +func preferPrereleaseVersionsFilter(versions []string) []string { + verMap := make(map[string][]string) + + // Group versions by core version + for _, v := range versions { + coreVer := util.Goversion(v).Core().String() + verMap[coreVer] = append(verMap[coreVer], v) + } + + result := []string{} + for _, versionSlice := range verMap { + prereleaseVersions := []string{} + + // Get prerelease versions + for _, version := range versionSlice { + if util.Goversion(version).Prerelease() != "" { + prereleaseVersions = append(prereleaseVersions, version) + } + } + + if len(prereleaseVersions) == 0 { + result = append(result, versionSlice...) + continue + } + result = append(result, prereleaseVersions...) + } + + return result +} + +// Regex returns a map[string]*Version for the specified image by filtering tags +// with the given list of versions and a regular expression. +// +// The regex argument must contain at least one matching group, which will be used +// to filter the necessary images. For example, given the regex "(^.*)(?:-logcollector)" +// and versions []string{"1.2.1"}, the tag "1.2.1-logcollector" will be included, +// while "1.3.1-logcollector", "1.2.1-some-string", and "1.2.1" will not be included. +// +// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". +func (f *VersionFiller) Regex(image string, regex string, versions []string) map[string]*vsAPI.Version { + return f.exec(getVersionMapRegex(f.RegistryClient, image, regex, versions)) +} + +// Latest returns a map[string]*Version with latest version tag of the specified image. +// +// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". +func (f *VersionFiller) Latest(image string) map[string]*vsAPI.Version { + return f.exec(getVersionMapLatestVer(f.RegistryClient, image, f.IncludeArchSuffixes)) +} + +func (f *VersionFiller) Error() error { + return errors.Join(f.errs...) +} + +func getVersionMapRegex(rc *registry.RegistryClient, image string, regex string, versions []string) (map[string]*vsAPI.Version, error) { + m := make(map[string]*vsAPI.Version) + r := regexp.MustCompile(regex) + for _, v := range versions { + images, err := rc.GetImages(image, func(tag string) bool { + matches := r.FindStringSubmatch(tag) + if len(matches) <= 1 { + return false + } + if matches[1] != v { + return false + } + return true + }) + if err != nil { + return nil, err + } + if len(images) == 0 { + log.Printf("DEBUG: tag %s for image %s with regexp %s was not found\n", v, image, regex) + continue + } + + vm, err := versionMapFromImages(v, images) + if err != nil { + return nil, err + } + for v, versionMap := range vm { + m[v] = versionMap + } + } + return m, nil +} + +func getVersionMap(rc *registry.RegistryClient, image string, versions []string, includeArchSuffixes bool) (map[string]*vsAPI.Version, error) { + m := make(map[string]*vsAPI.Version) + for _, v := range versions { + images, err := rc.GetImages(image, func(tag string) bool { + allowedSuffixes := []string{""} + if includeArchSuffixes { + allowedSuffixes = append(allowedSuffixes, util.GetArchSuffixes()...) + } + for _, s := range allowedSuffixes { + tagWithoutSuffix := tag + if s != "" { + var found bool + tagWithoutSuffix, found = strings.CutSuffix(tag, s) + if !found { + continue + } + } + if tagWithoutSuffix == v { + return true + } + } + return false + }) + if err != nil { + return nil, err + } + if len(images) == 0 { + log.Printf("DEBUG: tag %s for image %s was not found\n", v, image) + continue + } + vm, err := versionMapFromImages(v, images) + if err != nil { + return nil, err + } + for v, versionMap := range vm { + m[v] = versionMap + } + } + if len(m) == 0 { + return nil, fmt.Errorf("image %s with %v tags was not found", image, versions) + } + return m, nil +} + +func getVersionMapLatestVer(rc *registry.RegistryClient, imageName string, includeArchSuffixes bool) (map[string]*vsAPI.Version, error) { + image, err := rc.GetLatestImage(imageName, includeArchSuffixes) + if err != nil { + return nil, err + } + vm, err := versionMapFromImages(image.Tag, []registry.Image{image}) + if err != nil { + return nil, err + } + + return vm, nil +} + +// versionMapFromImages returns a Version map for a given list of images and a base tag without any suffixes. +// +// Some images on Docker Hub are tagged like , -arm64, -aarch64, -amd64, and -multi. +// This function adds images with amd64 and arm64 builds to the provided map. +// +// Specify includedArchSuffixes to include images with suffixes in resulting map. +// By default includedArchSuffixes includes only -multi and -amd64 images. +// +// Logic: +// - If an image supports both amd64 and arm64 architectures and has a "-multi" suffix in its tag, +// the function includes a version of the image tag without the "-multi" suffix in the map. +// - If no image with both amd64 and arm64 builds is found, separate images for amd64 and arm64 +// are added individually. (if both amd64 and arm64 image suffixes are specified in the includedArchSuffixes) +func versionMapFromImages(baseTag string, images []registry.Image, includedArchSuffixes ...string) (map[string]*vsAPI.Version, error) { + if len(includedArchSuffixes) == 0 { + includedArchSuffixes = []string{"-multi", "-amd64"} + } + + baseTag = trimArchSuffix(baseTag) + + slices.SortFunc(images, func(a, b registry.Image) int { + return util.Goversion(b.Tag).Compare(util.Goversion(a.Tag)) + }) + + var multiImage, amd64Image, arm64Image *registry.Image + for _, image := range images { + switch { + case image.DigestARM64 == "" && image.DigestAMD64 == "": + case image.DigestARM64 != "" && image.DigestAMD64 != "": + if image.Tag == baseTag || multiImage == nil { + multiImage = &image + } + case image.DigestARM64 != "": + if image.Tag == baseTag || arm64Image == nil { + arm64Image = &image + } + case image.DigestAMD64 != "": + if image.Tag == baseTag || amd64Image == nil { + amd64Image = &image + } + } + } + + if multiImage == nil && amd64Image == nil && arm64Image == nil { + return nil, fmt.Errorf("necessary tags for %s image were not found", images[0].Name) + } + + versions := make(map[string]*vsAPI.Version) + if multiImage != nil && slices.Contains(includedArchSuffixes, "-multi") { + versions[baseTag+getArchSuffix(multiImage.Tag)] = &vsAPI.Version{ + ImagePath: multiImage.FullName(), + ImageHash: multiImage.DigestAMD64, + ImageHashArm64: multiImage.DigestARM64, + } + if multiImage.Tag == baseTag { + return versions, nil + } + } + if amd64Image != nil && slices.Contains(includedArchSuffixes, "-amd64") { + versions[baseTag+getArchSuffix(amd64Image.Tag)] = &vsAPI.Version{ + ImagePath: amd64Image.FullName(), + ImageHash: amd64Image.DigestAMD64, + } + } + // Include arm64 if multi image is not specified + if multiImage == nil && arm64Image != nil && slices.Contains(includedArchSuffixes, "-arm64") && slices.Contains(includedArchSuffixes, "-aarch64") { + versions[baseTag+getArchSuffix(arm64Image.Tag)] = &vsAPI.Version{ + ImagePath: arm64Image.FullName(), + ImageHashArm64: arm64Image.DigestARM64, + } + } + + return versions, nil +} + +func trimArchSuffix(tag string) string { + return strings.TrimSuffix(tag, getArchSuffix(tag)) +} + +func getArchSuffix(tag string) string { + for _, suffix := range util.GetArchSuffixes() { + if strings.HasSuffix(tag, suffix) { + return suffix + } + } + return "" +} diff --git a/tools/operator-tool/internal/matrix/matrix.go b/tools/operator-tool/internal/matrix/matrix.go new file mode 100644 index 00000000..e6a09165 --- /dev/null +++ b/tools/operator-tool/internal/matrix/matrix.go @@ -0,0 +1,97 @@ +package matrix + +import ( + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + + "operator-tool/internal/filler" + productsapi "operator-tool/pkg/products-api" +) + +func PG(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { + pgVersions, err := productsapi.GetProductVersions("", "postgresql-distribution-17", "postgresql-distribution-16", "postgresql-distribution-15", "postgresql-distribution-14", "postgresql-distribution-13") + if err != nil { + return nil, err + } + + matrix := &vsAPI.VersionMatrix{ + Postgresql: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-postgres$)`, pgVersions), + Pgbackrest: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-pgbackrest)`, pgVersions), + Pgbouncer: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-pgbouncer)`, pgVersions), + Postgis: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-postgres-gis)`, pgVersions), + Pmm: f.Latest("percona/pmm-client"), + Operator: f.Normal("percona/percona-postgresql-operator", []string{version}, false), + } + if err := f.Error(); err != nil { + return nil, err + } + return matrix, nil +} + +func PS(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { + psVersions, err := productsapi.GetProductVersions("Percona-Server-", "Percona-Server-8.0") + if err != nil { + return nil, err + } + + matrix := &vsAPI.VersionMatrix{ + Mysql: f.Normal("percona/percona-server", psVersions, true), + Pmm: f.Latest("percona/pmm-client"), + Router: f.Normal("percona/percona-mysql-router", psVersions, true), + Backup: f.Latest("percona/percona-xtrabackup"), + Operator: f.Normal("percona/percona-server-mysql-operator", []string{version}, false), + Haproxy: f.Latest("percona/haproxy"), + Orchestrator: f.Latest("percona/percona-orchestrator"), + Toolkit: f.Latest("percona/percona-toolkit"), + } + + if err := f.Error(); err != nil { + return nil, err + } + return matrix, nil +} + +func PSMDB(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { + mongoVersions, err := productsapi.GetProductVersions("percona-server-mongodb-", "percona-server-mongodb-8.0", "percona-server-mongodb-7.0", "percona-server-mongodb-6.0") + if err != nil { + return nil, err + } + + matrix := &vsAPI.VersionMatrix{ + Mongod: f.Normal("percona/percona-server-mongodb", mongoVersions, true), + Pmm: f.Latest("percona/pmm-client"), + Backup: f.Latest("percona/percona-backup-mongodb"), + Operator: f.Normal("percona/percona-server-mongodb-operator", []string{version}, false), + } + + if err := f.Error(); err != nil { + return nil, err + } + + return matrix, nil +} + +func PXC(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { + pxcVersions, err := productsapi.GetProductVersions("Percona-XtraDB-Cluster-", "Percona-XtraDB-Cluster-84", "Percona-XtraDB-Cluster-80", "Percona-XtraDB-Cluster-57") + if err != nil { + return nil, err + } + + xtrabackupVersions, err := productsapi.GetProductVersions("Percona-XtraBackup-", "Percona-XtraBackup-8.4", "Percona-XtraBackup-8.0", "Percona-XtraBackup-2.4") + if err != nil { + return nil, err + } + matrix := &vsAPI.VersionMatrix{ + Pxc: f.Normal("percona/percona-xtradb-cluster", pxcVersions, true), + Pmm: f.Latest("percona/pmm-client"), + Proxysql: f.Latest("percona/proxysql2"), + Haproxy: f.Latest("percona/haproxy"), + Backup: f.Regex("percona/percona-xtradb-cluster-operator", `(?:^\d+\.\d+\.\d+-pxc\d+\.\d+-backup-pxb)(.*)`, xtrabackupVersions), + LogCollector: f.Regex("percona/percona-xtradb-cluster-operator", `(^.*)(?:-logcollector)`, []string{version}), + Operator: f.Normal("percona/percona-xtradb-cluster-operator", []string{version}, false), + } + + if err := f.Error(); err != nil { + return nil, err + } + return matrix, nil +} diff --git a/tools/operator-tool/internal/matrix/util.go b/tools/operator-tool/internal/matrix/util.go new file mode 100644 index 00000000..f3b83b29 --- /dev/null +++ b/tools/operator-tool/internal/matrix/util.go @@ -0,0 +1,24 @@ +package matrix + +import ( + "reflect" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" +) + +func Iterate(matrix *vsAPI.VersionMatrix, f func(fieldName string, fieldValue reflect.Value) error) error { + matrixType := reflect.TypeOf(matrix).Elem() + matrixValue := reflect.ValueOf(matrix).Elem() + + for i := 0; i < matrixValue.NumField(); i++ { + field := matrixType.Field(i) + // check if value is exported + if field.PkgPath != "" { + continue + } + if err := f(field.Name, matrixValue.Field(i)); err != nil { + return err + } + } + return nil +} diff --git a/tools/operator-tool/internal/util/arch.go b/tools/operator-tool/internal/util/arch.go new file mode 100644 index 00000000..25b58517 --- /dev/null +++ b/tools/operator-tool/internal/util/arch.go @@ -0,0 +1,21 @@ +package util + +import "strings" + +func GetArchSuffixes() []string { + return []string{ + "-arm64", + "-aarch64", + "-multi", + "-amd64", + } +} + +func HasArchSuffix(s string) bool { + for _, suffix := range GetArchSuffixes() { + if strings.HasSuffix(s, suffix) { + return true + } + } + return false +} diff --git a/tools/operator-tool/internal/util/marshal.go b/tools/operator-tool/internal/util/marshal.go new file mode 100644 index 00000000..3807370a --- /dev/null +++ b/tools/operator-tool/internal/util/marshal.go @@ -0,0 +1,112 @@ +package util + +import ( + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" +) + +// Marshal marshals ProductResponse to JSON, ensuring the "critical" field is always included, +// without requiring modifications to the `versionpb/api` package or creating custom types that +// implement the json.Marshaler interface. +// Use protojson.Marshal instead if omitting the "critical" field is acceptable. +func Marshal(product *vsAPI.ProductResponse) ([]byte, error) { + m, err := productToMap(product) + if err != nil { + return nil, fmt.Errorf("json conversion: %w", err) + } + + content, err := json.Marshal(m) + if err != nil { + return nil, fmt.Errorf("failed to marshal product response: %w", err) + } + return content, nil +} + +// productToMap is a recursive function that converts a ProductResponse into a map. +// The resulting map can be used with json.Marshal, ensuring that fields like "critical" are included. +func productToMap(v any) (any, error) { + val := reflect.ValueOf(v) + if val.Kind() == reflect.Ptr { + if val.IsNil() { // skip nil values + return nil, nil + } + val = val.Elem() // dereference + } + + switch val.Kind() { + case reflect.Struct: + if val.NumField() == 0 { + return nil, nil + } + m := make(map[string]any) + for i := 0; i < val.NumField(); i++ { + field := val.Type().Field(i) + fieldValue := val.Field(i) + + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + continue + } + jsonFieldName := strings.Split(jsonTag, ",")[0] + omitempty := slices.Contains(strings.Split(jsonTag, ","), "omitempty") + if fieldValue.Kind() == reflect.Bool { + omitempty = false // do not omit bool values like "critical" + } + + if omitempty { + zeroValue := reflect.Zero(fieldValue.Type()) + + // check if the value is equal to its zero value + if reflect.DeepEqual(fieldValue.Interface(), zeroValue.Interface()) { + continue + } + } + + if status, ok := fieldValue.Interface().(vsAPI.Status); ok { + m[jsonFieldName] = vsAPI.Status_name[int32(status)] + continue + } + + fieldVal, err := productToMap(fieldValue.Interface()) + if err != nil { + return nil, err + } + + m[jsonFieldName] = fieldVal + } + return m, nil + case reflect.Slice: + var slice []any + for j := 0; j < val.Len(); j++ { + element := val.Index(j) + elementValue, err := productToMap(element.Interface()) + if err != nil { + return nil, err + } + if elementValue == nil { + continue + } + slice = append(slice, elementValue) + } + return slice, nil + case reflect.Map: + m := make(map[string]any) + for _, key := range val.MapKeys() { + value := val.MapIndex(key) + + elementValue, err := productToMap(value.Interface()) + if err != nil { + return nil, err + } + m[key.Interface().(string)] = elementValue + } + return m, nil + default: + return val.Interface(), nil + } +} diff --git a/tools/operator-tool/internal/util/read.go b/tools/operator-tool/internal/util/read.go new file mode 100644 index 00000000..ead17b61 --- /dev/null +++ b/tools/operator-tool/internal/util/read.go @@ -0,0 +1,35 @@ +package util + +import ( + "fmt" + "os" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + "google.golang.org/protobuf/encoding/protojson" +) + +func ReadBaseFile(path string) (*vsAPI.ProductResponse, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + product := new(vsAPI.ProductResponse) + err = protojson.Unmarshal(content, product) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return product, nil +} + +func ReadPatchFile(path string) (*vsAPI.VersionMatrix, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + matrix := new(vsAPI.VersionMatrix) + err = protojson.Unmarshal(content, matrix) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return matrix, nil +} diff --git a/tools/operator-tool/internal/util/version.go b/tools/operator-tool/internal/util/version.go new file mode 100644 index 00000000..2a5c14ed --- /dev/null +++ b/tools/operator-tool/internal/util/version.go @@ -0,0 +1,9 @@ +package util + +import ( + gover "github.com/hashicorp/go-version" +) + +func Goversion(v string) *gover.Version { + return gover.Must(gover.NewVersion(v)) +} diff --git a/tools/operator-tool/patch-file.json.example b/tools/operator-tool/patch-file.json.example new file mode 100644 index 00000000..ddce8895 --- /dev/null +++ b/tools/operator-tool/patch-file.json.example @@ -0,0 +1,12 @@ +{ + "backup": { + "2.4.28": { + "image_path": "some-path:tag" + } + }, + "pmm": { + "2.50.1": { + "image_path": "some-path:tag" + } + } +} diff --git a/tools/operator-tool/pkg/products-api/client.go b/tools/operator-tool/pkg/products-api/client.go new file mode 100644 index 00000000..607f6455 --- /dev/null +++ b/tools/operator-tool/pkg/products-api/client.go @@ -0,0 +1,102 @@ +package productsapi + +import ( + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + gover "github.com/hashicorp/go-version" +) + +func GetProductVersions(trimPrefix string, products ...string) ([]string, error) { + versions := []string{} + for _, product := range products { + productVersions, err := get(trimPrefix, product) + if err != nil { + return nil, fmt.Errorf("failed to get product versions for %s: %w", product, err) + } + versions = append(versions, productVersions...) + } + if len(versions) == 0 { + return nil, errors.New("not found") + } + return versions, nil +} + +func get(trimPrefix, product string) ([]string, error) { + u := url.URL{ + Scheme: "https", + Host: "www.percona.com", + Path: "products-api.php", + } + values := make(url.Values) + values.Add("version", product) + resp, err := http.PostForm(u.String(), values) + if err != nil { + return nil, fmt.Errorf("failed to make request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("invalid status from docker registry: %s", resp.Status) + } + versions, err := parseXML(resp.Body, trimPrefix) + if err != nil { + return nil, fmt.Errorf("failed to parse XML: %w", err) + } + + versionMap := make(map[string]struct{}) + for _, v := range versions { + versionMap[v] = struct{}{} + ver := gover.Must(gover.NewVersion(v)) + versionMap[ver.Core().String()] = struct{}{} + } + return mapToSlice(versionMap), nil +} + +func mapToSlice(m map[string]struct{}) []string { + s := make([]string, 0, len(m)) + for k := range m { + s = append(s, k) + } + return s +} + +func parseXML(data io.Reader, trimPrefix string) ([]string, error) { + type option struct { + Value string `xml:"value,attr"` + Text string `xml:",chardata"` + } + + var options []option + decoder := xml.NewDecoder(data) + + for { + token, err := decoder.Token() + if err != nil { + break + } + if startElem, ok := token.(xml.StartElement); ok && startElem.Name.Local == "option" { + var option option + if err := decoder.DecodeElement(&option, &startElem); err != nil { + return nil, fmt.Errorf("failed to decode element: %w", err) + } + if option.Value != "" { + options = append(options, option) + } + } + } + + var versions []string + for _, option := range options { + if trimPrefix != "" && !strings.HasPrefix(option.Value, trimPrefix) { + return nil, errors.New("prefix " + trimPrefix + " was not found in versions") + } + versions = append(versions, strings.TrimPrefix(option.Value, trimPrefix)) + } + + return versions, nil +} diff --git a/tools/operator-tool/pkg/registry/registry.go b/tools/operator-tool/pkg/registry/registry.go new file mode 100644 index 00000000..8d65ac88 --- /dev/null +++ b/tools/operator-tool/pkg/registry/registry.go @@ -0,0 +1,238 @@ +package registry + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strconv" + "strings" + + "operator-tool/internal/util" +) + +type tagResp struct { + Count int `json:"count"` + NextPage string `json:"next"` + PreviousPage string `json:"previous"` + Results []tagRespResult `json:"results"` +} + +type tagRespResult struct { + ContentType string `json:"content_type,omitempty"` + Creator float64 `json:"creator,omitempty"` + Digest string `json:"digest,omitempty"` + FullSize float64 `json:"full_size,omitempty"` + ID float64 `json:"id,omitempty"` + Images []struct { + Architecture string `json:"architecture,omitempty"` + Digest string `json:"digest,omitempty"` + Features string `json:"features,omitempty"` + LastPulled string `json:"last_pulled,omitempty"` + LastPushed string `json:"last_pushed,omitempty"` + Os string `json:"os,omitempty"` + OsFeatures string `json:"os_features,omitempty"` + OsVersion string `json:"os_version,omitempty"` + Size float64 `json:"size,omitempty"` + Status string `json:"status,omitempty"` + Variant string `json:"variant,omitempty"` + } `json:"images,omitempty"` + LastUpdated string `json:"last_updated,omitempty"` + LastUpdater float64 `json:"last_updater,omitempty"` + LastUpdaterUsername string `json:"last_updater_username,omitempty"` + MediaType string `json:"media_type,omitempty"` + Name string `json:"name,omitempty"` + Repository float64 `json:"repository,omitempty"` + TagLastPulled string `json:"tag_last_pulled,omitempty"` + TagLastPushed string `json:"tag_last_pushed,omitempty"` + TagStatus string `json:"tag_status,omitempty"` + V2 bool `json:"v2,omitempty"` +} + +func (t tagRespResult) Image(imageName string) Image { + digestAMD64 := "" + digestARM64 := "" + for _, v := range t.Images { + if v.Os != "linux" { + continue + } + if v.Architecture == "amd64" { + digestAMD64 = v.Digest + } + if v.Architecture == "arm64" { + digestARM64 = v.Digest + } + } + return Image{ + Name: imageName, + Tag: t.Name, + DigestAMD64: strings.TrimPrefix(digestAMD64, "sha256:"), + DigestARM64: strings.TrimPrefix(digestARM64, "sha256:"), + } +} + +type Image struct { + Name string + Tag string + DigestAMD64 string + DigestARM64 string +} + +func (i Image) FullName() string { + return i.Name + ":" + i.Tag +} + +type RegistryClient struct { + c *http.Client + cache map[string]any +} + +const defaultPageSize = 100 + +func (r *RegistryClient) readTag(imageName string, tag string) (tagRespResult, error) { + u := url.URL{ + Scheme: "https", + Host: "registry.hub.docker.com", + Path: path.Join("v2", "repositories", imageName, "tags", tag), + } + + var result tagRespResult + cachedResult, ok := r.cache[u.String()] + if ok { + result, ok = cachedResult.(tagRespResult) + if ok { + return result, nil + } + panic("caching error") + } + + resp, err := http.Get(u.String()) + if err != nil { + return result, fmt.Errorf("get: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("invalid status from docker hub registry: %s", resp.Status) + } + + content, err := io.ReadAll(resp.Body) + if err != nil { + return result, fmt.Errorf("failed to read body: %w", err) + } + if err := json.Unmarshal(content, &result); err != nil { + return result, fmt.Errorf("failed to unmarshal: %w", err) + } + r.cache[u.String()] = result + return result, nil +} + +func (r *RegistryClient) listTags(imageName string, page int) (tagResp, error) { + u := url.URL{ + Scheme: "https", + Host: "registry.hub.docker.com", + Path: path.Join("v2", "repositories", imageName, "tags"), + RawQuery: "page_size=" + strconv.Itoa(defaultPageSize) + "&page=" + strconv.Itoa(page), + } + + var result tagResp + cachedResult, ok := r.cache[u.String()] + if ok { + result, ok = cachedResult.(tagResp) + if ok { + return result, nil + } + panic("caching error") + } + + resp, err := http.Get(u.String()) + if err != nil { + return result, fmt.Errorf("get: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("invalid status from docker hub registry: %s", resp.Status) + } + + content, err := io.ReadAll(resp.Body) + if err != nil { + return result, fmt.Errorf("failed to read body: %w", err) + } + if err := json.Unmarshal(content, &result); err != nil { + return result, fmt.Errorf("failed to unmarshal: %w", err) + } + r.cache[u.String()] = result + return result, nil +} + +func NewClient() *RegistryClient { + return &RegistryClient{ + c: new(http.Client), + cache: make(map[string]any), + } +} + +func (r *RegistryClient) GetLatestImage(imageName string, includeArchSuffix bool) (Image, error) { + resp, err := r.listTags(imageName, 1) + if err != nil { + return Image{}, fmt.Errorf("failed to get latest image: %w", err) + } + for _, result := range resp.Results { + if result.Name == "latest" { + continue + } + if !includeArchSuffix && util.HasArchSuffix(result.Name) { + continue + } + if strings.Count(result.Name, ".") == 2 { + return result.Image(imageName), nil + } + } + return Image{}, errors.New("image not found") +} + +func (r *RegistryClient) GetTag(image, tag string) (Image, error) { + resp, err := r.readTag(image, tag) + if err != nil { + return Image{}, fmt.Errorf("failed to read tag: %w", err) + } + return resp.Image(image), nil +} + +func (r *RegistryClient) GetTags(imageName string) ([]string, error) { + tags := []string{} + for page := 1; ; page++ { + resp, err := r.listTags(imageName, page) + if err != nil { + return nil, fmt.Errorf("failed to get page %d: %w", page, err) + } + for _, result := range resp.Results { + tags = append(tags, result.Name) + } + if resp.NextPage == "" || len(resp.Results) < defaultPageSize { + return tags, nil + } + } +} + +func (r *RegistryClient) GetImages(imageName string, filterFunc func(tag string) bool) ([]Image, error) { + images := []Image{} + for page := 1; ; page++ { + resp, err := r.listTags(imageName, page) + if err != nil { + return nil, fmt.Errorf("failed to get page %d: %w", page, err) + } + for _, result := range resp.Results { + if !filterFunc(result.Name) { + continue + } + + images = append(images, result.Image(imageName)) + } + if resp.NextPage == "" || len(resp.Results) < defaultPageSize { + return images, nil + } + } +} diff --git a/tools/tools.go b/tools/tools.go index d864cddd..b6a80b65 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -10,3 +10,10 @@ import ( _ "google.golang.org/grpc/cmd/protoc-gen-go-grpc" _ "google.golang.org/protobuf/cmd/protoc-gen-go" ) + +//go:generate go build -o ../bin/yq github.com/mikefarah/yq/v3 +//go:generate go build -o ../bin/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go +//go:generate go build -o ../bin/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc +//go:generate go build -o ../bin/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway +//go:generate go build -o ../bin/protoc-gen-openapiv2 github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 +//go:generate bash -c "cd operator-tool && go build -o ../../bin/operator-tool ./cmd"