From c591e1e0d82855501f0c201b02d53f4a40b21d86 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Tue, 22 Oct 2024 17:45:42 +0300 Subject: [PATCH 01/20] `operator-tool` init commit --- Makefile | 6 +- tools/operator-tool/README.md | 43 +++++ tools/operator-tool/cmd/filler.go | 212 +++++++++++++++++++++ tools/operator-tool/cmd/main.go | 126 ++++++++++++ tools/operator-tool/cmd/marshal.go | 112 +++++++++++ tools/operator-tool/cmd/matrix.go | 121 ++++++++++++ tools/operator-tool/cmd/util.go | 109 +++++++++++ tools/operator-tool/go.mod | 19 ++ tools/operator-tool/go.sum | 20 ++ tools/operator-tool/products-api/client.go | 102 ++++++++++ tools/operator-tool/registry/registry.go | 166 ++++++++++++++++ tools/tools.go | 7 + 12 files changed, 1038 insertions(+), 5 deletions(-) create mode 100644 tools/operator-tool/README.md create mode 100644 tools/operator-tool/cmd/filler.go create mode 100644 tools/operator-tool/cmd/main.go create mode 100644 tools/operator-tool/cmd/marshal.go create mode 100644 tools/operator-tool/cmd/matrix.go create mode 100644 tools/operator-tool/cmd/util.go create mode 100644 tools/operator-tool/go.mod create mode 100644 tools/operator-tool/go.sum create mode 100644 tools/operator-tool/products-api/client.go create mode 100644 tools/operator-tool/registry/registry.go diff --git a/Makefile b/Makefile index 2c59f58e..d9a247e0 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,7 @@ GIT_COMMIT:=$(shell git rev-parse --short HEAD) IMG ?= perconalab/version-service:$(GIT_BRANCH)-$(GIT_COMMIT) init: - go build -modfile=tools/go.mod -o bin/yq github.com/mikefarah/yq/v3 - go build -modfile=tools/go.mod -o bin/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go - go build -modfile=tools/go.mod -o bin/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc - go build -modfile=tools/go.mod -o bin/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway - go build -modfile=tools/go.mod -o bin/protoc-gen-openapiv2 github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 + cd tools && go generate -x -tags=tools curl -L "https://github.com/bufbuild/buf/releases/download/v1.34.0/buf-$(shell uname -s)-$(shell uname -m)" -o "./bin/buf" chmod +x ./bin/buf diff --git a/tools/operator-tool/README.md b/tools/operator-tool/README.md new file mode 100644 index 00000000..c8da0697 --- /dev/null +++ b/tools/operator-tool/README.md @@ -0,0 +1,43 @@ +# operator-tool + +`operator-tool` is designed to generate a source file for a version service. It retrieves a list of product versions from the [Percona Downloads](https://www.percona.com/downloads) API (`https://www.percona.com/products-api.php`) and searches for the corresponding images in the [Docker Hub repository](https://hub.docker.com/u/percona). If an image is not specified in the API, the latest tag of that image will be used. + +Build it using `make init`. + +## Usage + +### Help + +``` +$ ./bin/operator-tool --help +Usage of ./bin/operator-tool: + -file string + Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file. + -operator string + Operator name. Available values: [psmdb-operator pxc-operator ps-operator pg-operator] + -verbose + Show logs + -version string + Operator version + +``` + +### Generating source file from zero + +``` +$ ./bin/operator-tool --operator "psmdb-operator" --version "1.17.0" # outputs source file for psmdb-operator +... +$ ./bin/operator-tool --operator "pg-operator" --version "2.5.0" # outputs source file for pg-operator +... +$ ./bin/operator-tool --operator "ps-operator" --version "0.8.0" # outputs source file for ps-operator +... +$ ./bin/operator-tool --operator "pxc-operator" --version "1.15.1" # outputs source file for pxc-operator +... +``` + +### Generating source file based on older file + +``` +$ ./bin/operator-tool --file ./sources/operator.2.5.0.pg-operator.json --version "1.17.0" # outputs source file for pg-operator, excluding older versions specified in the file +... +``` diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go new file mode 100644 index 00000000..2636246b --- /dev/null +++ b/tools/operator-tool/cmd/filler.go @@ -0,0 +1,212 @@ +package main + +import ( + "errors" + "fmt" + "log" + "regexp" + "slices" + "strings" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + + "operator-tool/registry" +) + +// VersionMapFiller is a helper type for creating a map[string]*vsAPI.Version +// using information retrieved from Docker Hub. +type VersionMapFiller struct { + RegistryClient *registry.RegistryClient + errs []error +} + +func (f *VersionMapFiller) exec(vm map[string]*vsAPI.Version, err error) map[string]*vsAPI.Version { + if err != nil { + f.errs = append(f.errs, err) + return nil + } + + f.setRecommended(vm) + + return vm +} + +// setRecommended sets a recommended status to the latest version. +func (f *VersionMapFiller) setRecommended(vm map[string]*vsAPI.Version) { + maxVer := "" + for k := range vm { + if maxVer == "" { + maxVer = k + continue + } + + if goversion(k).Compare(goversion(maxVer)) > 0 { + maxVer = k + } + } + + if _, ok := vm[maxVer]; ok { + vm[maxVer].Status = vsAPI.Status_recommended + } +} + +// Normal returns a map[string]*Version for the specified image by filtering tags +// with the given list of versions. +// +// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". +func (f *VersionMapFiller) Normal(image string, versions []string) map[string]*vsAPI.Version { + return f.exec(getVersionMap(f.RegistryClient, image, versions)) +} + +// Regex returns a map[string]*Version for the specified image by filtering tags +// with the given list of versions and a regular expression. +// +// The regex argument must contain at least one matching group, which will be used +// to filter the necessary images. For example, given the regex "(^.*)(?:-logcollector)" +// and versions []string{"1.2.1"}, the tag "1.2.1-logcollector" will be included, +// while "1.3.1-logcollector", "1.2.1-some-string", and "1.2.1" will not be included. +// +// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". +func (f *VersionMapFiller) Regex(image string, regex string, versions []string) map[string]*vsAPI.Version { + return f.exec(getVersionMapRegex(f.RegistryClient, image, regex, versions)) +} + +// Latest returns a map[string]*Version with latest version tag of the specified image. +// +// The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". +func (f *VersionMapFiller) Latest(image string) map[string]*vsAPI.Version { + return f.exec(getVersionMapLatestVer(f.RegistryClient, image)) +} + +func (f *VersionMapFiller) Error() error { + return errors.Join(f.errs...) +} + +func getVersionMapRegex(rc *registry.RegistryClient, image string, regex string, versions []string) (map[string]*vsAPI.Version, error) { + m := make(map[string]*vsAPI.Version) + r := regexp.MustCompile(regex) + for _, v := range versions { + images, err := rc.GetImages(image, func(tag string) bool { + matches := r.FindStringSubmatch(tag) + if len(matches) <= 1 { + return false + } + if matches[1] != v { + return false + } + return true + }) + if err != nil { + return nil, err + } + if len(images) == 0 { + log.Printf("DEBUG: tag %s for image %s with regexp %s was not found\n", v, image, regex) + continue + } + + vm, err := versionMapFromImages(v, images) + if err != nil { + return nil, err + } + m[v] = vm + } + return m, nil +} + +func getVersionMap(rc *registry.RegistryClient, image string, versions []string) (map[string]*vsAPI.Version, error) { + m := make(map[string]*vsAPI.Version) + for _, v := range versions { + images, err := rc.GetImages(image, func(tag string) bool { + allowedSuffixes := []string{"", "-amd64", "-arm64", "-multi"} + for _, s := range allowedSuffixes { + if tag+s == v { + return true + } + } + return false + }) + if err != nil { + return nil, err + } + if len(images) == 0 { + log.Printf("DEBUG: tag %s for image %s was not found\n", v, image) + continue + } + vm, err := versionMapFromImages(v, images) + if err != nil { + return nil, err + } + m[v] = vm + } + return m, nil +} + +func getVersionMapLatestVer(rc *registry.RegistryClient, imageName string) (map[string]*vsAPI.Version, error) { + image, err := rc.GetLatestImage(imageName) + if err != nil { + return nil, err + } + vm, err := versionMapFromImages(image.Tag, []registry.Image{image}) + if err != nil { + return nil, err + } + return map[string]*vsAPI.Version{ + image.Tag: vm, + }, nil +} + +// versionMapFromImages returns a Version for a given list of images and a base tag without any suffixes. +// +// Some images on Docker Hub are tagged like , -arm64, -amd64, and -multi. +// This function attempts to use information from images with both amd64 and arm64 builds. If both are not available, it defaults to amd64. +// +// If multiple provided images share the same suffix, the function returns a Version with information for the latest image. +func versionMapFromImages(baseTag string, images []registry.Image) (*vsAPI.Version, error) { + slices.SortFunc(images, func(a, b registry.Image) int { + return goversion(b.Tag).Compare(goversion(a.Tag)) + }) + imageName := images[0].Name + var multiImage, amd64Image, arm64Image *registry.Image + for _, image := range images { + if strings.HasSuffix(image.Tag, "-arm64") { + arm64Image = &image + continue + } + if multiImage == nil { + if (image.DigestAMD64 != "" && image.DigestARM64 != "") || strings.HasSuffix(image.Tag, "-multi") { + multiImage = &image + continue + } + } + if image.Tag == baseTag || amd64Image == nil { + amd64Image = &image + continue + } + } + var imagePath, imageHash, imageHashArm64 string + + switch { + case multiImage != nil: + imagePath = multiImage.FullName() + imageHash = multiImage.DigestAMD64 + imageHashArm64 = multiImage.DigestARM64 + case amd64Image != nil && arm64Image != nil: + log.Printf("WARNING: Image %s has both %s and %s tags, but doesn't have \"-multi\" tag. Using %s\n", imageName, amd64Image, arm64Image, amd64Image) + fallthrough + case amd64Image != nil: + imagePath = amd64Image.FullName() + imageHash = amd64Image.DigestAMD64 + case arm64Image != nil: + imagePath = arm64Image.FullName() + imageHashArm64 = arm64Image.DigestARM64 + default: + return nil, fmt.Errorf("necessary tags for %s image were not found", imageName) + } + + return &vsAPI.Version{ + ImagePath: imagePath, + ImageHash: imageHash, + ImageHashArm64: imageHashArm64, + Status: vsAPI.Status_available, + }, nil +} diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go new file mode 100644 index 00000000..3b6ca0d0 --- /dev/null +++ b/tools/operator-tool/cmd/main.go @@ -0,0 +1,126 @@ +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "slices" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + + "operator-tool/registry" +) + +const ( + operatorNamePSMDB = "psmdb-operator" + operatorNamePXC = "pxc-operator" + operatorNamePS = "ps-operator" + operatorNamePG = "pg-operator" +) + +var validOperatorNames = []string{ + operatorNamePSMDB, + operatorNamePXC, + operatorNamePS, + operatorNamePG, +} + +var ( + operatorName = flag.String("operator", "", fmt.Sprintf("Operator name. Available values: %v", validOperatorNames)) + version = flag.String("version", "", "Operator version") + filePath = flag.String("file", "", "Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file.") + verbose = flag.Bool("verbose", false, "Show logs") +) + +func main() { + flag.Parse() + + if *version == "" { + log.Println("ERROR: --version should be provided") + os.Exit(1) + } + + if *filePath != "" { + product, err := readBaseFile(*filePath) + if err != nil { + log.Println("ERROR: failed to read base file:", err.Error()) + os.Exit(1) + } + *operatorName = product.Versions[0].Product + } else { + if *operatorName == "" { + log.Println("ERROR: --operator or --file should be provided") + os.Exit(1) + } + } + + switch { + case slices.Contains(validOperatorNames, *operatorName): + if !*verbose { + log.SetOutput(io.Discard) + } + + if err := printSourceFile(*operatorName, *version, *filePath); err != nil { + log.Println("ERROR: failed to generate source file: ", err.Error()) + os.Exit(1) + } + default: + log.Printf("ERROR: Unknown operator name: %s. Available values: %v\n", *operatorName, validOperatorNames) + os.Exit(1) + } +} + +func printSourceFile(operatorName, version, file string) error { + r, err := getProductResponse(operatorName, version) + if err != nil { + return fmt.Errorf("failed to get product response: %w", err) + } + if file != "" { + if err := deleteOldVersions(file, r.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) + } + } + + content, err := marshal(r) + if err != nil { + return fmt.Errorf("failed to marshal product response: %w", err) + } + + fmt.Println(string(content)) + return nil +} + +func getProductResponse(operatorName, version string) (*vsAPI.ProductResponse, error) { + var versionMatrix *vsAPI.VersionMatrix + var err error + + f := &VersionMapFiller{ + RegistryClient: registry.NewClient(), + } + switch operatorName { + case operatorNamePG: + versionMatrix, err = pgVersionMatrix(f, operatorName, version) + case operatorNamePS: + versionMatrix, err = psVersionMatrix(f, operatorName, version) + case operatorNamePSMDB: + versionMatrix, err = psmdbVersionMatrix(f, operatorName, version) + case operatorNamePXC: + versionMatrix, err = pxcVersionMatrix(f, operatorName, version) + default: + panic("problems with validation. unknown operator name " + operatorName) + } + if err != nil { + return nil, fmt.Errorf("failed to get version matrix: %w", err) + } + return &vsAPI.ProductResponse{ + Versions: []*vsAPI.OperatorVersion{ + { + Product: operatorName, + Operator: version, + Matrix: versionMatrix, + }, + }, + }, nil +} diff --git a/tools/operator-tool/cmd/marshal.go b/tools/operator-tool/cmd/marshal.go new file mode 100644 index 00000000..38c8d81f --- /dev/null +++ b/tools/operator-tool/cmd/marshal.go @@ -0,0 +1,112 @@ +package main + +import ( + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" +) + +// marshal marshals ProductResponse to JSON, ensuring the "critical" field is always included, +// without requiring modifications to the `versionpb/api` package or creating custom types that +// implement the json.Marshaler interface. +// Use protojson.Marshal instead if omitting the "critical" field is acceptable. +func marshal(product *vsAPI.ProductResponse) ([]byte, error) { + m, err := productToMap(product) + if err != nil { + return nil, fmt.Errorf("json conversion: %w", err) + } + + content, err := json.Marshal(m) + if err != nil { + return nil, fmt.Errorf("failed to marshal product response: %w", err) + } + return content, nil +} + +// productToMap is a recursive function that converts a ProductResponse into a map. +// The resulting map can be used with json.Marshal, ensuring that fields like "critical" are included. +func productToMap(v any) (any, error) { + val := reflect.ValueOf(v) + if val.Kind() == reflect.Ptr { + if val.IsNil() { // skip nil values + return nil, nil + } + val = val.Elem() // dereference + } + + switch val.Kind() { + case reflect.Struct: + if val.NumField() == 0 { + return nil, nil + } + m := make(map[string]any) + for i := 0; i < val.NumField(); i++ { + field := val.Type().Field(i) + fieldValue := val.Field(i) + + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + continue + } + jsonFieldName := strings.Split(jsonTag, ",")[0] + omitempty := slices.Contains(strings.Split(jsonTag, ","), "omitempty") + if fieldValue.Kind() == reflect.Bool { + omitempty = false // do not omit bool values like "critical" + } + + if omitempty { + zeroValue := reflect.Zero(fieldValue.Type()) + + // check if the value is equal to its zero value + if reflect.DeepEqual(fieldValue.Interface(), zeroValue.Interface()) { + continue + } + } + + if status, ok := fieldValue.Interface().(vsAPI.Status); ok { + m[jsonFieldName] = vsAPI.Status_name[int32(status)] + continue + } + + fieldVal, err := productToMap(fieldValue.Interface()) + if err != nil { + return nil, err + } + + m[jsonFieldName] = fieldVal + } + return m, nil + case reflect.Slice: + var slice []any + for j := 0; j < val.Len(); j++ { + element := val.Index(j) + elementValue, err := productToMap(element.Interface()) + if err != nil { + return nil, err + } + if elementValue == nil { + continue + } + slice = append(slice, elementValue) + } + return slice, nil + case reflect.Map: + m := make(map[string]any) + for _, key := range val.MapKeys() { + value := val.MapIndex(key) + + elementValue, err := productToMap(value.Interface()) + if err != nil { + return nil, err + } + m[key.Interface().(string)] = elementValue + } + return m, nil + default: + return val.Interface(), nil + } +} diff --git a/tools/operator-tool/cmd/matrix.go b/tools/operator-tool/cmd/matrix.go new file mode 100644 index 00000000..f1ea7db0 --- /dev/null +++ b/tools/operator-tool/cmd/matrix.go @@ -0,0 +1,121 @@ +package main + +import ( + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + + productsapi "operator-tool/products-api" +) + +func pgVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI.VersionMatrix, error) { + pgVersions, err := productsapi.GetProductVersions("", "postgresql-distribution-16", "postgresql-distribution-15", "postgresql-distribution-14", "postgresql-distribution-13", "postgresql-distribution-12") + if err != nil { + return nil, err + } + + pmmVersions, err := productsapi.GetProductVersions("", "pmm2") + if err != nil { + return nil, err + } + + matrix := &vsAPI.VersionMatrix{ + Postgresql: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-postgres$)`, pgVersions), + Pgbackrest: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-pgbackrest)`, pgVersions), + Pgbouncer: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-pgbouncer)`, pgVersions), + Postgis: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-postgres-gis)`, pgVersions), + Pmm: f.Normal("percona/pmm-client", pmmVersions), + Operator: f.Normal("percona/percona-postgresql-operator", []string{version}), + } + if err := f.Error(); err != nil { + return nil, err + } + return matrix, nil +} + +func psVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI.VersionMatrix, error) { + psVersions, err := productsapi.GetProductVersions("Percona-Server-", "Percona-Server-8.0") + if err != nil { + return nil, err + } + + pmmVersions, err := productsapi.GetProductVersions("", "pmm2") + if err != nil { + return nil, err + } + + matrix := &vsAPI.VersionMatrix{ + Mysql: f.Normal("percona/percona-server", psVersions), + Pmm: f.Normal("percona/pmm-client", pmmVersions), + Router: f.Normal("percona/percona-mysql-router", psVersions), + Backup: f.Normal("percona/percona-xtrabackup", psVersions), + Operator: f.Normal("percona/percona-server-mysql-operator", []string{version}), + Haproxy: f.Latest("percona/haproxy"), + Orchestrator: f.Latest("percona/percona-orchestrator"), + Toolkit: f.Latest("percona/percona-toolkit"), + } + + if err := f.Error(); err != nil { + return nil, err + } + return matrix, nil +} + +func psmdbVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI.VersionMatrix, error) { + mongoVersions, err := productsapi.GetProductVersions("percona-server-mongodb-", "percona-server-mongodb-7.0", "percona-server-mongodb-6.0", "percona-server-mongodb-5.0") + if err != nil { + return nil, err + } + + pmmVersions, err := productsapi.GetProductVersions("", "pmm2") + if err != nil { + return nil, err + } + + pbmVersions, err := productsapi.GetProductVersions("percona-backup-mongodb-", "percona-backup-mongodb") + if err != nil { + return nil, err + } + + matrix := &vsAPI.VersionMatrix{ + Mongod: f.Normal("percona/percona-server-mongodb", mongoVersions), + Pmm: f.Normal("percona/pmm-client", pmmVersions), + Backup: f.Normal("percona/percona-backup-mongodb", pbmVersions), + Operator: f.Normal("percona/percona-server-mongodb-operator", []string{version}), + } + + if err := f.Error(); err != nil { + return nil, err + } + + return matrix, nil +} + +func pxcVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI.VersionMatrix, error) { + pxcVersions, err := productsapi.GetProductVersions("Percona-XtraDB-Cluster-", "Percona-XtraDB-Cluster-80", "Percona-XtraDB-Cluster-57") + if err != nil { + return nil, err + } + + pmmVersions, err := productsapi.GetProductVersions("", "pmm2") + if err != nil { + return nil, err + } + + xtrabackupVersions, err := productsapi.GetProductVersions("Percona-XtraBackup-", "Percona-XtraBackup-8.0", "Percona-XtraBackup-2.4") + if err != nil { + return nil, err + } + matrix := &vsAPI.VersionMatrix{ + Pxc: f.Normal("percona/percona-xtradb-cluster", pxcVersions), + Pmm: f.Normal("percona/pmm-client", pmmVersions), + Proxysql: f.Latest("percona/proxysql"), + Haproxy: f.Latest("percona/haproxy"), + Backup: f.Regex("percona/percona-xtradb-cluster-operator", `(?:^\d+\.\d+\.\d+-pxc\d+\.\d+-backup-pxb)(.*)`, xtrabackupVersions), + LogCollector: f.Regex("percona/percona-xtradb-cluster-operator", `(^.*)(?:-logcollector)`, []string{version}), + Operator: f.Normal("percona/percona-xtradb-cluster-operator", []string{version}), + } + + if err := f.Error(); err != nil { + return nil, err + } + return matrix, nil +} diff --git a/tools/operator-tool/cmd/util.go b/tools/operator-tool/cmd/util.go new file mode 100644 index 00000000..bc4dd953 --- /dev/null +++ b/tools/operator-tool/cmd/util.go @@ -0,0 +1,109 @@ +package main + +import ( + "fmt" + "os" + "reflect" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + gover "github.com/hashicorp/go-version" + "google.golang.org/protobuf/encoding/protojson" +) + +func readBaseFile(path string) (*vsAPI.ProductResponse, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + product := new(vsAPI.ProductResponse) + err = protojson.Unmarshal(content, product) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return product, nil +} + +// deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in the file. +func deleteOldVersions(file string, matrix *vsAPI.VersionMatrix) error { + minVersions, err := getOldestVersions(file) + if err != nil { + return fmt.Errorf("failed to get oldest versions from base file: %w", err) + } + deleteOldVersionsWithMap(matrix, minVersions) + return nil +} + +// deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in oldestVersions. +func deleteOldVersionsWithMap(matrix *vsAPI.VersionMatrix, oldestVersions map[string]*gover.Version) { + matrixType := reflect.TypeOf(matrix).Elem() + matrixValue := reflect.ValueOf(matrix).Elem() + + for i := 0; i < matrixValue.NumField(); i++ { + field := matrixType.Field(i) + // check if value is exported + if field.PkgPath != "" { + continue + } + oldestVersion, ok := oldestVersions[field.Name] + if !ok { + continue + } + + value := matrixValue.Field(i) + + m := value.Interface().(map[string]*vsAPI.Version) + if len(m) == 0 { + continue + } + + for k := range m { + if goversion(k).Compare(oldestVersion) < 0 { + value.SetMapIndex(reflect.ValueOf(k), reflect.Value{}) // delete old version from map + } + } + } +} + +// getOldestVersions returns a map where each key is a struct field name from the VersionMatrix +// of the specified file, and each value is the corresponding oldest version for that field. +func getOldestVersions(filePath string) (map[string]*gover.Version, error) { + prod, err := readBaseFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read base file: %w", err) + } + + matrixType := reflect.TypeOf(prod.Versions[0].Matrix).Elem() + matrixValue := reflect.ValueOf(prod.Versions[0].Matrix).Elem() + + versions := make(map[string]*gover.Version) + for i := 0; i < matrixValue.NumField(); i++ { + field := matrixType.Field(i) + // ignore if value is not exported + if field.PkgPath != "" { + continue + } + versionMapValue := matrixValue.Field(i) + + versionMap := versionMapValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + continue + } + oldestVersion := "" + for k := range versionMap { + if oldestVersion == "" { + oldestVersion = k + continue + } + if goversion(oldestVersion).Compare(goversion(k)) > 0 { + oldestVersion = k + } + } + versions[field.Name] = goversion(oldestVersion) + } + + return versions, nil +} + +func goversion(v string) *gover.Version { + return gover.Must(gover.NewVersion(v)) +} diff --git a/tools/operator-tool/go.mod b/tools/operator-tool/go.mod new file mode 100644 index 00000000..bd4aca86 --- /dev/null +++ b/tools/operator-tool/go.mod @@ -0,0 +1,19 @@ +module operator-tool + +go 1.23.1 + +require github.com/Percona-Lab/percona-version-service v0.0.0-20241013113618-2966a16cabb1 + +replace github.com/Percona-Lab/percona-version-service => ../../ + +require ( + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/hashicorp/go-version v1.7.0 + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/grpc v1.65.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) diff --git a/tools/operator-tool/go.sum b/tools/operator-tool/go.sum new file mode 100644 index 00000000..f5b6db17 --- /dev/null +++ b/tools/operator-tool/go.sum @@ -0,0 +1,20 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/tools/operator-tool/products-api/client.go b/tools/operator-tool/products-api/client.go new file mode 100644 index 00000000..607f6455 --- /dev/null +++ b/tools/operator-tool/products-api/client.go @@ -0,0 +1,102 @@ +package productsapi + +import ( + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + gover "github.com/hashicorp/go-version" +) + +func GetProductVersions(trimPrefix string, products ...string) ([]string, error) { + versions := []string{} + for _, product := range products { + productVersions, err := get(trimPrefix, product) + if err != nil { + return nil, fmt.Errorf("failed to get product versions for %s: %w", product, err) + } + versions = append(versions, productVersions...) + } + if len(versions) == 0 { + return nil, errors.New("not found") + } + return versions, nil +} + +func get(trimPrefix, product string) ([]string, error) { + u := url.URL{ + Scheme: "https", + Host: "www.percona.com", + Path: "products-api.php", + } + values := make(url.Values) + values.Add("version", product) + resp, err := http.PostForm(u.String(), values) + if err != nil { + return nil, fmt.Errorf("failed to make request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("invalid status from docker registry: %s", resp.Status) + } + versions, err := parseXML(resp.Body, trimPrefix) + if err != nil { + return nil, fmt.Errorf("failed to parse XML: %w", err) + } + + versionMap := make(map[string]struct{}) + for _, v := range versions { + versionMap[v] = struct{}{} + ver := gover.Must(gover.NewVersion(v)) + versionMap[ver.Core().String()] = struct{}{} + } + return mapToSlice(versionMap), nil +} + +func mapToSlice(m map[string]struct{}) []string { + s := make([]string, 0, len(m)) + for k := range m { + s = append(s, k) + } + return s +} + +func parseXML(data io.Reader, trimPrefix string) ([]string, error) { + type option struct { + Value string `xml:"value,attr"` + Text string `xml:",chardata"` + } + + var options []option + decoder := xml.NewDecoder(data) + + for { + token, err := decoder.Token() + if err != nil { + break + } + if startElem, ok := token.(xml.StartElement); ok && startElem.Name.Local == "option" { + var option option + if err := decoder.DecodeElement(&option, &startElem); err != nil { + return nil, fmt.Errorf("failed to decode element: %w", err) + } + if option.Value != "" { + options = append(options, option) + } + } + } + + var versions []string + for _, option := range options { + if trimPrefix != "" && !strings.HasPrefix(option.Value, trimPrefix) { + return nil, errors.New("prefix " + trimPrefix + " was not found in versions") + } + versions = append(versions, strings.TrimPrefix(option.Value, trimPrefix)) + } + + return versions, nil +} diff --git a/tools/operator-tool/registry/registry.go b/tools/operator-tool/registry/registry.go new file mode 100644 index 00000000..41135f97 --- /dev/null +++ b/tools/operator-tool/registry/registry.go @@ -0,0 +1,166 @@ +package registry + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" +) + +type tagResp struct { + Count int `json:"count"` + NextPage string `json:"next"` + PreviousPage string `json:"previous"` + Results []tagRespResult `json:"results"` +} + +type tagRespResult struct { + ContentType string `json:"content_type,omitempty"` + Creator float64 `json:"creator,omitempty"` + Digest string `json:"digest,omitempty"` + FullSize float64 `json:"full_size,omitempty"` + ID float64 `json:"id,omitempty"` + Images []struct { + Architecture string `json:"architecture,omitempty"` + Digest string `json:"digest,omitempty"` + Features string `json:"features,omitempty"` + LastPulled string `json:"last_pulled,omitempty"` + LastPushed string `json:"last_pushed,omitempty"` + Os string `json:"os,omitempty"` + OsFeatures string `json:"os_features,omitempty"` + OsVersion string `json:"os_version,omitempty"` + Size float64 `json:"size,omitempty"` + Status string `json:"status,omitempty"` + Variant string `json:"variant,omitempty"` + } `json:"images,omitempty"` + LastUpdated string `json:"last_updated,omitempty"` + LastUpdater float64 `json:"last_updater,omitempty"` + LastUpdaterUsername string `json:"last_updater_username,omitempty"` + MediaType string `json:"media_type,omitempty"` + Name string `json:"name,omitempty"` + Repository float64 `json:"repository,omitempty"` + TagLastPulled string `json:"tag_last_pulled,omitempty"` + TagLastPushed string `json:"tag_last_pushed,omitempty"` + TagStatus string `json:"tag_status,omitempty"` + V2 bool `json:"v2,omitempty"` +} + +func (t tagRespResult) Image(imageName string) Image { + digestAMD64 := "" + digestARM64 := "" + for _, v := range t.Images { + if v.Os != "linux" { + continue + } + if v.Architecture == "amd64" { + digestAMD64 = v.Digest + } + if v.Architecture == "arm64" { + digestARM64 = v.Digest + } + } + return Image{ + Name: imageName, + Tag: t.Name, + DigestAMD64: strings.TrimPrefix(digestAMD64, "sha256:"), + DigestARM64: strings.TrimPrefix(digestARM64, "sha256:"), + } +} + +type Image struct { + Name string + Tag string + DigestAMD64 string + DigestARM64 string +} + +func (i Image) FullName() string { + return i.Name + ":" + i.Tag +} + +type RegistryClient struct { + c *http.Client + cache map[string]tagResp +} + +const defaultPageSize = 100 + +func (r *RegistryClient) get(imageName string, page int) (tagResp, error) { + u := url.URL{ + Scheme: "https", + Host: "registry.hub.docker.com", + Path: "v2/repositories/" + imageName + "/tags", + RawQuery: "page_size=" + strconv.Itoa(defaultPageSize) + "&page=" + strconv.Itoa(page), + } + + result, ok := r.cache[u.String()] + if ok { + return result, nil + } + + resp, err := http.Get(u.String()) + if err != nil { + return result, fmt.Errorf("get: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("invalid status from docker hub registry: %s", resp.Status) + } + + content, err := io.ReadAll(resp.Body) + if err != nil { + return result, fmt.Errorf("failed to read body: %w", err) + } + if err := json.Unmarshal(content, &result); err != nil { + return result, fmt.Errorf("failed to unmarshal: %w", err) + } + r.cache[u.String()] = result + return result, nil +} + +func NewClient() *RegistryClient { + return &RegistryClient{ + c: new(http.Client), + cache: make(map[string]tagResp), + } +} + +func (r *RegistryClient) GetLatestImage(imageName string) (Image, error) { + resp, err := r.get(imageName, 1) + if err != nil { + return Image{}, fmt.Errorf("failed to get latest image: %w", err) + } + for _, result := range resp.Results { + if result.Name == "latest" { + continue + } + if strings.Count(result.Name, ".") == 2 { + return result.Image(imageName), nil + } + } + return Image{}, errors.New("image not found") +} + +func (r *RegistryClient) GetImages(imageName string, filterFunc func(tag string) bool) ([]Image, error) { + images := []Image{} + for page := 1; ; page++ { + resp, err := r.get(imageName, page) + if err != nil { + return nil, fmt.Errorf("failed to get page %d: %w", page, err) + } + for _, result := range resp.Results { + if !filterFunc(result.Name) { + continue + } + + images = append(images, result.Image(imageName)) + } + if resp.NextPage == "" || len(resp.Results) < defaultPageSize { + return images, nil + } + } +} diff --git a/tools/tools.go b/tools/tools.go index d864cddd..b6a80b65 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -10,3 +10,10 @@ import ( _ "google.golang.org/grpc/cmd/protoc-gen-go-grpc" _ "google.golang.org/protobuf/cmd/protoc-gen-go" ) + +//go:generate go build -o ../bin/yq github.com/mikefarah/yq/v3 +//go:generate go build -o ../bin/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go +//go:generate go build -o ../bin/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc +//go:generate go build -o ../bin/protoc-gen-grpc-gateway github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway +//go:generate go build -o ../bin/protoc-gen-openapiv2 github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 +//go:generate bash -c "cd operator-tool && go build -o ../../bin/operator-tool ./cmd" From 06c3504e0a6127d57bbcb5bc8cdd153c93ead6f7 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Mon, 28 Oct 2024 09:07:11 +0200 Subject: [PATCH 02/20] use latest pmm image --- tools/operator-tool/cmd/matrix.go | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/tools/operator-tool/cmd/matrix.go b/tools/operator-tool/cmd/matrix.go index f1ea7db0..82dcf674 100644 --- a/tools/operator-tool/cmd/matrix.go +++ b/tools/operator-tool/cmd/matrix.go @@ -12,17 +12,12 @@ func pgVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI. return nil, err } - pmmVersions, err := productsapi.GetProductVersions("", "pmm2") - if err != nil { - return nil, err - } - matrix := &vsAPI.VersionMatrix{ Postgresql: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-postgres$)`, pgVersions), Pgbackrest: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-pgbackrest)`, pgVersions), Pgbouncer: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-pgbouncer)`, pgVersions), Postgis: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-postgres-gis)`, pgVersions), - Pmm: f.Normal("percona/pmm-client", pmmVersions), + Pmm: f.Latest("percona/pmm-client"), Operator: f.Normal("percona/percona-postgresql-operator", []string{version}), } if err := f.Error(); err != nil { @@ -37,14 +32,9 @@ func psVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI. return nil, err } - pmmVersions, err := productsapi.GetProductVersions("", "pmm2") - if err != nil { - return nil, err - } - matrix := &vsAPI.VersionMatrix{ Mysql: f.Normal("percona/percona-server", psVersions), - Pmm: f.Normal("percona/pmm-client", pmmVersions), + Pmm: f.Latest("percona/pmm-client"), Router: f.Normal("percona/percona-mysql-router", psVersions), Backup: f.Normal("percona/percona-xtrabackup", psVersions), Operator: f.Normal("percona/percona-server-mysql-operator", []string{version}), @@ -65,11 +55,6 @@ func psmdbVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsA return nil, err } - pmmVersions, err := productsapi.GetProductVersions("", "pmm2") - if err != nil { - return nil, err - } - pbmVersions, err := productsapi.GetProductVersions("percona-backup-mongodb-", "percona-backup-mongodb") if err != nil { return nil, err @@ -77,7 +62,7 @@ func psmdbVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsA matrix := &vsAPI.VersionMatrix{ Mongod: f.Normal("percona/percona-server-mongodb", mongoVersions), - Pmm: f.Normal("percona/pmm-client", pmmVersions), + Pmm: f.Latest("percona/pmm-client"), Backup: f.Normal("percona/percona-backup-mongodb", pbmVersions), Operator: f.Normal("percona/percona-server-mongodb-operator", []string{version}), } @@ -95,18 +80,13 @@ func pxcVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI return nil, err } - pmmVersions, err := productsapi.GetProductVersions("", "pmm2") - if err != nil { - return nil, err - } - xtrabackupVersions, err := productsapi.GetProductVersions("Percona-XtraBackup-", "Percona-XtraBackup-8.0", "Percona-XtraBackup-2.4") if err != nil { return nil, err } matrix := &vsAPI.VersionMatrix{ Pxc: f.Normal("percona/percona-xtradb-cluster", pxcVersions), - Pmm: f.Normal("percona/pmm-client", pmmVersions), + Pmm: f.Latest("percona/pmm-client"), Proxysql: f.Latest("percona/proxysql"), Haproxy: f.Latest("percona/haproxy"), Backup: f.Regex("percona/percona-xtradb-cluster-operator", `(?:^\d+\.\d+\.\d+-pxc\d+\.\d+-backup-pxb)(.*)`, xtrabackupVersions), From 221623cb78f0348764ac11cdaa4afb8f11b17e34 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Mon, 28 Oct 2024 12:48:39 +0200 Subject: [PATCH 03/20] update `--operator` option values --- tools/operator-tool/README.md | 10 +++++----- tools/operator-tool/cmd/main.go | 20 +++++++++++--------- tools/operator-tool/cmd/matrix.go | 8 ++++---- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/tools/operator-tool/README.md b/tools/operator-tool/README.md index c8da0697..3de112af 100644 --- a/tools/operator-tool/README.md +++ b/tools/operator-tool/README.md @@ -14,7 +14,7 @@ Usage of ./bin/operator-tool: -file string Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file. -operator string - Operator name. Available values: [psmdb-operator pxc-operator ps-operator pg-operator] + Operator name. Available values: [psmdb pxc ps pg] -verbose Show logs -version string @@ -25,13 +25,13 @@ Usage of ./bin/operator-tool: ### Generating source file from zero ``` -$ ./bin/operator-tool --operator "psmdb-operator" --version "1.17.0" # outputs source file for psmdb-operator +$ ./bin/operator-tool --operator "psmdb" --version "1.17.0" # outputs source file for psmdb-operator ... -$ ./bin/operator-tool --operator "pg-operator" --version "2.5.0" # outputs source file for pg-operator +$ ./bin/operator-tool --operator "pg" --version "2.5.0" # outputs source file for pg-operator ... -$ ./bin/operator-tool --operator "ps-operator" --version "0.8.0" # outputs source file for ps-operator +$ ./bin/operator-tool --operator "ps" --version "0.8.0" # outputs source file for ps-operator ... -$ ./bin/operator-tool --operator "pxc-operator" --version "1.15.1" # outputs source file for pxc-operator +$ ./bin/operator-tool --operator "pxc" --version "1.15.1" # outputs source file for pxc-operator ... ``` diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index 3b6ca0d0..872bdbf9 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -14,10 +14,12 @@ import ( ) const ( - operatorNamePSMDB = "psmdb-operator" - operatorNamePXC = "pxc-operator" - operatorNamePS = "ps-operator" - operatorNamePG = "pg-operator" + operatorNameSuffix = "-operator" + + operatorNamePSMDB = "psmdb" + operatorNamePXC = "pxc" + operatorNamePS = "ps" + operatorNamePG = "pg" ) var validOperatorNames = []string{ @@ -101,13 +103,13 @@ func getProductResponse(operatorName, version string) (*vsAPI.ProductResponse, e } switch operatorName { case operatorNamePG: - versionMatrix, err = pgVersionMatrix(f, operatorName, version) + versionMatrix, err = pgVersionMatrix(f, version) case operatorNamePS: - versionMatrix, err = psVersionMatrix(f, operatorName, version) + versionMatrix, err = psVersionMatrix(f, version) case operatorNamePSMDB: - versionMatrix, err = psmdbVersionMatrix(f, operatorName, version) + versionMatrix, err = psmdbVersionMatrix(f, version) case operatorNamePXC: - versionMatrix, err = pxcVersionMatrix(f, operatorName, version) + versionMatrix, err = pxcVersionMatrix(f, version) default: panic("problems with validation. unknown operator name " + operatorName) } @@ -117,7 +119,7 @@ func getProductResponse(operatorName, version string) (*vsAPI.ProductResponse, e return &vsAPI.ProductResponse{ Versions: []*vsAPI.OperatorVersion{ { - Product: operatorName, + Product: operatorName + operatorNameSuffix, Operator: version, Matrix: versionMatrix, }, diff --git a/tools/operator-tool/cmd/matrix.go b/tools/operator-tool/cmd/matrix.go index 82dcf674..c0678816 100644 --- a/tools/operator-tool/cmd/matrix.go +++ b/tools/operator-tool/cmd/matrix.go @@ -6,7 +6,7 @@ import ( productsapi "operator-tool/products-api" ) -func pgVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI.VersionMatrix, error) { +func pgVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { pgVersions, err := productsapi.GetProductVersions("", "postgresql-distribution-16", "postgresql-distribution-15", "postgresql-distribution-14", "postgresql-distribution-13", "postgresql-distribution-12") if err != nil { return nil, err @@ -26,7 +26,7 @@ func pgVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI. return matrix, nil } -func psVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI.VersionMatrix, error) { +func psVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { psVersions, err := productsapi.GetProductVersions("Percona-Server-", "Percona-Server-8.0") if err != nil { return nil, err @@ -49,7 +49,7 @@ func psVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI. return matrix, nil } -func psmdbVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI.VersionMatrix, error) { +func psmdbVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { mongoVersions, err := productsapi.GetProductVersions("percona-server-mongodb-", "percona-server-mongodb-7.0", "percona-server-mongodb-6.0", "percona-server-mongodb-5.0") if err != nil { return nil, err @@ -74,7 +74,7 @@ func psmdbVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsA return matrix, nil } -func pxcVersionMatrix(f *VersionMapFiller, operatorName, version string) (*vsAPI.VersionMatrix, error) { +func pxcVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { pxcVersions, err := productsapi.GetProductVersions("Percona-XtraDB-Cluster-", "Percona-XtraDB-Cluster-80", "Percona-XtraDB-Cluster-57") if err != nil { return nil, err From cd15072d2d316a86f9cf0b3b8587ae8b20f6ec4c Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Mon, 28 Oct 2024 13:29:26 +0200 Subject: [PATCH 04/20] return error if image was not found --- tools/operator-tool/cmd/filler.go | 3 +++ tools/operator-tool/cmd/main.go | 16 ++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index 2636246b..a1c6d792 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -138,6 +138,9 @@ func getVersionMap(rc *registry.RegistryClient, image string, versions []string) } m[v] = vm } + if len(m) == 0 { + return nil, fmt.Errorf("image %s with %v tags was not found", image, versions) + } return m, nil } diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index 872bdbf9..08ca1741 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -40,21 +40,18 @@ func main() { flag.Parse() if *version == "" { - log.Println("ERROR: --version should be provided") - os.Exit(1) + log.Fatalln("ERROR: --version should be provided") } if *filePath != "" { product, err := readBaseFile(*filePath) if err != nil { - log.Println("ERROR: failed to read base file:", err.Error()) - os.Exit(1) + log.Fatalln("ERROR: failed to read base file:", err.Error()) } *operatorName = product.Versions[0].Product } else { if *operatorName == "" { - log.Println("ERROR: --operator or --file should be provided") - os.Exit(1) + log.Fatalln("ERROR: --operator or --file should be provided") } } @@ -65,12 +62,11 @@ func main() { } if err := printSourceFile(*operatorName, *version, *filePath); err != nil { - log.Println("ERROR: failed to generate source file: ", err.Error()) - os.Exit(1) + log.SetOutput(os.Stderr) + log.Fatalln("ERROR: failed to generate source file:", err.Error()) } default: - log.Printf("ERROR: Unknown operator name: %s. Available values: %v\n", *operatorName, validOperatorNames) - os.Exit(1) + log.Fatalf("ERROR: Unknown operator name: %s. Available values: %v\n", *operatorName, validOperatorNames) } } From 53a8acd2de79eefc125a7b1822a065ff16d8e238 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Mon, 28 Oct 2024 14:31:32 +0200 Subject: [PATCH 05/20] add more versions from docker hub --- tools/operator-tool/cmd/filler.go | 44 +++++++++++++++++++++++- tools/operator-tool/registry/registry.go | 16 +++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index a1c6d792..55e292e0 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -9,6 +9,7 @@ import ( "strings" vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + gover "github.com/hashicorp/go-version" "operator-tool/registry" ) @@ -20,9 +21,13 @@ type VersionMapFiller struct { errs []error } +func (f *VersionMapFiller) addErr(err error) { + f.errs = append(f.errs, err) +} + func (f *VersionMapFiller) exec(vm map[string]*vsAPI.Version, err error) map[string]*vsAPI.Version { if err != nil { - f.errs = append(f.errs, err) + f.addErr(err) return nil } @@ -50,11 +55,48 @@ func (f *VersionMapFiller) setRecommended(vm map[string]*vsAPI.Version) { } } +// addVersionsFromRegistry searches Docker Hub for all tags associated with the specified image +// and appends any missing tags that match the MAJOR.MINOR.PATCH version format to the returned versions slice. +// +// Tags with a "-debug" suffix are excluded. +func (f *VersionMapFiller) addVersionsFromRegistry(image string, versions []string) []string { + wantedVerisons := make(map[string]struct{}, len(versions)) + coreVersions := make(map[string]struct{}) + for _, v := range versions { + wantedVerisons[v] = struct{}{} + coreVersions[goversion(v).Core().String()] = struct{}{} + } + + tags, err := f.RegistryClient.GetTags(image) + if err != nil { + f.addErr(err) + return nil + } + + for _, tag := range tags { + if strings.HasSuffix(tag, "-debug") { + continue + } + if _, err := gover.NewVersion(tag); err != nil { + continue + } + if _, ok := coreVersions[goversion(tag).Core().String()]; !ok { + continue + } + if _, ok := wantedVerisons[tag]; ok { + continue + } + versions = append(versions, tag) + } + return versions +} + // Normal returns a map[string]*Version for the specified image by filtering tags // with the given list of versions. // // The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". func (f *VersionMapFiller) Normal(image string, versions []string) map[string]*vsAPI.Version { + versions = f.addVersionsFromRegistry(image, versions) return f.exec(getVersionMap(f.RegistryClient, image, versions)) } diff --git a/tools/operator-tool/registry/registry.go b/tools/operator-tool/registry/registry.go index 41135f97..6a581b58 100644 --- a/tools/operator-tool/registry/registry.go +++ b/tools/operator-tool/registry/registry.go @@ -145,6 +145,22 @@ func (r *RegistryClient) GetLatestImage(imageName string) (Image, error) { return Image{}, errors.New("image not found") } +func (r *RegistryClient) GetTags(imageName string) ([]string, error) { + tags := []string{} + for page := 1; ; page++ { + resp, err := r.get(imageName, page) + if err != nil { + return nil, fmt.Errorf("failed to get page %d: %w", page, err) + } + for _, result := range resp.Results { + tags = append(tags, result.Name) + } + if resp.NextPage == "" || len(resp.Results) < defaultPageSize { + return tags, nil + } + } +} + func (r *RegistryClient) GetImages(imageName string, filterFunc func(tag string) bool) ([]Image, error) { images := []Image{} for page := 1; ; page++ { From e7c3f6adc4c74cf8ee20904dbcd19d519dc10487 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Tue, 29 Oct 2024 14:32:55 +0200 Subject: [PATCH 06/20] add --patch option --- tools/operator-tool/README.md | 34 ++++- tools/operator-tool/cmd/filler.go | 23 --- tools/operator-tool/cmd/main.go | 47 ++++-- tools/operator-tool/cmd/util.go | 158 ++++++++++++++++++++ tools/operator-tool/patch-file.json.example | 12 ++ tools/operator-tool/registry/registry.go | 69 +++++++-- 6 files changed, 293 insertions(+), 50 deletions(-) create mode 100644 tools/operator-tool/patch-file.json.example diff --git a/tools/operator-tool/README.md b/tools/operator-tool/README.md index 3de112af..e0115785 100644 --- a/tools/operator-tool/README.md +++ b/tools/operator-tool/README.md @@ -8,23 +8,24 @@ Build it using `make init`. ### Help -``` +```sh $ ./bin/operator-tool --help Usage of ./bin/operator-tool: -file string Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file. -operator string Operator name. Available values: [psmdb pxc ps pg] + -patch string + Provide a path to a patch file to add additional images. Must be used together with the --file option. -verbose Show logs -version string Operator version - ``` ### Generating source file from zero -``` +```sh $ ./bin/operator-tool --operator "psmdb" --version "1.17.0" # outputs source file for psmdb-operator ... $ ./bin/operator-tool --operator "pg" --version "2.5.0" # outputs source file for pg-operator @@ -37,7 +38,32 @@ $ ./bin/operator-tool --operator "pxc" --version "1.15.1" # outputs source fil ### Generating source file based on older file -``` +```sh $ ./bin/operator-tool --file ./sources/operator.2.5.0.pg-operator.json --version "1.17.0" # outputs source file for pg-operator, excluding older versions specified in the file ... ``` + +### Patching existing source file with a patch file + +```sh +$ ./bin/operator-tool --file ./sources/operator.2.5.0.pg-operator.json --patch ./tools/operator-tool/patch-file.json.example +... +``` + +*Patch file example:* +The example patch file can be found [here](./patch-file.json.example). + +```json +{ + "operator": { + "2.4.28": { + "image_path": "some-path:tag" + } + }, + "pmm": { + "2.50.1": { + "image_path": "some-path:tag" + } + } +} +``` diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index 55e292e0..a05e7201 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -30,31 +30,9 @@ func (f *VersionMapFiller) exec(vm map[string]*vsAPI.Version, err error) map[str f.addErr(err) return nil } - - f.setRecommended(vm) - return vm } -// setRecommended sets a recommended status to the latest version. -func (f *VersionMapFiller) setRecommended(vm map[string]*vsAPI.Version) { - maxVer := "" - for k := range vm { - if maxVer == "" { - maxVer = k - continue - } - - if goversion(k).Compare(goversion(maxVer)) > 0 { - maxVer = k - } - } - - if _, ok := vm[maxVer]; ok { - vm[maxVer].Status = vsAPI.Status_recommended - } -} - // addVersionsFromRegistry searches Docker Hub for all tags associated with the specified image // and appends any missing tags that match the MAJOR.MINOR.PATCH version format to the returned versions slice. // @@ -252,6 +230,5 @@ func versionMapFromImages(baseTag string, images []registry.Image) (*vsAPI.Versi ImagePath: imagePath, ImageHash: imageHash, ImageHashArm64: imageHashArm64, - Status: vsAPI.Status_available, }, nil } diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index 08ca1741..97d14a75 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -7,6 +7,7 @@ import ( "log" "os" "slices" + "strings" vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" @@ -33,13 +34,14 @@ var ( operatorName = flag.String("operator", "", fmt.Sprintf("Operator name. Available values: %v", validOperatorNames)) version = flag.String("version", "", "Operator version") filePath = flag.String("file", "", "Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file.") + patch = flag.String("patch", "", "Provide a path to a patch file to add additional images. Must be used together with the --file option.") verbose = flag.Bool("verbose", false, "Show logs") ) func main() { flag.Parse() - if *version == "" { + if *version == "" && *patch == "" { log.Fatalln("ERROR: --version should be provided") } @@ -48,11 +50,14 @@ func main() { if err != nil { log.Fatalln("ERROR: failed to read base file:", err.Error()) } - *operatorName = product.Versions[0].Product + *operatorName = strings.TrimSuffix(product.Versions[0].Product, operatorNameSuffix) } else { if *operatorName == "" { log.Fatalln("ERROR: --operator or --file should be provided") } + if *patch != "" { + log.Fatalln("ERROR: --file should be provided when --patch is used") + } } switch { @@ -61,7 +66,7 @@ func main() { log.SetOutput(io.Discard) } - if err := printSourceFile(*operatorName, *version, *filePath); err != nil { + if err := printSourceFile(*operatorName, *version, *filePath, *patch); err != nil { log.SetOutput(os.Stderr) log.Fatalln("ERROR: failed to generate source file:", err.Error()) } @@ -70,18 +75,32 @@ func main() { } } -func printSourceFile(operatorName, version, file string) error { - r, err := getProductResponse(operatorName, version) - if err != nil { - return fmt.Errorf("failed to get product response: %w", err) - } - if file != "" { - if err := deleteOldVersions(file, r.Versions[0].Matrix); err != nil { - return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) +func printSourceFile(operatorName, version, file, patchFile string) error { + var productResponse *vsAPI.ProductResponse + var err error + + registryClient := registry.NewClient() + + if file == "" || patchFile == "" { + productResponse, err = getProductResponse(registryClient, operatorName, version) + if err != nil { + return fmt.Errorf("failed to get product response: %w", err) + } + if file != "" { + if err := deleteOldVersions(file, productResponse.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) + } + } + } else { + productResponse, err = patchProductResponse(registryClient, file, patchFile) + if err != nil { + return fmt.Errorf("failed to patch product response: %w", err) } } - content, err := marshal(r) + updateMatrixStatuses(productResponse.Versions[0].Matrix) + + content, err := marshal(productResponse) if err != nil { return fmt.Errorf("failed to marshal product response: %w", err) } @@ -90,12 +109,12 @@ func printSourceFile(operatorName, version, file string) error { return nil } -func getProductResponse(operatorName, version string) (*vsAPI.ProductResponse, error) { +func getProductResponse(rc *registry.RegistryClient, operatorName, version string) (*vsAPI.ProductResponse, error) { var versionMatrix *vsAPI.VersionMatrix var err error f := &VersionMapFiller{ - RegistryClient: registry.NewClient(), + RegistryClient: rc, } switch operatorName { case operatorNamePG: diff --git a/tools/operator-tool/cmd/util.go b/tools/operator-tool/cmd/util.go index bc4dd953..508f66e4 100644 --- a/tools/operator-tool/cmd/util.go +++ b/tools/operator-tool/cmd/util.go @@ -1,13 +1,17 @@ package main import ( + "encoding/json" "fmt" "os" "reflect" + "strings" vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" gover "github.com/hashicorp/go-version" "google.golang.org/protobuf/encoding/protojson" + + "operator-tool/registry" ) func readBaseFile(path string) (*vsAPI.ProductResponse, error) { @@ -23,6 +27,19 @@ func readBaseFile(path string) (*vsAPI.ProductResponse, error) { return product, nil } +func readPatchFile(path string) (*vsAPI.VersionMatrix, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + matrix := new(vsAPI.VersionMatrix) + err = protojson.Unmarshal(content, matrix) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return matrix, nil +} + // deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in the file. func deleteOldVersions(file string, matrix *vsAPI.VersionMatrix) error { minVersions, err := getOldestVersions(file) @@ -107,3 +124,144 @@ func getOldestVersions(filePath string) (map[string]*gover.Version, error) { func goversion(v string) *gover.Version { return gover.Must(gover.NewVersion(v)) } + +func patchProductResponse(rc *registry.RegistryClient, baseFilepath, patchFilepath string) (*vsAPI.ProductResponse, error) { + baseFile, err := readBaseFile(baseFilepath) + if err != nil { + return nil, fmt.Errorf("failed to read base file: %w", err) + } + patchFile, err := readPatchFile(patchFilepath) + if err != nil { + return nil, fmt.Errorf("failed to read patch file: %w", err) + } + if err := updateMatrixHashes(rc, patchFile); err != nil { + return nil, fmt.Errorf("failed to update patch matrix hashes: %w", err) + } + + matrixToMap := func(matrix *vsAPI.VersionMatrix) (map[string]map[string]map[string]any, error) { + data, err := protojson.Marshal(matrix) + if err != nil { + return nil, fmt.Errorf("failed to marshal: %w", err) + } + + m := make(map[string]map[string]map[string]any) + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return m, nil + } + + baseMatrix, err := matrixToMap(baseFile.Versions[0].Matrix) + if err != nil { + return nil, fmt.Errorf("failed to convert base matrix to map: %w", err) + } + patchMatrix, err := matrixToMap(patchFile) + if err != nil { + return nil, fmt.Errorf("failed to convert patch matrix to map: %w", err) + } + + for product, versions := range patchMatrix { + for version, verInfo := range versions { + if _, ok := baseMatrix[product]; !ok { + baseMatrix[product] = make(map[string]map[string]any) + } + baseMatrix[product][version] = verInfo + } + } + + mapToMatrix := func(m map[string]map[string]map[string]any) (*vsAPI.VersionMatrix, error) { + data, err := json.Marshal(m) + if err != nil { + return nil, fmt.Errorf("failed to marshal: %w", err) + } + + matrix := new(vsAPI.VersionMatrix) + if err := protojson.Unmarshal(data, matrix); err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + + return matrix, nil + } + + baseFile.Versions[0].Matrix, err = mapToMatrix(baseMatrix) + if err != nil { + return nil, fmt.Errorf("failed to convert patched map to matrix: %w", err) + } + return baseFile, nil +} + +func updateMatrixHashes(rc *registry.RegistryClient, matrix *vsAPI.VersionMatrix) error { + matrixType := reflect.TypeOf(matrix).Elem() + matrixValue := reflect.ValueOf(matrix).Elem() + + for i := 0; i < matrixValue.NumField(); i++ { + field := matrixType.Field(i) + // ignore if value is not exported + if field.PkgPath != "" { + continue + } + versionMapValue := matrixValue.Field(i) + + versionMap := versionMapValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + continue + } + + for k, v := range versionMap { + imageSpl := strings.Split(v.ImagePath, ":") + if len(imageSpl) == 1 { + return fmt.Errorf("image %s doesn't have tag", v.ImagePath) + } + tag := imageSpl[len(imageSpl)-1] + imageName := strings.TrimSuffix(v.ImagePath, ":"+tag) + image, err := rc.GetTag(imageName, tag) + if err != nil { + return fmt.Errorf("failed to get tag %s for image %s: %w", tag, imageName, err) + } + versionMap[k].ImageHash = image.DigestAMD64 + versionMap[k].ImageHashArm64 = image.DigestARM64 + } + } + return nil +} + +func updateMatrixStatuses(matrix *vsAPI.VersionMatrix) error { + matrixType := reflect.TypeOf(matrix).Elem() + matrixValue := reflect.ValueOf(matrix).Elem() + + for i := 0; i < matrixValue.NumField(); i++ { + field := matrixType.Field(i) + // ignore if value is not exported + if field.PkgPath != "" { + continue + } + versionMapValue := matrixValue.Field(i) + + versionMap := versionMapValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + continue + } + setStatus(versionMap) + } + return nil +} + +// setStatus sets a recommended status to the latest version and available status to other versions. +func setStatus(vm map[string]*vsAPI.Version) { + maxVer := "" + for k := range vm { + vm[k].Status = vsAPI.Status_available + if maxVer == "" { + maxVer = k + continue + } + + if goversion(k).Compare(goversion(maxVer)) > 0 { + maxVer = k + } + } + + if _, ok := vm[maxVer]; ok { + vm[maxVer].Status = vsAPI.Status_recommended + } +} diff --git a/tools/operator-tool/patch-file.json.example b/tools/operator-tool/patch-file.json.example new file mode 100644 index 00000000..ddce8895 --- /dev/null +++ b/tools/operator-tool/patch-file.json.example @@ -0,0 +1,12 @@ +{ + "backup": { + "2.4.28": { + "image_path": "some-path:tag" + } + }, + "pmm": { + "2.50.1": { + "image_path": "some-path:tag" + } + } +} diff --git a/tools/operator-tool/registry/registry.go b/tools/operator-tool/registry/registry.go index 6a581b58..46e08abb 100644 --- a/tools/operator-tool/registry/registry.go +++ b/tools/operator-tool/registry/registry.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/url" + "path" "strconv" "strings" ) @@ -84,22 +85,64 @@ func (i Image) FullName() string { type RegistryClient struct { c *http.Client - cache map[string]tagResp + cache map[string]any } const defaultPageSize = 100 -func (r *RegistryClient) get(imageName string, page int) (tagResp, error) { +func (r *RegistryClient) readTag(imageName string, tag string) (tagRespResult, error) { + u := url.URL{ + Scheme: "https", + Host: "registry.hub.docker.com", + Path: path.Join("v2", "repositories", imageName, "tags", tag), + } + + var result tagRespResult + cachedResult, ok := r.cache[u.String()] + if ok { + result, ok = cachedResult.(tagRespResult) + if ok { + return result, nil + } + panic("caching error") + } + + resp, err := http.Get(u.String()) + if err != nil { + return result, fmt.Errorf("get: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("invalid status from docker hub registry: %s", resp.Status) + } + + content, err := io.ReadAll(resp.Body) + if err != nil { + return result, fmt.Errorf("failed to read body: %w", err) + } + if err := json.Unmarshal(content, &result); err != nil { + return result, fmt.Errorf("failed to unmarshal: %w", err) + } + r.cache[u.String()] = result + return result, nil +} + +func (r *RegistryClient) listTags(imageName string, page int) (tagResp, error) { u := url.URL{ Scheme: "https", Host: "registry.hub.docker.com", - Path: "v2/repositories/" + imageName + "/tags", + Path: path.Join("v2", "repositories", imageName, "tags"), RawQuery: "page_size=" + strconv.Itoa(defaultPageSize) + "&page=" + strconv.Itoa(page), } - result, ok := r.cache[u.String()] + var result tagResp + cachedResult, ok := r.cache[u.String()] if ok { - return result, nil + result, ok = cachedResult.(tagResp) + if ok { + return result, nil + } + panic("caching error") } resp, err := http.Get(u.String()) @@ -125,12 +168,12 @@ func (r *RegistryClient) get(imageName string, page int) (tagResp, error) { func NewClient() *RegistryClient { return &RegistryClient{ c: new(http.Client), - cache: make(map[string]tagResp), + cache: make(map[string]any), } } func (r *RegistryClient) GetLatestImage(imageName string) (Image, error) { - resp, err := r.get(imageName, 1) + resp, err := r.listTags(imageName, 1) if err != nil { return Image{}, fmt.Errorf("failed to get latest image: %w", err) } @@ -145,10 +188,18 @@ func (r *RegistryClient) GetLatestImage(imageName string) (Image, error) { return Image{}, errors.New("image not found") } +func (r *RegistryClient) GetTag(image, tag string) (Image, error) { + resp, err := r.readTag(image, tag) + if err != nil { + return Image{}, fmt.Errorf("failed to read tag: %w", err) + } + return resp.Image(image), nil +} + func (r *RegistryClient) GetTags(imageName string) ([]string, error) { tags := []string{} for page := 1; ; page++ { - resp, err := r.get(imageName, page) + resp, err := r.listTags(imageName, page) if err != nil { return nil, fmt.Errorf("failed to get page %d: %w", page, err) } @@ -164,7 +215,7 @@ func (r *RegistryClient) GetTags(imageName string) ([]string, error) { func (r *RegistryClient) GetImages(imageName string, filterFunc func(tag string) bool) ([]Image, error) { images := []Image{} for page := 1; ; page++ { - resp, err := r.get(imageName, page) + resp, err := r.listTags(imageName, page) if err != nil { return nil, fmt.Errorf("failed to get page %d: %w", page, err) } From b345ec2b37eac072bd62395b4aa9604d81415dec Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 31 Oct 2024 09:20:03 +0200 Subject: [PATCH 07/20] fix adding versions from registry --- tools/operator-tool/cmd/filler.go | 6 ++++-- tools/operator-tool/cmd/matrix.go | 20 ++++++++++---------- tools/operator-tool/go.mod | 8 +++++--- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index a05e7201..7a294e33 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -73,8 +73,10 @@ func (f *VersionMapFiller) addVersionsFromRegistry(image string, versions []stri // with the given list of versions. // // The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". -func (f *VersionMapFiller) Normal(image string, versions []string) map[string]*vsAPI.Version { - versions = f.addVersionsFromRegistry(image, versions) +func (f *VersionMapFiller) Normal(image string, versions []string, addVersionsFromRegistry bool) map[string]*vsAPI.Version { + if addVersionsFromRegistry { + versions = f.addVersionsFromRegistry(image, versions) + } return f.exec(getVersionMap(f.RegistryClient, image, versions)) } diff --git a/tools/operator-tool/cmd/matrix.go b/tools/operator-tool/cmd/matrix.go index c0678816..cdabc367 100644 --- a/tools/operator-tool/cmd/matrix.go +++ b/tools/operator-tool/cmd/matrix.go @@ -18,7 +18,7 @@ func pgVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, Pgbouncer: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-pgbouncer)`, pgVersions), Postgis: f.Regex("percona/percona-postgresql-operator", `(?:^\d+\.\d+\.\d+-ppg)(\d+\.\d+)(?:-postgres-gis)`, pgVersions), Pmm: f.Latest("percona/pmm-client"), - Operator: f.Normal("percona/percona-postgresql-operator", []string{version}), + Operator: f.Normal("percona/percona-postgresql-operator", []string{version}, false), } if err := f.Error(); err != nil { return nil, err @@ -33,11 +33,11 @@ func psVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, } matrix := &vsAPI.VersionMatrix{ - Mysql: f.Normal("percona/percona-server", psVersions), + Mysql: f.Normal("percona/percona-server", psVersions, true), Pmm: f.Latest("percona/pmm-client"), - Router: f.Normal("percona/percona-mysql-router", psVersions), - Backup: f.Normal("percona/percona-xtrabackup", psVersions), - Operator: f.Normal("percona/percona-server-mysql-operator", []string{version}), + Router: f.Normal("percona/percona-mysql-router", psVersions, true), + Backup: f.Normal("percona/percona-xtrabackup", psVersions, true), + Operator: f.Normal("percona/percona-server-mysql-operator", []string{version}, false), Haproxy: f.Latest("percona/haproxy"), Orchestrator: f.Latest("percona/percona-orchestrator"), Toolkit: f.Latest("percona/percona-toolkit"), @@ -61,10 +61,10 @@ func psmdbVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatr } matrix := &vsAPI.VersionMatrix{ - Mongod: f.Normal("percona/percona-server-mongodb", mongoVersions), + Mongod: f.Normal("percona/percona-server-mongodb", mongoVersions, true), Pmm: f.Latest("percona/pmm-client"), - Backup: f.Normal("percona/percona-backup-mongodb", pbmVersions), - Operator: f.Normal("percona/percona-server-mongodb-operator", []string{version}), + Backup: f.Normal("percona/percona-backup-mongodb", pbmVersions, true), + Operator: f.Normal("percona/percona-server-mongodb-operator", []string{version}, false), } if err := f.Error(); err != nil { @@ -85,13 +85,13 @@ func pxcVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix return nil, err } matrix := &vsAPI.VersionMatrix{ - Pxc: f.Normal("percona/percona-xtradb-cluster", pxcVersions), + Pxc: f.Normal("percona/percona-xtradb-cluster", pxcVersions, true), Pmm: f.Latest("percona/pmm-client"), Proxysql: f.Latest("percona/proxysql"), Haproxy: f.Latest("percona/haproxy"), Backup: f.Regex("percona/percona-xtradb-cluster-operator", `(?:^\d+\.\d+\.\d+-pxc\d+\.\d+-backup-pxb)(.*)`, xtrabackupVersions), LogCollector: f.Regex("percona/percona-xtradb-cluster-operator", `(^.*)(?:-logcollector)`, []string{version}), - Operator: f.Normal("percona/percona-xtradb-cluster-operator", []string{version}), + Operator: f.Normal("percona/percona-xtradb-cluster-operator", []string{version}, false), } if err := f.Error(); err != nil { diff --git a/tools/operator-tool/go.mod b/tools/operator-tool/go.mod index bd4aca86..52f71a26 100644 --- a/tools/operator-tool/go.mod +++ b/tools/operator-tool/go.mod @@ -2,18 +2,20 @@ module operator-tool go 1.23.1 -require github.com/Percona-Lab/percona-version-service v0.0.0-20241013113618-2966a16cabb1 +require ( + github.com/Percona-Lab/percona-version-service v0.0.0-20241013113618-2966a16cabb1 + github.com/hashicorp/go-version v1.7.0 + google.golang.org/protobuf v1.34.2 +) replace github.com/Percona-Lab/percona-version-service => ../../ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect - github.com/hashicorp/go-version v1.7.0 golang.org/x/net v0.26.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/grpc v1.65.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect ) From bf190054fac79e1af542dcac7d9a73d1ca92d886 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 31 Oct 2024 09:23:13 +0200 Subject: [PATCH 08/20] move packages to pkg --- tools/operator-tool/cmd/filler.go | 2 +- tools/operator-tool/cmd/main.go | 2 +- tools/operator-tool/cmd/matrix.go | 2 +- tools/operator-tool/cmd/util.go | 2 +- tools/operator-tool/{ => pkg}/products-api/client.go | 0 tools/operator-tool/{ => pkg}/registry/registry.go | 0 6 files changed, 4 insertions(+), 4 deletions(-) rename tools/operator-tool/{ => pkg}/products-api/client.go (100%) rename tools/operator-tool/{ => pkg}/registry/registry.go (100%) diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index 7a294e33..18291d4d 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -11,7 +11,7 @@ import ( vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" gover "github.com/hashicorp/go-version" - "operator-tool/registry" + "operator-tool/pkg/registry" ) // VersionMapFiller is a helper type for creating a map[string]*vsAPI.Version diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index 97d14a75..7027ee38 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -11,7 +11,7 @@ import ( vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" - "operator-tool/registry" + "operator-tool/pkg/registry" ) const ( diff --git a/tools/operator-tool/cmd/matrix.go b/tools/operator-tool/cmd/matrix.go index cdabc367..c8e8e18c 100644 --- a/tools/operator-tool/cmd/matrix.go +++ b/tools/operator-tool/cmd/matrix.go @@ -3,7 +3,7 @@ package main import ( vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" - productsapi "operator-tool/products-api" + productsapi "operator-tool/pkg/products-api" ) func pgVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { diff --git a/tools/operator-tool/cmd/util.go b/tools/operator-tool/cmd/util.go index 508f66e4..f7a9eeca 100644 --- a/tools/operator-tool/cmd/util.go +++ b/tools/operator-tool/cmd/util.go @@ -11,7 +11,7 @@ import ( gover "github.com/hashicorp/go-version" "google.golang.org/protobuf/encoding/protojson" - "operator-tool/registry" + "operator-tool/pkg/registry" ) func readBaseFile(path string) (*vsAPI.ProductResponse, error) { diff --git a/tools/operator-tool/products-api/client.go b/tools/operator-tool/pkg/products-api/client.go similarity index 100% rename from tools/operator-tool/products-api/client.go rename to tools/operator-tool/pkg/products-api/client.go diff --git a/tools/operator-tool/registry/registry.go b/tools/operator-tool/pkg/registry/registry.go similarity index 100% rename from tools/operator-tool/registry/registry.go rename to tools/operator-tool/pkg/registry/registry.go From 09339b3402be51ce9d15c78cc83a54d3a2507a88 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 31 Oct 2024 13:41:44 +0200 Subject: [PATCH 09/20] fix bugs --- tools/operator-tool/cmd/filler.go | 148 ++++++++++++++++++++---------- tools/operator-tool/cmd/matrix.go | 2 +- tools/operator-tool/cmd/util.go | 26 +++--- 3 files changed, 114 insertions(+), 62 deletions(-) diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index 18291d4d..213eaf6f 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -14,6 +14,13 @@ import ( "operator-tool/pkg/registry" ) +var archSuffixes = []string{ + "-arm64", + "-aarch64", + "-multi", + "-amd64", +} + // VersionMapFiller is a helper type for creating a map[string]*vsAPI.Version // using information retrieved from Docker Hub. type VersionMapFiller struct { @@ -51,8 +58,20 @@ func (f *VersionMapFiller) addVersionsFromRegistry(image string, versions []stri return nil } + // getVersionMap will search for images with these suffixes. We don't need them in this function + ignoredSuffixes := append(archSuffixes, "-debug") + + hasIgnoredSuffix := func(tag string) bool { + for _, s := range ignoredSuffixes { + if strings.HasSuffix(tag, s) { + return true + } + } + return false + } + for _, tag := range tags { - if strings.HasSuffix(tag, "-debug") { + if hasIgnoredSuffix(tag) { continue } if _, err := gover.NewVersion(tag); err != nil { @@ -130,7 +149,9 @@ func getVersionMapRegex(rc *registry.RegistryClient, image string, regex string, if err != nil { return nil, err } - m[v] = vm + for v, versionMap := range vm { + m[v] = versionMap + } } return m, nil } @@ -139,9 +160,17 @@ func getVersionMap(rc *registry.RegistryClient, image string, versions []string) m := make(map[string]*vsAPI.Version) for _, v := range versions { images, err := rc.GetImages(image, func(tag string) bool { - allowedSuffixes := []string{"", "-amd64", "-arm64", "-multi"} + allowedSuffixes := append(archSuffixes, "") for _, s := range allowedSuffixes { - if tag+s == v { + tagWithoutSuffix := tag + if s != "" { + var found bool + tagWithoutSuffix, found = strings.CutSuffix(tag, s) + if !found { + continue + } + } + if tagWithoutSuffix == v { return true } } @@ -158,7 +187,9 @@ func getVersionMap(rc *registry.RegistryClient, image string, versions []string) if err != nil { return nil, err } - m[v] = vm + for v, versionMap := range vm { + m[v] = versionMap + } } if len(m) == 0 { return nil, fmt.Errorf("image %s with %v tags was not found", image, versions) @@ -175,62 +206,81 @@ func getVersionMapLatestVer(rc *registry.RegistryClient, imageName string) (map[ if err != nil { return nil, err } - return map[string]*vsAPI.Version{ - image.Tag: vm, - }, nil + + return vm, nil } -// versionMapFromImages returns a Version for a given list of images and a base tag without any suffixes. +// versionMapFromImages returns a Version map for a given list of images and a base tag without any suffixes. // -// Some images on Docker Hub are tagged like , -arm64, -amd64, and -multi. -// This function attempts to use information from images with both amd64 and arm64 builds. If both are not available, it defaults to amd64. +// Some images on Docker Hub are tagged like , -arm64, -aarch64, -amd64, and -multi. +// This function adds images with amd64 and arm64 builds to the provided map. // -// If multiple provided images share the same suffix, the function returns a Version with information for the latest image. -func versionMapFromImages(baseTag string, images []registry.Image) (*vsAPI.Version, error) { +// Logic: +// - If an image supports both amd64 and arm64 architectures and has a "-multi" suffix in its tag, +// the function includes a version of the image tag without the "-multi" suffix in the map. +// - If no image with both amd64 and arm64 builds is found, separate images for amd64 and arm64 +// are added individually. +func versionMapFromImages(baseTag string, images []registry.Image) (map[string]*vsAPI.Version, error) { slices.SortFunc(images, func(a, b registry.Image) int { return goversion(b.Tag).Compare(goversion(a.Tag)) }) - imageName := images[0].Name + var multiImage, amd64Image, arm64Image *registry.Image for _, image := range images { - if strings.HasSuffix(image.Tag, "-arm64") { - arm64Image = &image - continue - } - if multiImage == nil { - if (image.DigestAMD64 != "" && image.DigestARM64 != "") || strings.HasSuffix(image.Tag, "-multi") { + switch { + case image.DigestARM64 == "" && image.DigestAMD64 == "": + case image.DigestARM64 != "" && image.DigestAMD64 != "": + if image.Tag == baseTag || multiImage == nil { multiImage = &image - continue + } + case image.DigestARM64 != "": + if image.Tag == baseTag || arm64Image == nil { + arm64Image = &image + } + case image.DigestAMD64 != "": + if image.Tag == baseTag || amd64Image == nil { + amd64Image = &image } } - if image.Tag == baseTag || amd64Image == nil { - amd64Image = &image - continue + } + + extractSuffix := func(tag string) string { + for _, suffix := range archSuffixes { + if strings.HasSuffix(tag, suffix) { + return suffix + } } + return "" } - var imagePath, imageHash, imageHashArm64 string - - switch { - case multiImage != nil: - imagePath = multiImage.FullName() - imageHash = multiImage.DigestAMD64 - imageHashArm64 = multiImage.DigestARM64 - case amd64Image != nil && arm64Image != nil: - log.Printf("WARNING: Image %s has both %s and %s tags, but doesn't have \"-multi\" tag. Using %s\n", imageName, amd64Image, arm64Image, amd64Image) - fallthrough - case amd64Image != nil: - imagePath = amd64Image.FullName() - imageHash = amd64Image.DigestAMD64 - case arm64Image != nil: - imagePath = arm64Image.FullName() - imageHashArm64 = arm64Image.DigestARM64 - default: - return nil, fmt.Errorf("necessary tags for %s image were not found", imageName) - } - - return &vsAPI.Version{ - ImagePath: imagePath, - ImageHash: imageHash, - ImageHashArm64: imageHashArm64, - }, nil + + if multiImage == nil && amd64Image == nil && arm64Image == nil { + return nil, fmt.Errorf("necessary tags for %s image were not found", images[0].Name) + } + + versions := make(map[string]*vsAPI.Version) + if multiImage != nil { + versions[baseTag+extractSuffix(multiImage.Tag)] = &vsAPI.Version{ + ImagePath: multiImage.FullName(), + ImageHash: multiImage.DigestAMD64, + ImageHashArm64: multiImage.DigestARM64, + } + if multiImage.Tag == baseTag { + return versions, nil + } + } + if amd64Image != nil { + versions[baseTag+extractSuffix(amd64Image.Tag)] = &vsAPI.Version{ + ImagePath: amd64Image.FullName(), + ImageHash: amd64Image.DigestAMD64, + } + } + // Include arm64 if multi image is not specified + if multiImage == nil && arm64Image != nil { + versions[baseTag+extractSuffix(arm64Image.Tag)] = &vsAPI.Version{ + ImagePath: arm64Image.FullName(), + ImageHashArm64: arm64Image.DigestARM64, + } + } + + return versions, nil } diff --git a/tools/operator-tool/cmd/matrix.go b/tools/operator-tool/cmd/matrix.go index c8e8e18c..36033021 100644 --- a/tools/operator-tool/cmd/matrix.go +++ b/tools/operator-tool/cmd/matrix.go @@ -87,7 +87,7 @@ func pxcVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix matrix := &vsAPI.VersionMatrix{ Pxc: f.Normal("percona/percona-xtradb-cluster", pxcVersions, true), Pmm: f.Latest("percona/pmm-client"), - Proxysql: f.Latest("percona/proxysql"), + Proxysql: f.Latest("percona/proxysql2"), Haproxy: f.Latest("percona/haproxy"), Backup: f.Regex("percona/percona-xtradb-cluster-operator", `(?:^\d+\.\d+\.\d+-pxc\d+\.\d+-backup-pxb)(.*)`, xtrabackupVersions), LogCollector: f.Regex("percona/percona-xtradb-cluster-operator", `(^.*)(?:-logcollector)`, []string{version}), diff --git a/tools/operator-tool/cmd/util.go b/tools/operator-tool/cmd/util.go index f7a9eeca..0fe4dfb1 100644 --- a/tools/operator-tool/cmd/util.go +++ b/tools/operator-tool/cmd/util.go @@ -246,22 +246,24 @@ func updateMatrixStatuses(matrix *vsAPI.VersionMatrix) error { return nil } -// setStatus sets a recommended status to the latest version and available status to other versions. +// setStatus updates the statuses of version map. +// For each major version, it sets the highest version as "recommended" +// and all other versions as "available". func setStatus(vm map[string]*vsAPI.Version) { - maxVer := "" - for k := range vm { - vm[k].Status = vsAPI.Status_available - if maxVer == "" { - maxVer = k - continue - } + highestVersions := make(map[int]string) + for version := range vm { + vm[version].Status = vsAPI.Status_available + + majorVersion := goversion(version).Segments()[0] + + currentHighestVersion, ok := highestVersions[majorVersion] - if goversion(k).Compare(goversion(maxVer)) > 0 { - maxVer = k + if !ok || goversion(version).Compare(goversion(currentHighestVersion)) > 0 { + highestVersions[majorVersion] = version } } - if _, ok := vm[maxVer]; ok { - vm[maxVer].Status = vsAPI.Status_recommended + for _, version := range highestVersions { + vm[version].Status = vsAPI.Status_recommended } } From 42d83e04347902534d6efc26e9dd8f2d4b238d64 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 31 Oct 2024 14:29:31 +0200 Subject: [PATCH 10/20] refactor util.go --- tools/operator-tool/cmd/util.go | 89 +++++++++++++-------------------- 1 file changed, 34 insertions(+), 55 deletions(-) diff --git a/tools/operator-tool/cmd/util.go b/tools/operator-tool/cmd/util.go index 0fe4dfb1..6742626e 100644 --- a/tools/operator-tool/cmd/util.go +++ b/tools/operator-tool/cmd/util.go @@ -50,8 +50,7 @@ func deleteOldVersions(file string, matrix *vsAPI.VersionMatrix) error { return nil } -// deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in oldestVersions. -func deleteOldVersionsWithMap(matrix *vsAPI.VersionMatrix, oldestVersions map[string]*gover.Version) { +func iterateOverMatrixFields(matrix *vsAPI.VersionMatrix, f func(fieldName string, fieldValue reflect.Value) error) error { matrixType := reflect.TypeOf(matrix).Elem() matrixValue := reflect.ValueOf(matrix).Elem() @@ -61,24 +60,33 @@ func deleteOldVersionsWithMap(matrix *vsAPI.VersionMatrix, oldestVersions map[st if field.PkgPath != "" { continue } - oldestVersion, ok := oldestVersions[field.Name] - if !ok { - continue + if err := f(field.Name, matrixValue.Field(i)); err != nil { + return err } + } + return nil +} - value := matrixValue.Field(i) +// deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in oldestVersions. +func deleteOldVersionsWithMap(matrix *vsAPI.VersionMatrix, oldestVersions map[string]*gover.Version) { + iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { + oldestVersion, ok := oldestVersions[fieldName] + if !ok { + return nil + } - m := value.Interface().(map[string]*vsAPI.Version) + m := fieldValue.Interface().(map[string]*vsAPI.Version) if len(m) == 0 { - continue + return nil } for k := range m { if goversion(k).Compare(oldestVersion) < 0 { - value.SetMapIndex(reflect.ValueOf(k), reflect.Value{}) // delete old version from map + fieldValue.SetMapIndex(reflect.ValueOf(k), reflect.Value{}) // delete old version from map } } - } + return nil + }) } // getOldestVersions returns a map where each key is a struct field name from the VersionMatrix @@ -89,21 +97,11 @@ func getOldestVersions(filePath string) (map[string]*gover.Version, error) { return nil, fmt.Errorf("failed to read base file: %w", err) } - matrixType := reflect.TypeOf(prod.Versions[0].Matrix).Elem() - matrixValue := reflect.ValueOf(prod.Versions[0].Matrix).Elem() - versions := make(map[string]*gover.Version) - for i := 0; i < matrixValue.NumField(); i++ { - field := matrixType.Field(i) - // ignore if value is not exported - if field.PkgPath != "" { - continue - } - versionMapValue := matrixValue.Field(i) - - versionMap := versionMapValue.Interface().(map[string]*vsAPI.Version) + iterateOverMatrixFields(prod.Versions[0].Matrix, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) if len(versionMap) == 0 { - continue + return nil } oldestVersion := "" for k := range versionMap { @@ -115,8 +113,9 @@ func getOldestVersions(filePath string) (map[string]*gover.Version, error) { oldestVersion = k } } - versions[field.Name] = goversion(oldestVersion) - } + versions[fieldName] = goversion(oldestVersion) + return nil + }) return versions, nil } @@ -191,20 +190,10 @@ func patchProductResponse(rc *registry.RegistryClient, baseFilepath, patchFilepa } func updateMatrixHashes(rc *registry.RegistryClient, matrix *vsAPI.VersionMatrix) error { - matrixType := reflect.TypeOf(matrix).Elem() - matrixValue := reflect.ValueOf(matrix).Elem() - - for i := 0; i < matrixValue.NumField(); i++ { - field := matrixType.Field(i) - // ignore if value is not exported - if field.PkgPath != "" { - continue - } - versionMapValue := matrixValue.Field(i) - - versionMap := versionMapValue.Interface().(map[string]*vsAPI.Version) + return iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) if len(versionMap) == 0 { - continue + return nil } for k, v := range versionMap { @@ -221,29 +210,19 @@ func updateMatrixHashes(rc *registry.RegistryClient, matrix *vsAPI.VersionMatrix versionMap[k].ImageHash = image.DigestAMD64 versionMap[k].ImageHashArm64 = image.DigestARM64 } - } - return nil + return nil + }) } func updateMatrixStatuses(matrix *vsAPI.VersionMatrix) error { - matrixType := reflect.TypeOf(matrix).Elem() - matrixValue := reflect.ValueOf(matrix).Elem() - - for i := 0; i < matrixValue.NumField(); i++ { - field := matrixType.Field(i) - // ignore if value is not exported - if field.PkgPath != "" { - continue - } - versionMapValue := matrixValue.Field(i) - - versionMap := versionMapValue.Interface().(map[string]*vsAPI.Version) + return iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) if len(versionMap) == 0 { - continue + return nil } setStatus(versionMap) - } - return nil + return nil + }) } // setStatus updates the statuses of version map. From 267c2a86b580b7621ea3b05d9adb343e3bcef545 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Mon, 4 Nov 2024 11:49:30 +0200 Subject: [PATCH 11/20] add `-include-arch-images` option --- tools/operator-tool/README.md | 6 ++++-- tools/operator-tool/cmd/filler.go | 14 +++++++++----- tools/operator-tool/cmd/main.go | 22 ++++++++++++---------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/tools/operator-tool/README.md b/tools/operator-tool/README.md index e0115785..9ccdb310 100644 --- a/tools/operator-tool/README.md +++ b/tools/operator-tool/README.md @@ -12,11 +12,13 @@ Build it using `make init`. $ ./bin/operator-tool --help Usage of ./bin/operator-tool: -file string - Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file. + Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file + -include-arch-images + Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file -operator string Operator name. Available values: [psmdb pxc ps pg] -patch string - Provide a path to a patch file to add additional images. Must be used together with the --file option. + Provide a path to a patch file to add additional images. Must be used together with the --file option -verbose Show logs -version string diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index 213eaf6f..a094bb3a 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -24,8 +24,9 @@ var archSuffixes = []string{ // VersionMapFiller is a helper type for creating a map[string]*vsAPI.Version // using information retrieved from Docker Hub. type VersionMapFiller struct { - RegistryClient *registry.RegistryClient - errs []error + RegistryClient *registry.RegistryClient + errs []error + includeArchSuffixes bool } func (f *VersionMapFiller) addErr(err error) { @@ -96,7 +97,7 @@ func (f *VersionMapFiller) Normal(image string, versions []string, addVersionsFr if addVersionsFromRegistry { versions = f.addVersionsFromRegistry(image, versions) } - return f.exec(getVersionMap(f.RegistryClient, image, versions)) + return f.exec(getVersionMap(f.RegistryClient, image, versions, f.includeArchSuffixes)) } // Regex returns a map[string]*Version for the specified image by filtering tags @@ -156,11 +157,14 @@ func getVersionMapRegex(rc *registry.RegistryClient, image string, regex string, return m, nil } -func getVersionMap(rc *registry.RegistryClient, image string, versions []string) (map[string]*vsAPI.Version, error) { +func getVersionMap(rc *registry.RegistryClient, image string, versions []string, includeArchSuffixes bool) (map[string]*vsAPI.Version, error) { m := make(map[string]*vsAPI.Version) for _, v := range versions { images, err := rc.GetImages(image, func(tag string) bool { - allowedSuffixes := append(archSuffixes, "") + allowedSuffixes := []string{""} + if includeArchSuffixes { + allowedSuffixes = append(allowedSuffixes, archSuffixes...) + } for _, s := range allowedSuffixes { tagWithoutSuffix := tag if s != "" { diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index 7027ee38..1466fdfd 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -31,11 +31,12 @@ var validOperatorNames = []string{ } var ( - operatorName = flag.String("operator", "", fmt.Sprintf("Operator name. Available values: %v", validOperatorNames)) - version = flag.String("version", "", "Operator version") - filePath = flag.String("file", "", "Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file.") - patch = flag.String("patch", "", "Provide a path to a patch file to add additional images. Must be used together with the --file option.") - verbose = flag.Bool("verbose", false, "Show logs") + operatorName = flag.String("operator", "", fmt.Sprintf("Operator name. Available values: %v", validOperatorNames)) + version = flag.String("version", "", "Operator version") + filePath = flag.String("file", "", "Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file") + patch = flag.String("patch", "", "Provide a path to a patch file to add additional images. Must be used together with the --file option") + verbose = flag.Bool("verbose", false, "Show logs") + includeMultiImages = flag.Bool("include-arch-images", false, `Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file`) ) func main() { @@ -66,7 +67,7 @@ func main() { log.SetOutput(io.Discard) } - if err := printSourceFile(*operatorName, *version, *filePath, *patch); err != nil { + if err := printSourceFile(*operatorName, *version, *filePath, *patch, *includeMultiImages); err != nil { log.SetOutput(os.Stderr) log.Fatalln("ERROR: failed to generate source file:", err.Error()) } @@ -75,14 +76,14 @@ func main() { } } -func printSourceFile(operatorName, version, file, patchFile string) error { +func printSourceFile(operatorName, version, file, patchFile string, includeArchSuffixes bool) error { var productResponse *vsAPI.ProductResponse var err error registryClient := registry.NewClient() if file == "" || patchFile == "" { - productResponse, err = getProductResponse(registryClient, operatorName, version) + productResponse, err = getProductResponse(registryClient, operatorName, version, includeArchSuffixes) if err != nil { return fmt.Errorf("failed to get product response: %w", err) } @@ -109,12 +110,13 @@ func printSourceFile(operatorName, version, file, patchFile string) error { return nil } -func getProductResponse(rc *registry.RegistryClient, operatorName, version string) (*vsAPI.ProductResponse, error) { +func getProductResponse(rc *registry.RegistryClient, operatorName, version string, includeArchSuffixes bool) (*vsAPI.ProductResponse, error) { var versionMatrix *vsAPI.VersionMatrix var err error f := &VersionMapFiller{ - RegistryClient: rc, + RegistryClient: rc, + includeArchSuffixes: includeArchSuffixes, } switch operatorName { case operatorNamePG: From e86480f83520977e0d0def6b64bed983ca2aba9e Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Mon, 4 Nov 2024 12:55:12 +0200 Subject: [PATCH 12/20] prefer prerelease versions --- tools/operator-tool/cmd/filler.go | 64 +++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index a094bb3a..490c19c8 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -93,13 +93,53 @@ func (f *VersionMapFiller) addVersionsFromRegistry(image string, versions []stri // with the given list of versions. // // The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". +// Prerelease versions are preferred for each core version when available. See preferPrereleaseVersionsFilter function. func (f *VersionMapFiller) Normal(image string, versions []string, addVersionsFromRegistry bool) map[string]*vsAPI.Version { if addVersionsFromRegistry { versions = f.addVersionsFromRegistry(image, versions) } + + versions = preferPrereleaseVersionsFilter(versions) + return f.exec(getVersionMap(f.RegistryClient, image, versions, f.includeArchSuffixes)) } +// preferPrereleaseVersionsFilter filters a slice of version strings to prioritize prerelease versions +// for each unique core version. For example, if the input is []string{"4.0.4-40", "4.0.4"}, the output +// will be []string{"4.0.4-40"}, as the prerelease version is preferred. +// +// If no prerelease versions are found for a core version, the function returns the non-prerelease versions +// for that core version instead. +func preferPrereleaseVersionsFilter(versions []string) []string { + verMap := make(map[string][]string) + + // Group versions by core version + for _, v := range versions { + coreVer := goversion(v).Core().String() + verMap[coreVer] = append(verMap[coreVer], v) + } + + result := []string{} + for _, versionSlice := range verMap { + prereleaseVersions := []string{} + + // Get prerelease versions + for _, version := range versionSlice { + if goversion(version).Prerelease() != "" { + prereleaseVersions = append(prereleaseVersions, version) + } + } + + if len(prereleaseVersions) == 0 { + result = append(result, versionSlice...) + continue + } + result = append(result, prereleaseVersions...) + } + + return result +} + // Regex returns a map[string]*Version for the specified image by filtering tags // with the given list of versions and a regular expression. // @@ -248,22 +288,13 @@ func versionMapFromImages(baseTag string, images []registry.Image) (map[string]* } } - extractSuffix := func(tag string) string { - for _, suffix := range archSuffixes { - if strings.HasSuffix(tag, suffix) { - return suffix - } - } - return "" - } - if multiImage == nil && amd64Image == nil && arm64Image == nil { return nil, fmt.Errorf("necessary tags for %s image were not found", images[0].Name) } versions := make(map[string]*vsAPI.Version) if multiImage != nil { - versions[baseTag+extractSuffix(multiImage.Tag)] = &vsAPI.Version{ + versions[baseTag+getArchSuffix(multiImage.Tag)] = &vsAPI.Version{ ImagePath: multiImage.FullName(), ImageHash: multiImage.DigestAMD64, ImageHashArm64: multiImage.DigestARM64, @@ -273,14 +304,14 @@ func versionMapFromImages(baseTag string, images []registry.Image) (map[string]* } } if amd64Image != nil { - versions[baseTag+extractSuffix(amd64Image.Tag)] = &vsAPI.Version{ + versions[baseTag+getArchSuffix(amd64Image.Tag)] = &vsAPI.Version{ ImagePath: amd64Image.FullName(), ImageHash: amd64Image.DigestAMD64, } } // Include arm64 if multi image is not specified if multiImage == nil && arm64Image != nil { - versions[baseTag+extractSuffix(arm64Image.Tag)] = &vsAPI.Version{ + versions[baseTag+getArchSuffix(arm64Image.Tag)] = &vsAPI.Version{ ImagePath: arm64Image.FullName(), ImageHashArm64: arm64Image.DigestARM64, } @@ -288,3 +319,12 @@ func versionMapFromImages(baseTag string, images []registry.Image) (map[string]* return versions, nil } + +func getArchSuffix(tag string) string { + for _, suffix := range archSuffixes { + if strings.HasSuffix(tag, suffix) { + return suffix + } + } + return "" +} From e9e8c9b92722614f3b99756d8d154c06b759def1 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 13 Nov 2024 13:38:34 +0200 Subject: [PATCH 13/20] get latest backups --- tools/operator-tool/cmd/filler.go | 6 ++++++ tools/operator-tool/cmd/matrix.go | 9 ++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/cmd/filler.go index 490c19c8..67e0d679 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/cmd/filler.go @@ -265,6 +265,8 @@ func getVersionMapLatestVer(rc *registry.RegistryClient, imageName string) (map[ // - If no image with both amd64 and arm64 builds is found, separate images for amd64 and arm64 // are added individually. func versionMapFromImages(baseTag string, images []registry.Image) (map[string]*vsAPI.Version, error) { + baseTag = trimArchSuffix(baseTag) + slices.SortFunc(images, func(a, b registry.Image) int { return goversion(b.Tag).Compare(goversion(a.Tag)) }) @@ -320,6 +322,10 @@ func versionMapFromImages(baseTag string, images []registry.Image) (map[string]* return versions, nil } +func trimArchSuffix(tag string) string { + return strings.TrimSuffix(tag, getArchSuffix(tag)) +} + func getArchSuffix(tag string) string { for _, suffix := range archSuffixes { if strings.HasSuffix(tag, suffix) { diff --git a/tools/operator-tool/cmd/matrix.go b/tools/operator-tool/cmd/matrix.go index 36033021..0cf6c29f 100644 --- a/tools/operator-tool/cmd/matrix.go +++ b/tools/operator-tool/cmd/matrix.go @@ -36,7 +36,7 @@ func psVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, Mysql: f.Normal("percona/percona-server", psVersions, true), Pmm: f.Latest("percona/pmm-client"), Router: f.Normal("percona/percona-mysql-router", psVersions, true), - Backup: f.Normal("percona/percona-xtrabackup", psVersions, true), + Backup: f.Latest("percona/percona-xtrabackup"), Operator: f.Normal("percona/percona-server-mysql-operator", []string{version}, false), Haproxy: f.Latest("percona/haproxy"), Orchestrator: f.Latest("percona/percona-orchestrator"), @@ -55,15 +55,10 @@ func psmdbVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatr return nil, err } - pbmVersions, err := productsapi.GetProductVersions("percona-backup-mongodb-", "percona-backup-mongodb") - if err != nil { - return nil, err - } - matrix := &vsAPI.VersionMatrix{ Mongod: f.Normal("percona/percona-server-mongodb", mongoVersions, true), Pmm: f.Latest("percona/pmm-client"), - Backup: f.Normal("percona/percona-backup-mongodb", pbmVersions, true), + Backup: f.Latest("percona/percona-backup-mongodb"), Operator: f.Normal("percona/percona-server-mongodb-operator", []string{version}, false), } From e5bd2554d95f9cf523e1ea3fb7f763cf5526f95c Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 13 Nov 2024 14:43:47 +0200 Subject: [PATCH 14/20] implement `--cap` --- tools/operator-tool/README.md | 2 ++ tools/operator-tool/cmd/main.go | 12 +++++++++--- tools/operator-tool/cmd/util.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tools/operator-tool/README.md b/tools/operator-tool/README.md index 9ccdb310..519905bc 100644 --- a/tools/operator-tool/README.md +++ b/tools/operator-tool/README.md @@ -11,6 +11,8 @@ Build it using `make init`. ```sh $ ./bin/operator-tool --help Usage of ./bin/operator-tool: + -cap int + Sets a limit on the number of versions allowed for each major version of a product -file string Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file -include-arch-images diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index 1466fdfd..cc4cd404 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -37,6 +37,7 @@ var ( patch = flag.String("patch", "", "Provide a path to a patch file to add additional images. Must be used together with the --file option") verbose = flag.Bool("verbose", false, "Show logs") includeMultiImages = flag.Bool("include-arch-images", false, `Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file`) + versionCap = flag.Int("cap", 0, `Sets a limit on the number of versions allowed for each major version of a product`) ) func main() { @@ -67,7 +68,7 @@ func main() { log.SetOutput(io.Discard) } - if err := printSourceFile(*operatorName, *version, *filePath, *patch, *includeMultiImages); err != nil { + if err := printSourceFile(*operatorName, *version, *filePath, *patch, *includeMultiImages, *versionCap); err != nil { log.SetOutput(os.Stderr) log.Fatalln("ERROR: failed to generate source file:", err.Error()) } @@ -76,7 +77,7 @@ func main() { } } -func printSourceFile(operatorName, version, file, patchFile string, includeArchSuffixes bool) error { +func printSourceFile(operatorName, version, file, patchFile string, includeArchSuffixes bool, capacity int) error { var productResponse *vsAPI.ProductResponse var err error @@ -99,7 +100,12 @@ func printSourceFile(operatorName, version, file, patchFile string, includeArchS } } - updateMatrixStatuses(productResponse.Versions[0].Matrix) + if err := updateMatrixStatuses(productResponse.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to update matrix statuses: %w", err) + } + if err := limitMajorVersions(productResponse.Versions[0].Matrix, capacity); err != nil { + return fmt.Errorf("failed to delete versions exceeding capacity: %w", err) + } content, err := marshal(productResponse) if err != nil { diff --git a/tools/operator-tool/cmd/util.go b/tools/operator-tool/cmd/util.go index 6742626e..d0132c5f 100644 --- a/tools/operator-tool/cmd/util.go +++ b/tools/operator-tool/cmd/util.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "reflect" + "slices" "strings" vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" @@ -214,6 +215,35 @@ func updateMatrixHashes(rc *registry.RegistryClient, matrix *vsAPI.VersionMatrix }) } +func limitMajorVersions(matrix *vsAPI.VersionMatrix, capacity int) error { + if capacity <= 0 { + return nil + } + return iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + versionsByMajorVer := make(map[int][]string) + for v := range versionMap { + majorVer := goversion(v).Segments()[0] + versionsByMajorVer[majorVer] = append(versionsByMajorVer[majorVer], v) + } + for _, versions := range versionsByMajorVer { + if len(versions) <= capacity { + return nil + } + slices.SortFunc(versions, func(a, b string) int { + return goversion(b).Compare(goversion(a)) + }) + + versionsToDelete := versions[capacity:] + for _, v := range versionsToDelete { + fieldValue.SetMapIndex(reflect.ValueOf(v), reflect.Value{}) + } + } + + return nil + }) +} + func updateMatrixStatuses(matrix *vsAPI.VersionMatrix) error { return iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) From 9a01dc613cbd20e15e8a1c9ec016484170b868fa Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 13 Nov 2024 17:33:04 +0200 Subject: [PATCH 15/20] refactor --- tools/operator-tool/cmd/edit.go | 135 +++++++++ tools/operator-tool/cmd/main.go | 19 +- tools/operator-tool/cmd/patch.go | 105 +++++++ tools/operator-tool/cmd/util.go | 278 ------------------ .../{cmd => internal/filler}/filler.go | 38 +-- .../{cmd => internal/matrix}/matrix.go | 11 +- tools/operator-tool/internal/matrix/util.go | 24 ++ .../{cmd => internal/util}/marshal.go | 6 +- tools/operator-tool/internal/util/read.go | 35 +++ tools/operator-tool/internal/util/version.go | 9 + 10 files changed, 348 insertions(+), 312 deletions(-) create mode 100644 tools/operator-tool/cmd/edit.go create mode 100644 tools/operator-tool/cmd/patch.go delete mode 100644 tools/operator-tool/cmd/util.go rename tools/operator-tool/{cmd => internal/filler}/filler.go (89%) rename tools/operator-tool/{cmd => internal/matrix}/matrix.go (90%) create mode 100644 tools/operator-tool/internal/matrix/util.go rename tools/operator-tool/{cmd => internal/util}/marshal.go (95%) create mode 100644 tools/operator-tool/internal/util/read.go create mode 100644 tools/operator-tool/internal/util/version.go diff --git a/tools/operator-tool/cmd/edit.go b/tools/operator-tool/cmd/edit.go new file mode 100644 index 00000000..272ad9ad --- /dev/null +++ b/tools/operator-tool/cmd/edit.go @@ -0,0 +1,135 @@ +package main + +import ( + "fmt" + "reflect" + "slices" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + gover "github.com/hashicorp/go-version" + + "operator-tool/internal/matrix" + "operator-tool/internal/util" +) + +// deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in the file. +func deleteOldVersions(file string, m *vsAPI.VersionMatrix) error { + oldestVersions, err := getOldestVersions(file) + if err != nil { + return fmt.Errorf("failed to get oldest versions from base file: %w", err) + } + + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + oldestVersion, ok := oldestVersions[fieldName] + if !ok { + return nil + } + + m := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(m) == 0 { + return nil + } + + for k := range m { + if util.Goversion(k).Compare(oldestVersion) < 0 { + fieldValue.SetMapIndex(reflect.ValueOf(k), reflect.Value{}) // delete old version from map + } + } + return nil + }) +} + +// getOldestVersions returns a map where each key is a struct field name from the VersionMatrix +// of the specified file, and each value is the corresponding oldest version for that field. +func getOldestVersions(filePath string) (map[string]*gover.Version, error) { + prod, err := util.ReadBaseFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read base file: %w", err) + } + + versions := make(map[string]*gover.Version) + if err := matrix.Iterate(prod.Versions[0].Matrix, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + return nil + } + oldestVersion := "" + for k := range versionMap { + if oldestVersion == "" { + oldestVersion = k + continue + } + if util.Goversion(oldestVersion).Compare(util.Goversion(k)) > 0 { + oldestVersion = k + } + } + versions[fieldName] = util.Goversion(oldestVersion) + return nil + }); err != nil { + return nil, err + } + + return versions, nil +} + +func limitMajorVersions(m *vsAPI.VersionMatrix, capacity int) error { + if capacity <= 0 { + return nil + } + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + versionsByMajorVer := make(map[int][]string) + for v := range versionMap { + majorVer := util.Goversion(v).Segments()[0] + versionsByMajorVer[majorVer] = append(versionsByMajorVer[majorVer], v) + } + for _, versions := range versionsByMajorVer { + if len(versions) <= capacity { + return nil + } + slices.SortFunc(versions, func(a, b string) int { + return util.Goversion(b).Compare(util.Goversion(a)) + }) + + versionsToDelete := versions[capacity:] + for _, v := range versionsToDelete { + fieldValue.SetMapIndex(reflect.ValueOf(v), reflect.Value{}) + } + } + + return nil + }) +} + +// updateMatrixStatuses updates the statuses of version maps. +// For each major version, it sets the highest version as "recommended" +// and all other versions as "available". +func updateMatrixStatuses(m *vsAPI.VersionMatrix) error { + setStatus := func(vm map[string]*vsAPI.Version) { + highestVersions := make(map[int]string) + for version := range vm { + vm[version].Status = vsAPI.Status_available + + majorVersion := util.Goversion(version).Segments()[0] + + currentHighestVersion, ok := highestVersions[majorVersion] + + if !ok || util.Goversion(version).Compare(util.Goversion(currentHighestVersion)) > 0 { + highestVersions[majorVersion] = version + } + } + + for _, version := range highestVersions { + vm[version].Status = vsAPI.Status_recommended + } + } + + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + return nil + } + setStatus(versionMap) + return nil + }) +} diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index cc4cd404..2a057acf 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -11,6 +11,9 @@ import ( vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + "operator-tool/internal/filler" + "operator-tool/internal/matrix" + "operator-tool/internal/util" "operator-tool/pkg/registry" ) @@ -48,7 +51,7 @@ func main() { } if *filePath != "" { - product, err := readBaseFile(*filePath) + product, err := util.ReadBaseFile(*filePath) if err != nil { log.Fatalln("ERROR: failed to read base file:", err.Error()) } @@ -107,7 +110,7 @@ func printSourceFile(operatorName, version, file, patchFile string, includeArchS return fmt.Errorf("failed to delete versions exceeding capacity: %w", err) } - content, err := marshal(productResponse) + content, err := util.Marshal(productResponse) if err != nil { return fmt.Errorf("failed to marshal product response: %w", err) } @@ -120,19 +123,19 @@ func getProductResponse(rc *registry.RegistryClient, operatorName, version strin var versionMatrix *vsAPI.VersionMatrix var err error - f := &VersionMapFiller{ + f := &filler.VersionFiller{ RegistryClient: rc, - includeArchSuffixes: includeArchSuffixes, + IncludeArchSuffixes: includeArchSuffixes, } switch operatorName { case operatorNamePG: - versionMatrix, err = pgVersionMatrix(f, version) + versionMatrix, err = matrix.PG(f, version) case operatorNamePS: - versionMatrix, err = psVersionMatrix(f, version) + versionMatrix, err = matrix.PS(f, version) case operatorNamePSMDB: - versionMatrix, err = psmdbVersionMatrix(f, version) + versionMatrix, err = matrix.PSMDB(f, version) case operatorNamePXC: - versionMatrix, err = pxcVersionMatrix(f, version) + versionMatrix, err = matrix.PXC(f, version) default: panic("problems with validation. unknown operator name " + operatorName) } diff --git a/tools/operator-tool/cmd/patch.go b/tools/operator-tool/cmd/patch.go new file mode 100644 index 00000000..5d36f3c4 --- /dev/null +++ b/tools/operator-tool/cmd/patch.go @@ -0,0 +1,105 @@ +package main + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + "google.golang.org/protobuf/encoding/protojson" + + "operator-tool/internal/matrix" + "operator-tool/internal/util" + "operator-tool/pkg/registry" +) + +func patchProductResponse(rc *registry.RegistryClient, baseFilepath, patchFilepath string) (*vsAPI.ProductResponse, error) { + baseFile, err := util.ReadBaseFile(baseFilepath) + if err != nil { + return nil, fmt.Errorf("failed to read base file: %w", err) + } + patchFile, err := util.ReadPatchFile(patchFilepath) + if err != nil { + return nil, fmt.Errorf("failed to read patch file: %w", err) + } + if err := updateMatrixData(rc, patchFile); err != nil { + return nil, fmt.Errorf("failed to update patch matrix hashes: %w", err) + } + + matrixToMap := func(matrix *vsAPI.VersionMatrix) (map[string]map[string]map[string]any, error) { + data, err := protojson.Marshal(matrix) + if err != nil { + return nil, fmt.Errorf("failed to marshal: %w", err) + } + + m := make(map[string]map[string]map[string]any) + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return m, nil + } + + baseMatrix, err := matrixToMap(baseFile.Versions[0].Matrix) + if err != nil { + return nil, fmt.Errorf("failed to convert base matrix to map: %w", err) + } + patchMatrix, err := matrixToMap(patchFile) + if err != nil { + return nil, fmt.Errorf("failed to convert patch matrix to map: %w", err) + } + + for product, versions := range patchMatrix { + for version, verInfo := range versions { + if _, ok := baseMatrix[product]; !ok { + baseMatrix[product] = make(map[string]map[string]any) + } + baseMatrix[product][version] = verInfo + } + } + + mapToMatrix := func(m map[string]map[string]map[string]any) (*vsAPI.VersionMatrix, error) { + data, err := json.Marshal(m) + if err != nil { + return nil, fmt.Errorf("failed to marshal: %w", err) + } + + matrix := new(vsAPI.VersionMatrix) + if err := protojson.Unmarshal(data, matrix); err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + + return matrix, nil + } + + baseFile.Versions[0].Matrix, err = mapToMatrix(baseMatrix) + if err != nil { + return nil, fmt.Errorf("failed to convert patched map to matrix: %w", err) + } + return baseFile, nil +} + +func updateMatrixData(rc *registry.RegistryClient, m *vsAPI.VersionMatrix) error { + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + return nil + } + + for k, v := range versionMap { + imageSpl := strings.Split(v.ImagePath, ":") + if len(imageSpl) == 1 { + return fmt.Errorf("image %s doesn't have tag", v.ImagePath) + } + tag := imageSpl[len(imageSpl)-1] + imageName := strings.TrimSuffix(v.ImagePath, ":"+tag) + image, err := rc.GetTag(imageName, tag) + if err != nil { + return fmt.Errorf("failed to get tag %s for image %s: %w", tag, imageName, err) + } + versionMap[k].ImageHash = image.DigestAMD64 + versionMap[k].ImageHashArm64 = image.DigestARM64 + } + return nil + }) +} diff --git a/tools/operator-tool/cmd/util.go b/tools/operator-tool/cmd/util.go deleted file mode 100644 index d0132c5f..00000000 --- a/tools/operator-tool/cmd/util.go +++ /dev/null @@ -1,278 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "reflect" - "slices" - "strings" - - vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" - gover "github.com/hashicorp/go-version" - "google.golang.org/protobuf/encoding/protojson" - - "operator-tool/pkg/registry" -) - -func readBaseFile(path string) (*vsAPI.ProductResponse, error) { - content, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read file: %w", err) - } - product := new(vsAPI.ProductResponse) - err = protojson.Unmarshal(content, product) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal: %w", err) - } - return product, nil -} - -func readPatchFile(path string) (*vsAPI.VersionMatrix, error) { - content, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read file: %w", err) - } - matrix := new(vsAPI.VersionMatrix) - err = protojson.Unmarshal(content, matrix) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal: %w", err) - } - return matrix, nil -} - -// deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in the file. -func deleteOldVersions(file string, matrix *vsAPI.VersionMatrix) error { - minVersions, err := getOldestVersions(file) - if err != nil { - return fmt.Errorf("failed to get oldest versions from base file: %w", err) - } - deleteOldVersionsWithMap(matrix, minVersions) - return nil -} - -func iterateOverMatrixFields(matrix *vsAPI.VersionMatrix, f func(fieldName string, fieldValue reflect.Value) error) error { - matrixType := reflect.TypeOf(matrix).Elem() - matrixValue := reflect.ValueOf(matrix).Elem() - - for i := 0; i < matrixValue.NumField(); i++ { - field := matrixType.Field(i) - // check if value is exported - if field.PkgPath != "" { - continue - } - if err := f(field.Name, matrixValue.Field(i)); err != nil { - return err - } - } - return nil -} - -// deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in oldestVersions. -func deleteOldVersionsWithMap(matrix *vsAPI.VersionMatrix, oldestVersions map[string]*gover.Version) { - iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { - oldestVersion, ok := oldestVersions[fieldName] - if !ok { - return nil - } - - m := fieldValue.Interface().(map[string]*vsAPI.Version) - if len(m) == 0 { - return nil - } - - for k := range m { - if goversion(k).Compare(oldestVersion) < 0 { - fieldValue.SetMapIndex(reflect.ValueOf(k), reflect.Value{}) // delete old version from map - } - } - return nil - }) -} - -// getOldestVersions returns a map where each key is a struct field name from the VersionMatrix -// of the specified file, and each value is the corresponding oldest version for that field. -func getOldestVersions(filePath string) (map[string]*gover.Version, error) { - prod, err := readBaseFile(filePath) - if err != nil { - return nil, fmt.Errorf("failed to read base file: %w", err) - } - - versions := make(map[string]*gover.Version) - iterateOverMatrixFields(prod.Versions[0].Matrix, func(fieldName string, fieldValue reflect.Value) error { - versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) - if len(versionMap) == 0 { - return nil - } - oldestVersion := "" - for k := range versionMap { - if oldestVersion == "" { - oldestVersion = k - continue - } - if goversion(oldestVersion).Compare(goversion(k)) > 0 { - oldestVersion = k - } - } - versions[fieldName] = goversion(oldestVersion) - return nil - }) - - return versions, nil -} - -func goversion(v string) *gover.Version { - return gover.Must(gover.NewVersion(v)) -} - -func patchProductResponse(rc *registry.RegistryClient, baseFilepath, patchFilepath string) (*vsAPI.ProductResponse, error) { - baseFile, err := readBaseFile(baseFilepath) - if err != nil { - return nil, fmt.Errorf("failed to read base file: %w", err) - } - patchFile, err := readPatchFile(patchFilepath) - if err != nil { - return nil, fmt.Errorf("failed to read patch file: %w", err) - } - if err := updateMatrixHashes(rc, patchFile); err != nil { - return nil, fmt.Errorf("failed to update patch matrix hashes: %w", err) - } - - matrixToMap := func(matrix *vsAPI.VersionMatrix) (map[string]map[string]map[string]any, error) { - data, err := protojson.Marshal(matrix) - if err != nil { - return nil, fmt.Errorf("failed to marshal: %w", err) - } - - m := make(map[string]map[string]map[string]any) - if err := json.Unmarshal(data, &m); err != nil { - return nil, fmt.Errorf("failed to unmarshal: %w", err) - } - return m, nil - } - - baseMatrix, err := matrixToMap(baseFile.Versions[0].Matrix) - if err != nil { - return nil, fmt.Errorf("failed to convert base matrix to map: %w", err) - } - patchMatrix, err := matrixToMap(patchFile) - if err != nil { - return nil, fmt.Errorf("failed to convert patch matrix to map: %w", err) - } - - for product, versions := range patchMatrix { - for version, verInfo := range versions { - if _, ok := baseMatrix[product]; !ok { - baseMatrix[product] = make(map[string]map[string]any) - } - baseMatrix[product][version] = verInfo - } - } - - mapToMatrix := func(m map[string]map[string]map[string]any) (*vsAPI.VersionMatrix, error) { - data, err := json.Marshal(m) - if err != nil { - return nil, fmt.Errorf("failed to marshal: %w", err) - } - - matrix := new(vsAPI.VersionMatrix) - if err := protojson.Unmarshal(data, matrix); err != nil { - return nil, fmt.Errorf("failed to unmarshal: %w", err) - } - - return matrix, nil - } - - baseFile.Versions[0].Matrix, err = mapToMatrix(baseMatrix) - if err != nil { - return nil, fmt.Errorf("failed to convert patched map to matrix: %w", err) - } - return baseFile, nil -} - -func updateMatrixHashes(rc *registry.RegistryClient, matrix *vsAPI.VersionMatrix) error { - return iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { - versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) - if len(versionMap) == 0 { - return nil - } - - for k, v := range versionMap { - imageSpl := strings.Split(v.ImagePath, ":") - if len(imageSpl) == 1 { - return fmt.Errorf("image %s doesn't have tag", v.ImagePath) - } - tag := imageSpl[len(imageSpl)-1] - imageName := strings.TrimSuffix(v.ImagePath, ":"+tag) - image, err := rc.GetTag(imageName, tag) - if err != nil { - return fmt.Errorf("failed to get tag %s for image %s: %w", tag, imageName, err) - } - versionMap[k].ImageHash = image.DigestAMD64 - versionMap[k].ImageHashArm64 = image.DigestARM64 - } - return nil - }) -} - -func limitMajorVersions(matrix *vsAPI.VersionMatrix, capacity int) error { - if capacity <= 0 { - return nil - } - return iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { - versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) - versionsByMajorVer := make(map[int][]string) - for v := range versionMap { - majorVer := goversion(v).Segments()[0] - versionsByMajorVer[majorVer] = append(versionsByMajorVer[majorVer], v) - } - for _, versions := range versionsByMajorVer { - if len(versions) <= capacity { - return nil - } - slices.SortFunc(versions, func(a, b string) int { - return goversion(b).Compare(goversion(a)) - }) - - versionsToDelete := versions[capacity:] - for _, v := range versionsToDelete { - fieldValue.SetMapIndex(reflect.ValueOf(v), reflect.Value{}) - } - } - - return nil - }) -} - -func updateMatrixStatuses(matrix *vsAPI.VersionMatrix) error { - return iterateOverMatrixFields(matrix, func(fieldName string, fieldValue reflect.Value) error { - versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) - if len(versionMap) == 0 { - return nil - } - setStatus(versionMap) - return nil - }) -} - -// setStatus updates the statuses of version map. -// For each major version, it sets the highest version as "recommended" -// and all other versions as "available". -func setStatus(vm map[string]*vsAPI.Version) { - highestVersions := make(map[int]string) - for version := range vm { - vm[version].Status = vsAPI.Status_available - - majorVersion := goversion(version).Segments()[0] - - currentHighestVersion, ok := highestVersions[majorVersion] - - if !ok || goversion(version).Compare(goversion(currentHighestVersion)) > 0 { - highestVersions[majorVersion] = version - } - } - - for _, version := range highestVersions { - vm[version].Status = vsAPI.Status_recommended - } -} diff --git a/tools/operator-tool/cmd/filler.go b/tools/operator-tool/internal/filler/filler.go similarity index 89% rename from tools/operator-tool/cmd/filler.go rename to tools/operator-tool/internal/filler/filler.go index 67e0d679..b4f96398 100644 --- a/tools/operator-tool/cmd/filler.go +++ b/tools/operator-tool/internal/filler/filler.go @@ -1,4 +1,4 @@ -package main +package filler import ( "errors" @@ -11,6 +11,7 @@ import ( vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" gover "github.com/hashicorp/go-version" + "operator-tool/internal/util" "operator-tool/pkg/registry" ) @@ -21,19 +22,20 @@ var archSuffixes = []string{ "-amd64", } -// VersionMapFiller is a helper type for creating a map[string]*vsAPI.Version +// VersionFiller is a helper type for creating a map[string]*vsAPI.Version // using information retrieved from Docker Hub. -type VersionMapFiller struct { +type VersionFiller struct { RegistryClient *registry.RegistryClient - errs []error - includeArchSuffixes bool + IncludeArchSuffixes bool + + errs []error } -func (f *VersionMapFiller) addErr(err error) { +func (f *VersionFiller) addErr(err error) { f.errs = append(f.errs, err) } -func (f *VersionMapFiller) exec(vm map[string]*vsAPI.Version, err error) map[string]*vsAPI.Version { +func (f *VersionFiller) exec(vm map[string]*vsAPI.Version, err error) map[string]*vsAPI.Version { if err != nil { f.addErr(err) return nil @@ -45,12 +47,12 @@ func (f *VersionMapFiller) exec(vm map[string]*vsAPI.Version, err error) map[str // and appends any missing tags that match the MAJOR.MINOR.PATCH version format to the returned versions slice. // // Tags with a "-debug" suffix are excluded. -func (f *VersionMapFiller) addVersionsFromRegistry(image string, versions []string) []string { +func (f *VersionFiller) addVersionsFromRegistry(image string, versions []string) []string { wantedVerisons := make(map[string]struct{}, len(versions)) coreVersions := make(map[string]struct{}) for _, v := range versions { wantedVerisons[v] = struct{}{} - coreVersions[goversion(v).Core().String()] = struct{}{} + coreVersions[util.Goversion(v).Core().String()] = struct{}{} } tags, err := f.RegistryClient.GetTags(image) @@ -78,7 +80,7 @@ func (f *VersionMapFiller) addVersionsFromRegistry(image string, versions []stri if _, err := gover.NewVersion(tag); err != nil { continue } - if _, ok := coreVersions[goversion(tag).Core().String()]; !ok { + if _, ok := coreVersions[util.Goversion(tag).Core().String()]; !ok { continue } if _, ok := wantedVerisons[tag]; ok { @@ -94,14 +96,14 @@ func (f *VersionMapFiller) addVersionsFromRegistry(image string, versions []stri // // The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". // Prerelease versions are preferred for each core version when available. See preferPrereleaseVersionsFilter function. -func (f *VersionMapFiller) Normal(image string, versions []string, addVersionsFromRegistry bool) map[string]*vsAPI.Version { +func (f *VersionFiller) Normal(image string, versions []string, addVersionsFromRegistry bool) map[string]*vsAPI.Version { if addVersionsFromRegistry { versions = f.addVersionsFromRegistry(image, versions) } versions = preferPrereleaseVersionsFilter(versions) - return f.exec(getVersionMap(f.RegistryClient, image, versions, f.includeArchSuffixes)) + return f.exec(getVersionMap(f.RegistryClient, image, versions, f.IncludeArchSuffixes)) } // preferPrereleaseVersionsFilter filters a slice of version strings to prioritize prerelease versions @@ -115,7 +117,7 @@ func preferPrereleaseVersionsFilter(versions []string) []string { // Group versions by core version for _, v := range versions { - coreVer := goversion(v).Core().String() + coreVer := util.Goversion(v).Core().String() verMap[coreVer] = append(verMap[coreVer], v) } @@ -125,7 +127,7 @@ func preferPrereleaseVersionsFilter(versions []string) []string { // Get prerelease versions for _, version := range versionSlice { - if goversion(version).Prerelease() != "" { + if util.Goversion(version).Prerelease() != "" { prereleaseVersions = append(prereleaseVersions, version) } } @@ -149,18 +151,18 @@ func preferPrereleaseVersionsFilter(versions []string) []string { // while "1.3.1-logcollector", "1.2.1-some-string", and "1.2.1" will not be included. // // The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". -func (f *VersionMapFiller) Regex(image string, regex string, versions []string) map[string]*vsAPI.Version { +func (f *VersionFiller) Regex(image string, regex string, versions []string) map[string]*vsAPI.Version { return f.exec(getVersionMapRegex(f.RegistryClient, image, regex, versions)) } // Latest returns a map[string]*Version with latest version tag of the specified image. // // The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". -func (f *VersionMapFiller) Latest(image string) map[string]*vsAPI.Version { +func (f *VersionFiller) Latest(image string) map[string]*vsAPI.Version { return f.exec(getVersionMapLatestVer(f.RegistryClient, image)) } -func (f *VersionMapFiller) Error() error { +func (f *VersionFiller) Error() error { return errors.Join(f.errs...) } @@ -268,7 +270,7 @@ func versionMapFromImages(baseTag string, images []registry.Image) (map[string]* baseTag = trimArchSuffix(baseTag) slices.SortFunc(images, func(a, b registry.Image) int { - return goversion(b.Tag).Compare(goversion(a.Tag)) + return util.Goversion(b.Tag).Compare(util.Goversion(a.Tag)) }) var multiImage, amd64Image, arm64Image *registry.Image diff --git a/tools/operator-tool/cmd/matrix.go b/tools/operator-tool/internal/matrix/matrix.go similarity index 90% rename from tools/operator-tool/cmd/matrix.go rename to tools/operator-tool/internal/matrix/matrix.go index 0cf6c29f..5a98e364 100644 --- a/tools/operator-tool/cmd/matrix.go +++ b/tools/operator-tool/internal/matrix/matrix.go @@ -1,12 +1,13 @@ -package main +package matrix import ( vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + "operator-tool/internal/filler" productsapi "operator-tool/pkg/products-api" ) -func pgVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { +func PG(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { pgVersions, err := productsapi.GetProductVersions("", "postgresql-distribution-16", "postgresql-distribution-15", "postgresql-distribution-14", "postgresql-distribution-13", "postgresql-distribution-12") if err != nil { return nil, err @@ -26,7 +27,7 @@ func pgVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, return matrix, nil } -func psVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { +func PS(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { psVersions, err := productsapi.GetProductVersions("Percona-Server-", "Percona-Server-8.0") if err != nil { return nil, err @@ -49,7 +50,7 @@ func psVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, return matrix, nil } -func psmdbVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { +func PSMDB(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { mongoVersions, err := productsapi.GetProductVersions("percona-server-mongodb-", "percona-server-mongodb-7.0", "percona-server-mongodb-6.0", "percona-server-mongodb-5.0") if err != nil { return nil, err @@ -69,7 +70,7 @@ func psmdbVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatr return matrix, nil } -func pxcVersionMatrix(f *VersionMapFiller, version string) (*vsAPI.VersionMatrix, error) { +func PXC(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { pxcVersions, err := productsapi.GetProductVersions("Percona-XtraDB-Cluster-", "Percona-XtraDB-Cluster-80", "Percona-XtraDB-Cluster-57") if err != nil { return nil, err diff --git a/tools/operator-tool/internal/matrix/util.go b/tools/operator-tool/internal/matrix/util.go new file mode 100644 index 00000000..f3b83b29 --- /dev/null +++ b/tools/operator-tool/internal/matrix/util.go @@ -0,0 +1,24 @@ +package matrix + +import ( + "reflect" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" +) + +func Iterate(matrix *vsAPI.VersionMatrix, f func(fieldName string, fieldValue reflect.Value) error) error { + matrixType := reflect.TypeOf(matrix).Elem() + matrixValue := reflect.ValueOf(matrix).Elem() + + for i := 0; i < matrixValue.NumField(); i++ { + field := matrixType.Field(i) + // check if value is exported + if field.PkgPath != "" { + continue + } + if err := f(field.Name, matrixValue.Field(i)); err != nil { + return err + } + } + return nil +} diff --git a/tools/operator-tool/cmd/marshal.go b/tools/operator-tool/internal/util/marshal.go similarity index 95% rename from tools/operator-tool/cmd/marshal.go rename to tools/operator-tool/internal/util/marshal.go index 38c8d81f..3807370a 100644 --- a/tools/operator-tool/cmd/marshal.go +++ b/tools/operator-tool/internal/util/marshal.go @@ -1,4 +1,4 @@ -package main +package util import ( "encoding/json" @@ -10,11 +10,11 @@ import ( vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" ) -// marshal marshals ProductResponse to JSON, ensuring the "critical" field is always included, +// Marshal marshals ProductResponse to JSON, ensuring the "critical" field is always included, // without requiring modifications to the `versionpb/api` package or creating custom types that // implement the json.Marshaler interface. // Use protojson.Marshal instead if omitting the "critical" field is acceptable. -func marshal(product *vsAPI.ProductResponse) ([]byte, error) { +func Marshal(product *vsAPI.ProductResponse) ([]byte, error) { m, err := productToMap(product) if err != nil { return nil, fmt.Errorf("json conversion: %w", err) diff --git a/tools/operator-tool/internal/util/read.go b/tools/operator-tool/internal/util/read.go new file mode 100644 index 00000000..ead17b61 --- /dev/null +++ b/tools/operator-tool/internal/util/read.go @@ -0,0 +1,35 @@ +package util + +import ( + "fmt" + "os" + + vsAPI "github.com/Percona-Lab/percona-version-service/versionpb/api" + "google.golang.org/protobuf/encoding/protojson" +) + +func ReadBaseFile(path string) (*vsAPI.ProductResponse, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + product := new(vsAPI.ProductResponse) + err = protojson.Unmarshal(content, product) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return product, nil +} + +func ReadPatchFile(path string) (*vsAPI.VersionMatrix, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + matrix := new(vsAPI.VersionMatrix) + err = protojson.Unmarshal(content, matrix) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal: %w", err) + } + return matrix, nil +} diff --git a/tools/operator-tool/internal/util/version.go b/tools/operator-tool/internal/util/version.go new file mode 100644 index 00000000..2a5c14ed --- /dev/null +++ b/tools/operator-tool/internal/util/version.go @@ -0,0 +1,9 @@ +package util + +import ( + gover "github.com/hashicorp/go-version" +) + +func Goversion(v string) *gover.Version { + return gover.Must(gover.NewVersion(v)) +} From 0fc80ccb6f71f7c8988d5ba1ed01be1e37d3b298 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 27 Nov 2024 17:59:15 +0200 Subject: [PATCH 16/20] add `--only-latest` --- tools/operator-tool/README.md | 2 ++ tools/operator-tool/cmd/edit.go | 30 +++++++++++++++++++++++++++++- tools/operator-tool/cmd/main.go | 26 +++++++++++++++++++++----- tools/operator-tool/cmd/patch.go | 15 +++++++-------- 4 files changed, 59 insertions(+), 14 deletions(-) diff --git a/tools/operator-tool/README.md b/tools/operator-tool/README.md index 519905bc..79c58f76 100644 --- a/tools/operator-tool/README.md +++ b/tools/operator-tool/README.md @@ -17,6 +17,8 @@ Usage of ./bin/operator-tool: Specify an older source file. The operator-tool will exclude any versions that are older than those listed in this file -include-arch-images Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file + -only-latest + Add only latest images to the specified "-file" -operator string Operator name. Available values: [psmdb pxc ps pg] -patch string diff --git a/tools/operator-tool/cmd/edit.go b/tools/operator-tool/cmd/edit.go index 272ad9ad..70d517df 100644 --- a/tools/operator-tool/cmd/edit.go +++ b/tools/operator-tool/cmd/edit.go @@ -12,7 +12,7 @@ import ( "operator-tool/internal/util" ) -// deleteOldVersionsWithMap removes versions from the matrix that are older than those specified in the file. +// deleteOldVersions removes versions from the matrix that are older than those specified in the file. func deleteOldVersions(file string, m *vsAPI.VersionMatrix) error { oldestVersions, err := getOldestVersions(file) if err != nil { @@ -39,6 +39,34 @@ func deleteOldVersions(file string, m *vsAPI.VersionMatrix) error { }) } +func keepOnlyLatestVersions(m *vsAPI.VersionMatrix) error { + return matrix.Iterate(m, func(fieldName string, fieldValue reflect.Value) error { + versionMap := fieldValue.Interface().(map[string]*vsAPI.Version) + if len(versionMap) == 0 { + return nil + } + + latestByMajorVer := make(map[int]string) + for v := range versionMap { + majorVer := util.Goversion(v).Segments()[0] + + curLatest, ok := latestByMajorVer[majorVer] + if !ok || util.Goversion(v).Compare(util.Goversion(curLatest)) > 0 { + latestByMajorVer[majorVer] = v + continue + } + } + + for v := range versionMap { + majorVer := util.Goversion(v).Segments()[0] + if latestByMajorVer[majorVer] != v { + fieldValue.SetMapIndex(reflect.ValueOf(v), reflect.Value{}) // delete non-latest version from map + } + } + return nil + }) +} + // getOldestVersions returns a map where each key is a struct field name from the VersionMatrix // of the specified file, and each value is the corresponding oldest version for that field. func getOldestVersions(filePath string) (map[string]*gover.Version, error) { diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index 2a057acf..ab40607a 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -41,6 +41,7 @@ var ( verbose = flag.Bool("verbose", false, "Show logs") includeMultiImages = flag.Bool("include-arch-images", false, `Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file`) versionCap = flag.Int("cap", 0, `Sets a limit on the number of versions allowed for each major version of a product`) + onlyLatest = flag.Bool("only-latest", false, `Add only latest images to the specified "-file"`) ) func main() { @@ -71,7 +72,7 @@ func main() { log.SetOutput(io.Discard) } - if err := printSourceFile(*operatorName, *version, *filePath, *patch, *includeMultiImages, *versionCap); err != nil { + if err := printSourceFile(*operatorName, *version, *filePath, *patch, *includeMultiImages, *versionCap, *onlyLatest); err != nil { log.SetOutput(os.Stderr) log.Fatalln("ERROR: failed to generate source file:", err.Error()) } @@ -80,7 +81,7 @@ func main() { } } -func printSourceFile(operatorName, version, file, patchFile string, includeArchSuffixes bool, capacity int) error { +func printSourceFile(operatorName, version, file, patchFile string, includeArchSuffixes bool, capacity int, onlyLatest bool) error { var productResponse *vsAPI.ProductResponse var err error @@ -92,12 +93,27 @@ func printSourceFile(operatorName, version, file, patchFile string, includeArchS return fmt.Errorf("failed to get product response: %w", err) } if file != "" { - if err := deleteOldVersions(file, productResponse.Versions[0].Matrix); err != nil { - return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) + if onlyLatest { + if err := keepOnlyLatestVersions(productResponse.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) + } + + productResponse, err = patchProductResponse(registryClient, file, productResponse.Versions[0].Matrix, version) + if err != nil { + return fmt.Errorf("failed to patch product response: %w", err) + } + } else { + if err := deleteOldVersions(file, productResponse.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) + } } } } else { - productResponse, err = patchProductResponse(registryClient, file, patchFile) + patchMatrix, err := util.ReadPatchFile(patchFile) + if err != nil { + return fmt.Errorf("failed to read patch file: %w", err) + } + productResponse, err = patchProductResponse(registryClient, file, patchMatrix, version) if err != nil { return fmt.Errorf("failed to patch product response: %w", err) } diff --git a/tools/operator-tool/cmd/patch.go b/tools/operator-tool/cmd/patch.go index 5d36f3c4..7eb2b9c1 100644 --- a/tools/operator-tool/cmd/patch.go +++ b/tools/operator-tool/cmd/patch.go @@ -14,16 +14,12 @@ import ( "operator-tool/pkg/registry" ) -func patchProductResponse(rc *registry.RegistryClient, baseFilepath, patchFilepath string) (*vsAPI.ProductResponse, error) { +func patchProductResponse(rc *registry.RegistryClient, baseFilepath string, patchMatrix *vsAPI.VersionMatrix, operatorVersion string) (*vsAPI.ProductResponse, error) { baseFile, err := util.ReadBaseFile(baseFilepath) if err != nil { return nil, fmt.Errorf("failed to read base file: %w", err) } - patchFile, err := util.ReadPatchFile(patchFilepath) - if err != nil { - return nil, fmt.Errorf("failed to read patch file: %w", err) - } - if err := updateMatrixData(rc, patchFile); err != nil { + if err := updateMatrixData(rc, patchMatrix); err != nil { return nil, fmt.Errorf("failed to update patch matrix hashes: %w", err) } @@ -44,12 +40,12 @@ func patchProductResponse(rc *registry.RegistryClient, baseFilepath, patchFilepa if err != nil { return nil, fmt.Errorf("failed to convert base matrix to map: %w", err) } - patchMatrix, err := matrixToMap(patchFile) + patchMatrixMap, err := matrixToMap(patchMatrix) if err != nil { return nil, fmt.Errorf("failed to convert patch matrix to map: %w", err) } - for product, versions := range patchMatrix { + for product, versions := range patchMatrixMap { for version, verInfo := range versions { if _, ok := baseMatrix[product]; !ok { baseMatrix[product] = make(map[string]map[string]any) @@ -76,6 +72,9 @@ func patchProductResponse(rc *registry.RegistryClient, baseFilepath, patchFilepa if err != nil { return nil, fmt.Errorf("failed to convert patched map to matrix: %w", err) } + if operatorVersion == "" { + baseFile.Versions[0].Operator = operatorVersion + } return baseFile, nil } From dff5d64759c9457af6d188057e1775fb01243d88 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Wed, 27 Nov 2024 18:06:43 +0200 Subject: [PATCH 17/20] use `includeArchSuffixes` for latest images --- tools/operator-tool/internal/filler/filler.go | 19 ++++++----------- tools/operator-tool/internal/util/arch.go | 21 +++++++++++++++++++ tools/operator-tool/pkg/registry/registry.go | 7 ++++++- 3 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 tools/operator-tool/internal/util/arch.go diff --git a/tools/operator-tool/internal/filler/filler.go b/tools/operator-tool/internal/filler/filler.go index b4f96398..32bcff6e 100644 --- a/tools/operator-tool/internal/filler/filler.go +++ b/tools/operator-tool/internal/filler/filler.go @@ -15,13 +15,6 @@ import ( "operator-tool/pkg/registry" ) -var archSuffixes = []string{ - "-arm64", - "-aarch64", - "-multi", - "-amd64", -} - // VersionFiller is a helper type for creating a map[string]*vsAPI.Version // using information retrieved from Docker Hub. type VersionFiller struct { @@ -62,7 +55,7 @@ func (f *VersionFiller) addVersionsFromRegistry(image string, versions []string) } // getVersionMap will search for images with these suffixes. We don't need them in this function - ignoredSuffixes := append(archSuffixes, "-debug") + ignoredSuffixes := append(util.GetArchSuffixes(), "-debug") hasIgnoredSuffix := func(tag string) bool { for _, s := range ignoredSuffixes { @@ -159,7 +152,7 @@ func (f *VersionFiller) Regex(image string, regex string, versions []string) map // // The map may include image tags with the following suffixes: "", "-amd64", "-arm64", and "-multi". func (f *VersionFiller) Latest(image string) map[string]*vsAPI.Version { - return f.exec(getVersionMapLatestVer(f.RegistryClient, image)) + return f.exec(getVersionMapLatestVer(f.RegistryClient, image, f.IncludeArchSuffixes)) } func (f *VersionFiller) Error() error { @@ -205,7 +198,7 @@ func getVersionMap(rc *registry.RegistryClient, image string, versions []string, images, err := rc.GetImages(image, func(tag string) bool { allowedSuffixes := []string{""} if includeArchSuffixes { - allowedSuffixes = append(allowedSuffixes, archSuffixes...) + allowedSuffixes = append(allowedSuffixes, util.GetArchSuffixes()...) } for _, s := range allowedSuffixes { tagWithoutSuffix := tag @@ -243,8 +236,8 @@ func getVersionMap(rc *registry.RegistryClient, image string, versions []string, return m, nil } -func getVersionMapLatestVer(rc *registry.RegistryClient, imageName string) (map[string]*vsAPI.Version, error) { - image, err := rc.GetLatestImage(imageName) +func getVersionMapLatestVer(rc *registry.RegistryClient, imageName string, includeArchSuffixes bool) (map[string]*vsAPI.Version, error) { + image, err := rc.GetLatestImage(imageName, includeArchSuffixes) if err != nil { return nil, err } @@ -329,7 +322,7 @@ func trimArchSuffix(tag string) string { } func getArchSuffix(tag string) string { - for _, suffix := range archSuffixes { + for _, suffix := range util.GetArchSuffixes() { if strings.HasSuffix(tag, suffix) { return suffix } diff --git a/tools/operator-tool/internal/util/arch.go b/tools/operator-tool/internal/util/arch.go new file mode 100644 index 00000000..25b58517 --- /dev/null +++ b/tools/operator-tool/internal/util/arch.go @@ -0,0 +1,21 @@ +package util + +import "strings" + +func GetArchSuffixes() []string { + return []string{ + "-arm64", + "-aarch64", + "-multi", + "-amd64", + } +} + +func HasArchSuffix(s string) bool { + for _, suffix := range GetArchSuffixes() { + if strings.HasSuffix(s, suffix) { + return true + } + } + return false +} diff --git a/tools/operator-tool/pkg/registry/registry.go b/tools/operator-tool/pkg/registry/registry.go index 46e08abb..8d65ac88 100644 --- a/tools/operator-tool/pkg/registry/registry.go +++ b/tools/operator-tool/pkg/registry/registry.go @@ -10,6 +10,8 @@ import ( "path" "strconv" "strings" + + "operator-tool/internal/util" ) type tagResp struct { @@ -172,7 +174,7 @@ func NewClient() *RegistryClient { } } -func (r *RegistryClient) GetLatestImage(imageName string) (Image, error) { +func (r *RegistryClient) GetLatestImage(imageName string, includeArchSuffix bool) (Image, error) { resp, err := r.listTags(imageName, 1) if err != nil { return Image{}, fmt.Errorf("failed to get latest image: %w", err) @@ -181,6 +183,9 @@ func (r *RegistryClient) GetLatestImage(imageName string) (Image, error) { if result.Name == "latest" { continue } + if !includeArchSuffix && util.HasArchSuffix(result.Name) { + continue + } if strings.Count(result.Name, ".") == 2 { return result.Image(imageName), nil } From 49e01e50fd146188ef26edb1d9ec705f650c8aa7 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 27 Feb 2025 12:09:04 +0200 Subject: [PATCH 18/20] update matrix --- tools/operator-tool/go.mod | 18 +++---- tools/operator-tool/go.sum | 48 +++++++++++++------ tools/operator-tool/internal/matrix/matrix.go | 8 ++-- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/tools/operator-tool/go.mod b/tools/operator-tool/go.mod index 52f71a26..6830fc9c 100644 --- a/tools/operator-tool/go.mod +++ b/tools/operator-tool/go.mod @@ -1,21 +1,23 @@ module operator-tool -go 1.23.1 +go 1.23.4 + +toolchain go1.23.5 require ( github.com/Percona-Lab/percona-version-service v0.0.0-20241013113618-2966a16cabb1 github.com/hashicorp/go-version v1.7.0 - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.36.0 ) replace github.com/Percona-Lab/percona-version-service => ../../ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/grpc v1.65.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241219192143-6b3ec007d9bb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb // indirect + google.golang.org/grpc v1.69.2 // indirect ) diff --git a/tools/operator-tool/go.sum b/tools/operator-tool/go.sum index f5b6db17..7149f859 100644 --- a/tools/operator-tool/go.sum +++ b/tools/operator-tool/go.sum @@ -1,20 +1,40 @@ +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= -google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +google.golang.org/genproto/googleapis/api v0.0.0-20241219192143-6b3ec007d9bb h1:B7GIB7sr443wZ/EAEl7VZjmh1V6qzkt5V+RYcUYtS1U= +google.golang.org/genproto/googleapis/api v0.0.0-20241219192143-6b3ec007d9bb/go.mod h1:E5//3O5ZIG2l71Xnt+P/CYUY8Bxs8E7WMoZ9tlcMbAY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb h1:3oy2tynMOP1QbTC0MsNNAV+Se8M2Bd0A5+x1QHyw+pI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= +google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= diff --git a/tools/operator-tool/internal/matrix/matrix.go b/tools/operator-tool/internal/matrix/matrix.go index 5a98e364..e6a09165 100644 --- a/tools/operator-tool/internal/matrix/matrix.go +++ b/tools/operator-tool/internal/matrix/matrix.go @@ -8,7 +8,7 @@ import ( ) func PG(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { - pgVersions, err := productsapi.GetProductVersions("", "postgresql-distribution-16", "postgresql-distribution-15", "postgresql-distribution-14", "postgresql-distribution-13", "postgresql-distribution-12") + pgVersions, err := productsapi.GetProductVersions("", "postgresql-distribution-17", "postgresql-distribution-16", "postgresql-distribution-15", "postgresql-distribution-14", "postgresql-distribution-13") if err != nil { return nil, err } @@ -51,7 +51,7 @@ func PS(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { } func PSMDB(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { - mongoVersions, err := productsapi.GetProductVersions("percona-server-mongodb-", "percona-server-mongodb-7.0", "percona-server-mongodb-6.0", "percona-server-mongodb-5.0") + mongoVersions, err := productsapi.GetProductVersions("percona-server-mongodb-", "percona-server-mongodb-8.0", "percona-server-mongodb-7.0", "percona-server-mongodb-6.0") if err != nil { return nil, err } @@ -71,12 +71,12 @@ func PSMDB(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error } func PXC(f *filler.VersionFiller, version string) (*vsAPI.VersionMatrix, error) { - pxcVersions, err := productsapi.GetProductVersions("Percona-XtraDB-Cluster-", "Percona-XtraDB-Cluster-80", "Percona-XtraDB-Cluster-57") + pxcVersions, err := productsapi.GetProductVersions("Percona-XtraDB-Cluster-", "Percona-XtraDB-Cluster-84", "Percona-XtraDB-Cluster-80", "Percona-XtraDB-Cluster-57") if err != nil { return nil, err } - xtrabackupVersions, err := productsapi.GetProductVersions("Percona-XtraBackup-", "Percona-XtraBackup-8.0", "Percona-XtraBackup-2.4") + xtrabackupVersions, err := productsapi.GetProductVersions("Percona-XtraBackup-", "Percona-XtraBackup-8.4", "Percona-XtraBackup-8.0", "Percona-XtraBackup-2.4") if err != nil { return nil, err } From fdc56d53715fd380cc62485c606a3fea2ac39748 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 27 Feb 2025 12:39:01 +0200 Subject: [PATCH 19/20] use --only-latest without --file --- tools/operator-tool/README.md | 2 +- tools/operator-tool/cmd/main.go | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/tools/operator-tool/README.md b/tools/operator-tool/README.md index 79c58f76..ae898d42 100644 --- a/tools/operator-tool/README.md +++ b/tools/operator-tool/README.md @@ -18,7 +18,7 @@ Usage of ./bin/operator-tool: -include-arch-images Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file -only-latest - Add only latest images to the specified "-file" + Add only latest major version images to the specified "-file". If "-file" is not specified, returns a file with latest major versions. -operator string Operator name. Available values: [psmdb pxc ps pg] -patch string diff --git a/tools/operator-tool/cmd/main.go b/tools/operator-tool/cmd/main.go index ab40607a..49a30dd0 100644 --- a/tools/operator-tool/cmd/main.go +++ b/tools/operator-tool/cmd/main.go @@ -41,7 +41,7 @@ var ( verbose = flag.Bool("verbose", false, "Show logs") includeMultiImages = flag.Bool("include-arch-images", false, `Include images with "-multi", "-arm64", "-aarch64" suffixes in the output file`) versionCap = flag.Int("cap", 0, `Sets a limit on the number of versions allowed for each major version of a product`) - onlyLatest = flag.Bool("only-latest", false, `Add only latest images to the specified "-file"`) + onlyLatest = flag.Bool("only-latest", false, `Add only latest major version images to the specified "-file". If "-file" is not specified, returns a file with latest major versions.`) ) func main() { @@ -87,17 +87,28 @@ func printSourceFile(operatorName, version, file, patchFile string, includeArchS registryClient := registry.NewClient() - if file == "" || patchFile == "" { + if file != "" && patchFile != "" { + patchMatrix, err := util.ReadPatchFile(patchFile) + if err != nil { + return fmt.Errorf("failed to read patch file: %w", err) + } + productResponse, err = patchProductResponse(registryClient, file, patchMatrix, version) + if err != nil { + return fmt.Errorf("failed to patch product response: %w", err) + } + } else { productResponse, err = getProductResponse(registryClient, operatorName, version, includeArchSuffixes) if err != nil { return fmt.Errorf("failed to get product response: %w", err) } + if onlyLatest { + if err := keepOnlyLatestVersions(productResponse.Versions[0].Matrix); err != nil { + return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) + } + } + if file != "" { if onlyLatest { - if err := keepOnlyLatestVersions(productResponse.Versions[0].Matrix); err != nil { - return fmt.Errorf("failed to delete old verisons from version matrix: %w", err) - } - productResponse, err = patchProductResponse(registryClient, file, productResponse.Versions[0].Matrix, version) if err != nil { return fmt.Errorf("failed to patch product response: %w", err) @@ -108,15 +119,6 @@ func printSourceFile(operatorName, version, file, patchFile string, includeArchS } } } - } else { - patchMatrix, err := util.ReadPatchFile(patchFile) - if err != nil { - return fmt.Errorf("failed to read patch file: %w", err) - } - productResponse, err = patchProductResponse(registryClient, file, patchMatrix, version) - if err != nil { - return fmt.Errorf("failed to patch product response: %w", err) - } } if err := updateMatrixStatuses(productResponse.Versions[0].Matrix); err != nil { From 6ebf2b3b5ca2fdc5ca02f222e62572e950fb7870 Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Thu, 27 Feb 2025 13:49:29 +0200 Subject: [PATCH 20/20] use only '-multi' and '-amd64' images --- tools/operator-tool/internal/filler/filler.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/operator-tool/internal/filler/filler.go b/tools/operator-tool/internal/filler/filler.go index 32bcff6e..a3b1b7cc 100644 --- a/tools/operator-tool/internal/filler/filler.go +++ b/tools/operator-tool/internal/filler/filler.go @@ -254,12 +254,19 @@ func getVersionMapLatestVer(rc *registry.RegistryClient, imageName string, inclu // Some images on Docker Hub are tagged like , -arm64, -aarch64, -amd64, and -multi. // This function adds images with amd64 and arm64 builds to the provided map. // +// Specify includedArchSuffixes to include images with suffixes in resulting map. +// By default includedArchSuffixes includes only -multi and -amd64 images. +// // Logic: // - If an image supports both amd64 and arm64 architectures and has a "-multi" suffix in its tag, // the function includes a version of the image tag without the "-multi" suffix in the map. // - If no image with both amd64 and arm64 builds is found, separate images for amd64 and arm64 -// are added individually. -func versionMapFromImages(baseTag string, images []registry.Image) (map[string]*vsAPI.Version, error) { +// are added individually. (if both amd64 and arm64 image suffixes are specified in the includedArchSuffixes) +func versionMapFromImages(baseTag string, images []registry.Image, includedArchSuffixes ...string) (map[string]*vsAPI.Version, error) { + if len(includedArchSuffixes) == 0 { + includedArchSuffixes = []string{"-multi", "-amd64"} + } + baseTag = trimArchSuffix(baseTag) slices.SortFunc(images, func(a, b registry.Image) int { @@ -290,7 +297,7 @@ func versionMapFromImages(baseTag string, images []registry.Image) (map[string]* } versions := make(map[string]*vsAPI.Version) - if multiImage != nil { + if multiImage != nil && slices.Contains(includedArchSuffixes, "-multi") { versions[baseTag+getArchSuffix(multiImage.Tag)] = &vsAPI.Version{ ImagePath: multiImage.FullName(), ImageHash: multiImage.DigestAMD64, @@ -300,14 +307,14 @@ func versionMapFromImages(baseTag string, images []registry.Image) (map[string]* return versions, nil } } - if amd64Image != nil { + if amd64Image != nil && slices.Contains(includedArchSuffixes, "-amd64") { versions[baseTag+getArchSuffix(amd64Image.Tag)] = &vsAPI.Version{ ImagePath: amd64Image.FullName(), ImageHash: amd64Image.DigestAMD64, } } // Include arm64 if multi image is not specified - if multiImage == nil && arm64Image != nil { + if multiImage == nil && arm64Image != nil && slices.Contains(includedArchSuffixes, "-arm64") && slices.Contains(includedArchSuffixes, "-aarch64") { versions[baseTag+getArchSuffix(arm64Image.Tag)] = &vsAPI.Version{ ImagePath: arm64Image.FullName(), ImageHashArm64: arm64Image.DigestARM64,