diff --git a/README.md b/README.md index 218f521..a358b4c 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ Table of Contents: | **[Instance Snapshot Cleaner](jobs/instances-snapshot-cleaner/README.md)**
Use Serverless Jobs to clean old instances snapshots | Go | [Console] | | **[Registry Tag Cleaner](jobs/registry-version-based-retention/README.md)**
Use Serverless Jobs to keep a desired amount of tags for each image | Go | [Console] | | **[Registry Empty Image Cleaner](jobs/registry-empty-ressource-cleaner/README.md)**
Use Serverless Jobs to clean container registry empty namespaces and images | Go | [Console] | +| **[Instance snapshots to S3](jobs/block-snapshot-s3-archiver/README.md)**
Use Serverless Jobs to move Instances snapshots to S3 Object Storage | Go | [Console] | | **[Block snapshots to S3](jobs/block-snapshot-s3-archiver/README.md)**
Use Serverless Jobs to move Block Storage snapshots to S3 Object Storage | Go | [Console] | ### 💬 Messaging and Queueing diff --git a/jobs/snapshot-s3-archiver/Dockerfile b/jobs/snapshot-s3-archiver/Dockerfile new file mode 100644 index 0000000..9bd7d51 --- /dev/null +++ b/jobs/snapshot-s3-archiver/Dockerfile @@ -0,0 +1,27 @@ +# Build stage +FROM golang:1.25-alpine AS builder + +WORKDIR /app + +# Copy required files +COPY go.mod go.sum ./ +RUN go mod download + +COPY *.go ./ + +# Build the executable +RUN CGO_ENABLED=0 GOOS=linux go build -o /jobs-snapshot-s3 + +# Final stage +FROM alpine:latest + +WORKDIR /app + +# Install CA certificates for HTTPS +RUN apk --no-cache add ca-certificates + +# Copy the binary from the builder stage +COPY --from=builder /jobs-snapshot-s3 /app/jobs-snapshot-s3 + +# Run the executable +ENTRYPOINT [ "/app/jobs-snapshot-s3" ] diff --git a/jobs/snapshot-s3-archiver/README.md b/jobs/snapshot-s3-archiver/README.md new file mode 100644 index 0000000..b09e677 --- /dev/null +++ b/jobs/snapshot-s3-archiver/README.md @@ -0,0 +1,81 @@ +# Scaleway Instance Snapshot Archiver + +Automated Serverless Job to archive Scaleway Instance snapshots to Object Storage S3. + +## Overview + +This tool automatically finds available snapshots of Scaleway Instances volumes, exports them to a specified S3 bucket in `.qcow2` format, and deletes the original snapshot to optimize storage costs. It's designed to run as a Serverless Job on Scaleway and skips snapshots that have already been archived. + +The main logic is implemented in `main.go`, which: +1. Loads configuration from environment variables. +2. Connects to Scaleway APIs using the Scaleway SDK. +3. Lists all available snapshots in the project. +4. Checks the target S3 bucket for already-archived snapshots. +5. Exports new snapshots to the bucket. +6. Deletes successfully exported snapshots to reduce storage costs. + +## Features + +- **Automated Export**: Finds available snapshots and exports them to an S3 bucket in `.qcow2` format. +- **Cost Optimization**: Deletes the source snapshot after successful export to reduce storage costs. +- **Idempotent**: Skips snapshots that are already archived in the bucket. +- **Serverless Ready**: Designed for [Scaleway Serverless Jobs](https://www.scaleway.com/en/serverless-jobs/). + +## Step 1 : Build and push to Container registry + +Serverless Jobs, like Serverless Containers (which are suited for HTTP applications), works +with containers. So first, use your terminal reach this folder and run the following commands: + +```shell +# First command is to login to container registry, you can find it in Scaleway console +docker login rg.fr-par.scw.cloud/snapshot-s3-archiver -u nologin --password-stdin <<< "$SCW_SECRET_KEY" + +# Here we build the image to push +docker buildx build --platform linux/amd64 -t rg.fr-par.scw.cloud/snapshot-s3-archiver/snapshot-s3-archiver:v1 . + +# Push the image online to be used on Serverless Jobs +docker push rg.fr-par.scw.cloud/snapshot-s3-archiver/snapshot-s3-archiver:v1 +``` +> [!TIP] +> As we do not expose a web server and we do not require features such as auto-scaling, Serverless Jobs are perfect for this use case. +To check if everyting is ok, on the Scaleway Console you can verify if your tag is present in Container Registry. + +## Step 2: Creating the Job Definition + +On Scaleway Console on the following link you can create a new Job Definition: https://console.scaleway.com/serverless-jobs/jobs/create?region=fr-par + +1. On Container image, select the image you created in the step before. +2. You can set the job definition name name to something clear. +3. Regarding the resources you can keep the default values, this job is fast and do not require specific compute power or memory. +4. To schedule your job for example every night at 2am, you can set the cron to `0 2 * * *`. +5. Important: advanced option, you need to set the following environment variables: + +> [!TIP] +> For sensitive data like `SCW_ACCESS_KEY` and `SCW_SECRET_KEY` we recommend to inject them via Secret Manager, [more info here](https://www.scaleway.com/en/docs/serverless/jobs/how-to/reference-secret-in-job/). +| Variable | Description | +|---|---| +| `SCW_DEFAULT_ORGANIZATION_ID` | Organization ID . | +| `SCW_DEFAULT_PROJECT_ID` | Project ID (Recommended resource grouping). | +| `SCW_ACCESS_KEY` | IAM Access Key. | +| `SCW_SECRET_KEY` | IAM Secret Key. | +| `SCW_ZONE` | Zone of the snapshots (e.g., `fr-par-1`). | +| `SCW_BUCKET_NAME` | S3 Bucket name for archives. | +| `SCW_BUCKET_ENDPOINT` | S3 Endpoint (e.g., `s3.fr-par.scw.cloud`). | + +* Then click "create job" + +## Step 3: Run the job + +On your created Job Definition, just click the button "Run Job" and within seconds it should be successful. + +## Troubleshooting + +If your Job Run state goes in error, you can use the "Logs" tab in Scaleway Console to get more informations about the error. + +# Additional content + +- [Jobs Documentation](https://www.scaleway.com/en/docs/serverless/jobs/how-to/create-job-from-scaleway-registry/) +- [Other methods to deploy Jobs](https://www.scaleway.com/en/docs/serverless/jobs/reference-content/deploy-job/) +- [Secret key / access key doc](https://www.scaleway.com/en/docs/identity-and-access-management/iam/how-to/create-api-keys/) +- [CRON schedule help](https://www.scaleway.com/en/docs/serverless/jobs/reference-content/cron-schedules/) +- \ No newline at end of file diff --git a/jobs/snapshot-s3-archiver/config.go b/jobs/snapshot-s3-archiver/config.go new file mode 100644 index 0000000..960347a --- /dev/null +++ b/jobs/snapshot-s3-archiver/config.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "os" + + "github.com/scaleway/scaleway-sdk-go/scw" +) + +// Environment variable constants +const ( + envOrgID = "SCW_DEFAULT_ORGANIZATION_ID" + envAccessKey = "SCW_ACCESS_KEY" + envSecretKey = "SCW_SECRET_KEY" + envProjectID = "SCW_DEFAULT_PROJECT_ID" + envZone = "SCW_ZONE" + envBucket = "SCW_BUCKET_NAME" + envBucketEndpoint = "SCW_BUCKET_ENDPOINT" +) + +type Config struct { + OrgID string + AccessKey string + SecretKey string + ProjectID string + Zone scw.Zone + BucketName string + BucketEndpoint string +} + +func LoadConfig() (*Config, error) { + // Mandatory variables + vars := map[string]*string{ + envAccessKey: new(string), + envSecretKey: new(string), + envProjectID: new(string), + envZone: new(string), + envBucket: new(string), + envBucketEndpoint: new(string), + } + + // Optional variables + orgID := os.Getenv(envOrgID) + + for envKey, valPtr := range vars { + val := os.Getenv(envKey) + if val == "" { + return nil, fmt.Errorf("missing environment variable %s", envKey) + } + *valPtr = val + } + + return &Config{ + OrgID: orgID, + AccessKey: *vars[envAccessKey], + SecretKey: *vars[envSecretKey], + ProjectID: *vars[envProjectID], + Zone: scw.Zone(*vars[envZone]), + BucketName: *vars[envBucket], + BucketEndpoint: *vars[envBucketEndpoint], + }, nil +} diff --git a/jobs/snapshot-s3-archiver/go.mod b/jobs/snapshot-s3-archiver/go.mod new file mode 100644 index 0000000..c420369 --- /dev/null +++ b/jobs/snapshot-s3-archiver/go.mod @@ -0,0 +1,31 @@ +module github.com/scaleway/serverless-examples/jobs/snapshot-s3-archiver + +go 1.25 + +require ( + github.com/minio/minio-go/v7 v7.0.95 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tinylib/msgp v1.4.0 // indirect + golang.org/x/crypto v0.42.0 // indirect + golang.org/x/net v0.44.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/jobs/snapshot-s3-archiver/go.sum b/jobs/snapshot-s3-archiver/go.sum new file mode 100644 index 0000000..79cac08 --- /dev/null +++ b/jobs/snapshot-s3-archiver/go.sum @@ -0,0 +1,62 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35 h1:8xfn1RzeI9yoCUuEwDy08F+No6PcKZGEDOQ6hrRyLts= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35/go.mod h1:47B1d/YXmSAxlJxUJxClzHR6b3T4M1WyCvwENPQNBWc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8= +github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jobs/snapshot-s3-archiver/main.go b/jobs/snapshot-s3-archiver/main.go new file mode 100644 index 0000000..ab0906e --- /dev/null +++ b/jobs/snapshot-s3-archiver/main.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "log/slog" + "os" + "slices" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +const snapshotExtension = ".qcow2" + +func main() { + // Configure valid JSON logger + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + slog.SetDefault(logger) + + // Load configuration + cfg, err := LoadConfig() + if err != nil { + slog.Error("Failed to load configuration", "error", err) + os.Exit(1) + } + + // Create Scaleway client using the implementation in config + client, err := scw.NewClient( + scw.WithDefaultOrganizationID(cfg.OrgID), + scw.WithAuth(cfg.AccessKey, cfg.SecretKey), + scw.WithDefaultProjectID(cfg.ProjectID), + scw.WithDefaultZone(cfg.Zone), + ) + if err != nil { + slog.Error("Failed to create Scaleway client", "error", err) + os.Exit(1) + } + + slog.Info("Initializing instance API...") + instanceAPI := instance.NewAPI(client) + + slog.Info("Reading all snapshots for the project...") + snapList, err := instanceAPI.ListSnapshots(&instance.ListSnapshotsRequest{}, scw.WithAllPages()) + if err != nil { + slog.Error("Failed to list snapshots", "error", err) + os.Exit(1) + } + + slog.Info("Reading all snapshots already in the bucket...") + filesInBucket, err := listBucketFiles(cfg) + if err != nil { + slog.Error("Failed to list bucket files", "error", err) + os.Exit(1) + } + + processSnapshots(instanceAPI, cfg, snapList.Snapshots, filesInBucket) +} + +func listBucketFiles(cfg *Config) ([]string, error) { + minioClient, err := minio.New(cfg.BucketEndpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: true, + }) + if err != nil { + return nil, err + } + + ctx := context.Background() + var files []string + + // List all objects in the bucket + // Use a closed channel to signal cancellation if needed (not used here but good practice) + // minioClient.ListObjects usually takes a channel for cancellation if needed, but here we pass context + opts := minio.ListObjectsOptions{ + Recursive: false, + WithMetadata: true, + } + + for object := range minioClient.ListObjects(ctx, cfg.BucketName, opts) { + if object.Err != nil { + return nil, object.Err + } + files = append(files, object.Key) + } + + return files, nil +} + +func processSnapshots(api *instance.API, cfg *Config, snapshots []*instance.Snapshot, filesInBucket []string) { + for _, snapshot := range snapshots { + logger := slog.With("snapshot_id", snapshot.ID, "snapshot_name", snapshot.Name) + + logger.Info("Checking snapshot") + + if snapshot.State != instance.SnapshotStateAvailable { + logger.Info("Skipping snapshot (not available)", "status", snapshot.State.String()) + continue + } + + filename := snapshot.Name + snapshotExtension + + if slices.Contains(filesInBucket, filename) { + logger.Info("File already exists in bucket, deleting local snapshot") + if err := api.DeleteSnapshot(&instance.DeleteSnapshotRequest{SnapshotID: snapshot.ID}); err != nil { + logger.Error("Failed to delete snapshot", "error", err) + } + continue + } + + logger.Info("Exporting snapshot to bucket") + snap, err := api.ExportSnapshot(&instance.ExportSnapshotRequest{ + SnapshotID: snapshot.ID, + Bucket: cfg.BucketName, + Key: filename, + }) + if err != nil { + logger.Error("Failed to export snapshot", "error", err) + continue + } + + logger.Info("Successfully started export", "task_id", snap.Task.ID, "bucket", cfg.BucketName, "description", snap.Task.Description) + } +}