From d957ca7fdf165371cdc62662d2a5ed9e60bc66c8 Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Wed, 3 Sep 2025 10:16:51 +0200 Subject: [PATCH 1/4] feat(jobs): move snapshots to s3 --- jobs/snapshot-s3-archiver/Dockerfile | 16 +++++ jobs/snapshot-s3-archiver/README.md | 1 + jobs/snapshot-s3-archiver/go.mod | 12 ++++ jobs/snapshot-s3-archiver/go.sum | 19 ++++++ jobs/snapshot-s3-archiver/main.go | 97 ++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+) create mode 100644 jobs/snapshot-s3-archiver/Dockerfile create mode 100644 jobs/snapshot-s3-archiver/README.md create mode 100644 jobs/snapshot-s3-archiver/go.mod create mode 100644 jobs/snapshot-s3-archiver/go.sum create mode 100644 jobs/snapshot-s3-archiver/main.go diff --git a/jobs/snapshot-s3-archiver/Dockerfile b/jobs/snapshot-s3-archiver/Dockerfile new file mode 100644 index 0000000..c95d9d6 --- /dev/null +++ b/jobs/snapshot-s3-archiver/Dockerfile @@ -0,0 +1,16 @@ +# Using apline/golang image +FROM golang:1.25-alpine + +# Set destination for COPY +WORKDIR /app + +# Copy required files +COPY go.mod ./ +COPY go.sum ./ +COPY main.go ./ + +# Build the executable +RUN go build -o /jobs-snapshot-s3 + +# Run the executable +ENTRYPOINT [ "/jobs-snapshot-s3" ] diff --git a/jobs/snapshot-s3-archiver/README.md b/jobs/snapshot-s3-archiver/README.md new file mode 100644 index 0000000..3a19e81 --- /dev/null +++ b/jobs/snapshot-s3-archiver/README.md @@ -0,0 +1 @@ +# Serverless Jobs to move snapshots to S3 \ No newline at end of file diff --git a/jobs/snapshot-s3-archiver/go.mod b/jobs/snapshot-s3-archiver/go.mod new file mode 100644 index 0000000..d5b7ce8 --- /dev/null +++ b/jobs/snapshot-s3-archiver/go.mod @@ -0,0 +1,12 @@ +module github.com/scaleway/serverless-examples/jobs/instances-snapshot + +go 1.25 + +require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.34 + +require ( + github.com/kr/pretty v0.3.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/jobs/snapshot-s3-archiver/go.sum b/jobs/snapshot-s3-archiver/go.sum new file mode 100644 index 0000000..55bdfbc --- /dev/null +++ b/jobs/snapshot-s3-archiver/go.sum @@ -0,0 +1,19 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.34 h1:48+VFHsyVcAHIN2v1Ao9v1/RkjJS5AwctFucBrfYNIA= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.34/go.mod h1:zFWiHphneiey3s8HOtAEnGrRlWivNaxW5T6d5Xfco7g= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/jobs/snapshot-s3-archiver/main.go b/jobs/snapshot-s3-archiver/main.go new file mode 100644 index 0000000..d1ff899 --- /dev/null +++ b/jobs/snapshot-s3-archiver/main.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "os" + + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/scaleway/scaleway-sdk-go/scw" +) + +// Environment variable constants used to configure the Scaleway API client. +// These must be set in the environment for the application to authenticate and interact with Scaleway services. +const ( + // envOrgID is the Scaleway Organization ID, used for billing and resource ownership (legacy; prefer Project ID). + envOrgID = "SCW_DEFAULT_ORGANIZATION_ID" + + // envAccessKey is the API access key for authenticating requests to Scaleway. + envAccessKey = "SCW_ACCESS_KEY" + + // envSecretKey is the secret key associated with the access key, used for signing requests. + envSecretKey = "SCW_SECRET_KEY" + + // envProjectID is the Scaleway Project ID, which groups resources and is the preferred way to organize infrastructure. + envProjectID = "SCW_DEFAULT_PROJECT_ID" + + // envZone specifies the geographical region/zone where resources will be created (e.g., fr-par-1). + envZone = "SCW_ZONE" + + // envBucket is a custom environment variable for specifying the name of an S3-compatible bucket. + // This is not a standard Scaleway variable and is application-specific. + envBucket = "SCW_BUCKET_NAME" +) + +func main() { + fmt.Println("moving snapshots to s3...") + + // Create a Scaleway client with credentials from environment variables. + client, err := scw.NewClient( + // Get your organization ID at https://console.scaleway.com/organization/settings + scw.WithDefaultOrganizationID(os.Getenv(envOrgID)), + + // Get your credentials at https://console.scaleway.com/iam/api-keys + scw.WithAuth(os.Getenv(envAccessKey), os.Getenv(envSecretKey)), + + // Set the default project ID to organize resources under a specific project + scw.WithDefaultProjectID(os.Getenv(envProjectID)), + + // Set the default zone where resources such as block volumes and snapshots are located + scw.WithDefaultZone(scw.Zone(os.Getenv(envZone))), + ) + if err != nil { + panic(err) + } + + instanceAPI := instance.NewAPI(client) + + snapList, err := instanceAPI.ListSnapshots(&instance.ListSnapshotsRequest{}, scw.WithAllPages()) + if err != nil { + panic(err) + } + + fmt.Printf("number of snapshots: %d\n", snapList.TotalCount) + + for _, snapshot := range snapList.Snapshots { + fmt.Printf("snap %s\n", snapshot.Name) + + if snapshot.State == instance.SnapshotStateAvailable { + fmt.Printf("Exporting snapshot %s (ID: %s) to bucket %s...\n", snapshot.Name, snapshot.ID, os.Getenv(envBucket)) + + snap, err := instanceAPI.ExportSnapshot(&instance.ExportSnapshotRequest{ + SnapshotID: snapshot.ID, + Bucket: os.Getenv(envBucket), + Key: snapshot.Name + ".qcow2", + }) + if err != nil { + fmt.Printf("Failed to export snapshot %s: %v\n", snapshot.Name, err) + + continue + } + + fmt.Printf("Successfully started export of %s to %s/%s\n", snap.Task.ID, os.Getenv(envBucket), snap.Task.Description) + } else { + fmt.Printf("Skipping snapshot %s (ID: %s) - status is %s, not available\n", snapshot.Name, snapshot.ID, snapshot.State.String()) + } + } +} + +// Check for mandatory variables before starting to work. +func init() { + mandatoryVariables := [...]string{envOrgID, envAccessKey, envSecretKey, envZone, envProjectID, envBucket} + + for idx := range mandatoryVariables { + if os.Getenv(mandatoryVariables[idx]) == "" { + panic("missing environment variable " + mandatoryVariables[idx]) + } + } +} From 57cf105f93470b1f939f0dad16297e3f5d934c82 Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Sun, 21 Sep 2025 17:13:07 +0200 Subject: [PATCH 2/4] feat(jobs): s3 snapshot utility --- jobs/snapshot-s3-archiver/README.md | 134 +++++++++++++++++++++++++++- jobs/snapshot-s3-archiver/go.mod | 23 ++++- jobs/snapshot-s3-archiver/go.sum | 47 +++++++++- jobs/snapshot-s3-archiver/main.go | 90 +++++++++++++++++-- 4 files changed, 282 insertions(+), 12 deletions(-) diff --git a/jobs/snapshot-s3-archiver/README.md b/jobs/snapshot-s3-archiver/README.md index 3a19e81..8f4cf60 100644 --- a/jobs/snapshot-s3-archiver/README.md +++ b/jobs/snapshot-s3-archiver/README.md @@ -1 +1,133 @@ -# Serverless Jobs to move snapshots to S3 \ No newline at end of file +# Scaleway Instance Snapshot Backup to S3 + +This project exports available Scaleway Instance snapshots to an S3-compatible bucket (e.g., Scaleway Object Storage), and optionally deletes the snapshot afterward if it's already backed up. It's designed to run as a **Scaleway Serverless Job**, making it ideal for automated, scheduled backups. + +--- + +## ๐Ÿ“ฆ Features + +- Lists all available block storage snapshots in a project. +- Checks if a snapshot with the same name already exists in the target bucket. +- Exports missing snapshots to the bucket in `.qcow2` format. +- Deletes local snapshot after successful export (if not already in bucket). +- Uses environment variables for full configuration. +- Built to run in a container on [Scaleway Serverless Jobs](https://www.scaleway.com/en/serverless-jobs/). + +--- + +## โš™๏ธ Environment Variables + +You must set the following environment variables when deploying the job: + +| Variable | Description | +|--------|-------------| +| `SCW_DEFAULT_ORGANIZATION_ID` | Your Scaleway Organization ID (legacy; prefer project ID). | +| `SCW_DEFAULT_PROJECT_ID` | Your Scaleway Project ID (preferred way to group resources). | +| `SCW_ACCESS_KEY` | API access key (from IAM). | +| `SCW_SECRET_KEY` | API secret key (from IAM). | +| `SCW_ZONE` | Zone where your snapshots are located (e.g., `fr-par-1`). | +| `SCW_BUCKET_NAME` | Name of the S3 bucket to store exported snapshots. | +| `SCW_BUCKET_ENDPOINT` | S3 endpoint (e.g., `s3.fr-par.scw.cloud`). | + +> ๐Ÿ” **Security Tip**: Use IAM API keys with minimal required permissions. + +--- + +## ๐Ÿ› ๏ธ Build & Deploy to Scaleway Serverless Jobs + +### 1. Build the Docker Image + +```bash +docker build -t snapshot-s3-backup . +``` + +### 2. Tag and Push to Scaleway Container Registry (or any registry) + +```bash +# Example using Scaleway CR +docker tag snapshot-s3-backup fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 +docker push fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 +``` + +> Replace `your-registry` with your actual container registry name. + +### 3. Create the Serverless Job + +Use the Scaleway CLI or Console: + +#### Using `scw` CLI: + +```bash +scw job create \ + name=backup-snapshots \ + image=fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 \ + memory-limit=512Mi \ + cpu-limit=500m \ + environment='{ + "SCW_DEFAULT_PROJECT_ID": "your-project-id", + "SCW_ACCESS_KEY": "your-access-key", + "SCW_SECRET_KEY": "your-secret-key", + "SCW_ZONE": "fr-par-1", + "SCW_BUCKET_NAME": "my-backup-bucket", + "SCW_BUCKET_ENDPOINT": "s3.fr-par.scw.cloud" + }' +``` + +### 4. (Optional) Schedule the Job + +Schedule it to run daily using a cron trigger: + +```bash +scw scheduler trigger create-cron \ + job-id=your-job-id \ + schedule="0 2 * * *" \ + name=daily-snapshot-backup +``` + +This runs the job every day at 2 AM. + +--- + +## ๐Ÿ“ Output Format + +Each snapshot is exported as: +``` +.qcow2 +``` + +Example: +``` +my-server-disk-2025-04-05.qcow2 +``` + +--- + +## โœ… Example Use Case + +Run nightly to: +1. Export new snapshots to object storage. +2. Clean up old snapshots once safely backed up. +3. Reduce storage costs and improve disaster recovery. + +--- + +## ๐Ÿงช Local Testing (Optional) + +Set environment variables: + +```bash +export SCW_DEFAULT_PROJECT_ID=... +export SCW_ACCESS_KEY=... +export SCW_SECRET_KEY=... +export SCW_ZONE=fr-par-1 +export SCW_BUCKET_NAME=my-backup-bucket +export SCW_BUCKET_ENDPOINT=s3.fr-par.scw.cloud +``` + +Run: + +```bash +go run main.go +``` + +--- \ No newline at end of file diff --git a/jobs/snapshot-s3-archiver/go.mod b/jobs/snapshot-s3-archiver/go.mod index d5b7ce8..c420369 100644 --- a/jobs/snapshot-s3-archiver/go.mod +++ b/jobs/snapshot-s3-archiver/go.mod @@ -1,12 +1,31 @@ -module github.com/scaleway/serverless-examples/jobs/instances-snapshot +module github.com/scaleway/serverless-examples/jobs/snapshot-s3-archiver go 1.25 -require github.com/scaleway/scaleway-sdk-go v1.0.0-beta.34 +require ( + github.com/minio/minio-go/v7 v7.0.95 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35 +) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/pretty v0.3.1 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tinylib/msgp v1.4.0 // indirect + golang.org/x/crypto v0.42.0 // indirect + golang.org/x/net v0.44.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/jobs/snapshot-s3-archiver/go.sum b/jobs/snapshot-s3-archiver/go.sum index 55bdfbc..79cac08 100644 --- a/jobs/snapshot-s3-archiver/go.sum +++ b/jobs/snapshot-s3-archiver/go.sum @@ -1,4 +1,21 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -6,14 +23,40 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.34 h1:48+VFHsyVcAHIN2v1Ao9v1/RkjJS5AwctFucBrfYNIA= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.34/go.mod h1:zFWiHphneiey3s8HOtAEnGrRlWivNaxW5T6d5Xfco7g= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35 h1:8xfn1RzeI9yoCUuEwDy08F+No6PcKZGEDOQ6hrRyLts= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.35/go.mod h1:47B1d/YXmSAxlJxUJxClzHR6b3T4M1WyCvwENPQNBWc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8= +github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jobs/snapshot-s3-archiver/main.go b/jobs/snapshot-s3-archiver/main.go index d1ff899..d068d72 100644 --- a/jobs/snapshot-s3-archiver/main.go +++ b/jobs/snapshot-s3-archiver/main.go @@ -1,11 +1,16 @@ package main import ( + "context" "fmt" "os" + "slices" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" "github.com/scaleway/scaleway-sdk-go/scw" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) // Environment variable constants used to configure the Scaleway API client. @@ -29,11 +34,11 @@ const ( // envBucket is a custom environment variable for specifying the name of an S3-compatible bucket. // This is not a standard Scaleway variable and is application-specific. envBucket = "SCW_BUCKET_NAME" + + envBucketEndpoint = "SCW_BUCKET_ENDPOINT" ) func main() { - fmt.Println("moving snapshots to s3...") - // Create a Scaleway client with credentials from environment variables. client, err := scw.NewClient( // Get your organization ID at https://console.scaleway.com/organization/settings @@ -52,25 +57,50 @@ func main() { panic(err) } + fmt.Println("Initializing instance API...") + instanceAPI := instance.NewAPI(client) + fmt.Println("Reading all snapshots for the project...") + snapList, err := instanceAPI.ListSnapshots(&instance.ListSnapshotsRequest{}, scw.WithAllPages()) if err != nil { panic(err) } - fmt.Printf("number of snapshots: %d\n", snapList.TotalCount) + fmt.Println("Reading all snapshots already in the bucket...") + + filesInBucket, err := listBucketFiles() + if err != nil { + panic(err) + } + + const snapshotExtension = ".qcow2" for _, snapshot := range snapList.Snapshots { - fmt.Printf("snap %s\n", snapshot.Name) + fmt.Printf("Checking for snapshot %s\n", snapshot.Name) if snapshot.State == instance.SnapshotStateAvailable { - fmt.Printf("Exporting snapshot %s (ID: %s) to bucket %s...\n", snapshot.Name, snapshot.ID, os.Getenv(envBucket)) + // Check if file already exists in bucket + if slices.Contains(filesInBucket, snapshot.Name+snapshotExtension) { + fmt.Printf("File %s already exists in bucket, can delete the snapshot and skip it\n", snapshot.Name+snapshotExtension) + + err = instanceAPI.DeleteSnapshot(&instance.DeleteSnapshotRequest{ + SnapshotID: snapshot.ID, + }) + if err != nil { + panic(err) + } + + continue + } + + fmt.Printf("File %s not present in the bucket, expording it to the bucket...\n", snapshot.Name+".qcow2") snap, err := instanceAPI.ExportSnapshot(&instance.ExportSnapshotRequest{ SnapshotID: snapshot.ID, Bucket: os.Getenv(envBucket), - Key: snapshot.Name + ".qcow2", + Key: snapshot.Name + snapshotExtension, }) if err != nil { fmt.Printf("Failed to export snapshot %s: %v\n", snapshot.Name, err) @@ -87,7 +117,15 @@ func main() { // Check for mandatory variables before starting to work. func init() { - mandatoryVariables := [...]string{envOrgID, envAccessKey, envSecretKey, envZone, envProjectID, envBucket} + mandatoryVariables := [...]string{ + envOrgID, + envAccessKey, + envSecretKey, + envZone, + envProjectID, + envBucket, + envBucketEndpoint, + } for idx := range mandatoryVariables { if os.Getenv(mandatoryVariables[idx]) == "" { @@ -95,3 +133,41 @@ func init() { } } } + +func listBucketFiles() ([]string, error) { + // Retrieve S3-compatible endpoint and credentials from environment + endpoint := os.Getenv(envBucketEndpoint) + accessKeyID := os.Getenv(envAccessKey) + secretAccessKey := os.Getenv(envSecretKey) + + // Create new MinIO client + minioClient, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: true, + }) + if err != nil { + return nil, err + } + + // Set up context and result slice + ctx := context.Background() + var files []string + + // Channel to signal listing completion + doneCh := make(chan struct{}) + defer close(doneCh) + + // List all objects in the bucket + for object := range minioClient.ListObjects(ctx, os.Getenv(envBucket), minio.ListObjectsOptions{ + Recursive: false, + WithMetadata: true, + }) { + if object.Err != nil { + return nil, object.Err + } + + files = append(files, object.Key) + } + + return files, nil +} From 12b761123b482e802e43ca2333efe0429e4ef120 Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Sat, 27 Dec 2025 11:43:38 +0100 Subject: [PATCH 3/4] modernize project structure --- jobs/snapshot-s3-archiver/Dockerfile | 27 ++-- jobs/snapshot-s3-archiver/README.md | 139 ++++--------------- jobs/snapshot-s3-archiver/config.go | 62 +++++++++ jobs/snapshot-s3-archiver/main.go | 198 ++++++++++----------------- 4 files changed, 181 insertions(+), 245 deletions(-) create mode 100644 jobs/snapshot-s3-archiver/config.go diff --git a/jobs/snapshot-s3-archiver/Dockerfile b/jobs/snapshot-s3-archiver/Dockerfile index c95d9d6..9bd7d51 100644 --- a/jobs/snapshot-s3-archiver/Dockerfile +++ b/jobs/snapshot-s3-archiver/Dockerfile @@ -1,16 +1,27 @@ -# Using apline/golang image -FROM golang:1.25-alpine +# Build stage +FROM golang:1.25-alpine AS builder -# Set destination for COPY WORKDIR /app # Copy required files -COPY go.mod ./ -COPY go.sum ./ -COPY main.go ./ +COPY go.mod go.sum ./ +RUN go mod download + +COPY *.go ./ # Build the executable -RUN go build -o /jobs-snapshot-s3 +RUN CGO_ENABLED=0 GOOS=linux go build -o /jobs-snapshot-s3 + +# Final stage +FROM alpine:latest + +WORKDIR /app + +# Install CA certificates for HTTPS +RUN apk --no-cache add ca-certificates + +# Copy the binary from the builder stage +COPY --from=builder /jobs-snapshot-s3 /app/jobs-snapshot-s3 # Run the executable -ENTRYPOINT [ "/jobs-snapshot-s3" ] +ENTRYPOINT [ "/app/jobs-snapshot-s3" ] diff --git a/jobs/snapshot-s3-archiver/README.md b/jobs/snapshot-s3-archiver/README.md index 8f4cf60..7d261a8 100644 --- a/jobs/snapshot-s3-archiver/README.md +++ b/jobs/snapshot-s3-archiver/README.md @@ -1,133 +1,44 @@ -# Scaleway Instance Snapshot Backup to S3 +# Scaleway Instance Snapshot Archiver -This project exports available Scaleway Instance snapshots to an S3-compatible bucket (e.g., Scaleway Object Storage), and optionally deletes the snapshot afterward if it's already backed up. It's designed to run as a **Scaleway Serverless Job**, making it ideal for automated, scheduled backups. +Automated serverless job to archive Scaleway Instance snapshots to Object Storage S3. ---- +## Features -## ๐Ÿ“ฆ Features +- **Automated Export**: Finds available snapshots and exports them to an S3 bucket in `.qcow2` format. +- **Cost Optimization**: Deletes the source snapshot after successful export to reduce storage costs. +- **Idempotent**: Skips snapshots that are already archived in the bucket. +- **Serverless Ready**: Designed for [Scaleway Serverless Jobs](https://www.scaleway.com/en/serverless-jobs/). -- Lists all available block storage snapshots in a project. -- Checks if a snapshot with the same name already exists in the target bucket. -- Exports missing snapshots to the bucket in `.qcow2` format. -- Deletes local snapshot after successful export (if not already in bucket). -- Uses environment variables for full configuration. -- Built to run in a container on [Scaleway Serverless Jobs](https://www.scaleway.com/en/serverless-jobs/). +## Configuration ---- - -## โš™๏ธ Environment Variables - -You must set the following environment variables when deploying the job: +Configure the job using environment variables: | Variable | Description | -|--------|-------------| -| `SCW_DEFAULT_ORGANIZATION_ID` | Your Scaleway Organization ID (legacy; prefer project ID). | -| `SCW_DEFAULT_PROJECT_ID` | Your Scaleway Project ID (preferred way to group resources). | -| `SCW_ACCESS_KEY` | API access key (from IAM). | -| `SCW_SECRET_KEY` | API secret key (from IAM). | -| `SCW_ZONE` | Zone where your snapshots are located (e.g., `fr-par-1`). | -| `SCW_BUCKET_NAME` | Name of the S3 bucket to store exported snapshots. | -| `SCW_BUCKET_ENDPOINT` | S3 endpoint (e.g., `s3.fr-par.scw.cloud`). | - -> ๐Ÿ” **Security Tip**: Use IAM API keys with minimal required permissions. - ---- - -## ๐Ÿ› ๏ธ Build & Deploy to Scaleway Serverless Jobs - -### 1. Build the Docker Image - -```bash -docker build -t snapshot-s3-backup . -``` - -### 2. Tag and Push to Scaleway Container Registry (or any registry) - -```bash -# Example using Scaleway CR -docker tag snapshot-s3-backup fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 -docker push fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 -``` - -> Replace `your-registry` with your actual container registry name. - -### 3. Create the Serverless Job - -Use the Scaleway CLI or Console: - -#### Using `scw` CLI: - -```bash -scw job create \ - name=backup-snapshots \ - image=fr-par.scw.cloud/your-registry/snapshot-s3-backup:v1 \ - memory-limit=512Mi \ - cpu-limit=500m \ - environment='{ - "SCW_DEFAULT_PROJECT_ID": "your-project-id", - "SCW_ACCESS_KEY": "your-access-key", - "SCW_SECRET_KEY": "your-secret-key", - "SCW_ZONE": "fr-par-1", - "SCW_BUCKET_NAME": "my-backup-bucket", - "SCW_BUCKET_ENDPOINT": "s3.fr-par.scw.cloud" - }' -``` +|---|---| +| `SCW_DEFAULT_ORGANIZATION_ID` | Organization ID (Legacy). | +| `SCW_DEFAULT_PROJECT_ID` | Project ID (Recommended resource grouping). | +| `SCW_ACCESS_KEY` | IAM Access Key. | +| `SCW_SECRET_KEY` | IAM Secret Key. | +| `SCW_ZONE` | Zone of the snapshots (e.g., `fr-par-1`). | +| `SCW_BUCKET_NAME` | S3 Bucket name for archives. | +| `SCW_BUCKET_ENDPOINT` | S3 Endpoint (e.g., `s3.fr-par.scw.cloud`). | -### 4. (Optional) Schedule the Job +## Usage -Schedule it to run daily using a cron trigger: +### 1. Build ```bash -scw scheduler trigger create-cron \ - job-id=your-job-id \ - schedule="0 2 * * *" \ - name=daily-snapshot-backup -``` - -This runs the job every day at 2 AM. - ---- - -## ๐Ÿ“ Output Format - -Each snapshot is exported as: -``` -.qcow2 -``` - -Example: -``` -my-server-disk-2025-04-05.qcow2 +docker build -t snapshot-archiver . ``` ---- - -## โœ… Example Use Case - -Run nightly to: -1. Export new snapshots to object storage. -2. Clean up old snapshots once safely backed up. -3. Reduce storage costs and improve disaster recovery. - ---- +### 2. Run Locally -## ๐Ÿงช Local Testing (Optional) - -Set environment variables: +Ensure all environment variables are set, then run: ```bash -export SCW_DEFAULT_PROJECT_ID=... -export SCW_ACCESS_KEY=... -export SCW_SECRET_KEY=... -export SCW_ZONE=fr-par-1 -export SCW_BUCKET_NAME=my-backup-bucket -export SCW_BUCKET_ENDPOINT=s3.fr-par.scw.cloud +go run . ``` -Run: - -```bash -go run main.go -``` +### 3. Deploy ---- \ No newline at end of file +Push the image to your container registry and create a Serverless Job definition pointing to it with the required environment variables. \ No newline at end of file diff --git a/jobs/snapshot-s3-archiver/config.go b/jobs/snapshot-s3-archiver/config.go new file mode 100644 index 0000000..960347a --- /dev/null +++ b/jobs/snapshot-s3-archiver/config.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "os" + + "github.com/scaleway/scaleway-sdk-go/scw" +) + +// Environment variable constants +const ( + envOrgID = "SCW_DEFAULT_ORGANIZATION_ID" + envAccessKey = "SCW_ACCESS_KEY" + envSecretKey = "SCW_SECRET_KEY" + envProjectID = "SCW_DEFAULT_PROJECT_ID" + envZone = "SCW_ZONE" + envBucket = "SCW_BUCKET_NAME" + envBucketEndpoint = "SCW_BUCKET_ENDPOINT" +) + +type Config struct { + OrgID string + AccessKey string + SecretKey string + ProjectID string + Zone scw.Zone + BucketName string + BucketEndpoint string +} + +func LoadConfig() (*Config, error) { + // Mandatory variables + vars := map[string]*string{ + envAccessKey: new(string), + envSecretKey: new(string), + envProjectID: new(string), + envZone: new(string), + envBucket: new(string), + envBucketEndpoint: new(string), + } + + // Optional variables + orgID := os.Getenv(envOrgID) + + for envKey, valPtr := range vars { + val := os.Getenv(envKey) + if val == "" { + return nil, fmt.Errorf("missing environment variable %s", envKey) + } + *valPtr = val + } + + return &Config{ + OrgID: orgID, + AccessKey: *vars[envAccessKey], + SecretKey: *vars[envSecretKey], + ProjectID: *vars[envProjectID], + Zone: scw.Zone(*vars[envZone]), + BucketName: *vars[envBucket], + BucketEndpoint: *vars[envBucketEndpoint], + }, nil +} diff --git a/jobs/snapshot-s3-archiver/main.go b/jobs/snapshot-s3-archiver/main.go index d068d72..ab0906e 100644 --- a/jobs/snapshot-s3-archiver/main.go +++ b/jobs/snapshot-s3-archiver/main.go @@ -2,172 +2,124 @@ package main import ( "context" - "fmt" + "log/slog" "os" "slices" - "github.com/scaleway/scaleway-sdk-go/api/instance/v1" - "github.com/scaleway/scaleway-sdk-go/scw" - "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/scaleway/scaleway-sdk-go/scw" ) -// Environment variable constants used to configure the Scaleway API client. -// These must be set in the environment for the application to authenticate and interact with Scaleway services. -const ( - // envOrgID is the Scaleway Organization ID, used for billing and resource ownership (legacy; prefer Project ID). - envOrgID = "SCW_DEFAULT_ORGANIZATION_ID" - - // envAccessKey is the API access key for authenticating requests to Scaleway. - envAccessKey = "SCW_ACCESS_KEY" - - // envSecretKey is the secret key associated with the access key, used for signing requests. - envSecretKey = "SCW_SECRET_KEY" - - // envProjectID is the Scaleway Project ID, which groups resources and is the preferred way to organize infrastructure. - envProjectID = "SCW_DEFAULT_PROJECT_ID" - - // envZone specifies the geographical region/zone where resources will be created (e.g., fr-par-1). - envZone = "SCW_ZONE" - - // envBucket is a custom environment variable for specifying the name of an S3-compatible bucket. - // This is not a standard Scaleway variable and is application-specific. - envBucket = "SCW_BUCKET_NAME" - - envBucketEndpoint = "SCW_BUCKET_ENDPOINT" -) +const snapshotExtension = ".qcow2" func main() { - // Create a Scaleway client with credentials from environment variables. - client, err := scw.NewClient( - // Get your organization ID at https://console.scaleway.com/organization/settings - scw.WithDefaultOrganizationID(os.Getenv(envOrgID)), - - // Get your credentials at https://console.scaleway.com/iam/api-keys - scw.WithAuth(os.Getenv(envAccessKey), os.Getenv(envSecretKey)), + // Configure valid JSON logger + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + slog.SetDefault(logger) - // Set the default project ID to organize resources under a specific project - scw.WithDefaultProjectID(os.Getenv(envProjectID)), + // Load configuration + cfg, err := LoadConfig() + if err != nil { + slog.Error("Failed to load configuration", "error", err) + os.Exit(1) + } - // Set the default zone where resources such as block volumes and snapshots are located - scw.WithDefaultZone(scw.Zone(os.Getenv(envZone))), + // Create Scaleway client using the implementation in config + client, err := scw.NewClient( + scw.WithDefaultOrganizationID(cfg.OrgID), + scw.WithAuth(cfg.AccessKey, cfg.SecretKey), + scw.WithDefaultProjectID(cfg.ProjectID), + scw.WithDefaultZone(cfg.Zone), ) if err != nil { - panic(err) + slog.Error("Failed to create Scaleway client", "error", err) + os.Exit(1) } - fmt.Println("Initializing instance API...") - + slog.Info("Initializing instance API...") instanceAPI := instance.NewAPI(client) - fmt.Println("Reading all snapshots for the project...") - + slog.Info("Reading all snapshots for the project...") snapList, err := instanceAPI.ListSnapshots(&instance.ListSnapshotsRequest{}, scw.WithAllPages()) if err != nil { - panic(err) + slog.Error("Failed to list snapshots", "error", err) + os.Exit(1) } - fmt.Println("Reading all snapshots already in the bucket...") - - filesInBucket, err := listBucketFiles() + slog.Info("Reading all snapshots already in the bucket...") + filesInBucket, err := listBucketFiles(cfg) if err != nil { - panic(err) - } - - const snapshotExtension = ".qcow2" - - for _, snapshot := range snapList.Snapshots { - fmt.Printf("Checking for snapshot %s\n", snapshot.Name) - - if snapshot.State == instance.SnapshotStateAvailable { - // Check if file already exists in bucket - if slices.Contains(filesInBucket, snapshot.Name+snapshotExtension) { - fmt.Printf("File %s already exists in bucket, can delete the snapshot and skip it\n", snapshot.Name+snapshotExtension) - - err = instanceAPI.DeleteSnapshot(&instance.DeleteSnapshotRequest{ - SnapshotID: snapshot.ID, - }) - if err != nil { - panic(err) - } - - continue - } - - fmt.Printf("File %s not present in the bucket, expording it to the bucket...\n", snapshot.Name+".qcow2") - - snap, err := instanceAPI.ExportSnapshot(&instance.ExportSnapshotRequest{ - SnapshotID: snapshot.ID, - Bucket: os.Getenv(envBucket), - Key: snapshot.Name + snapshotExtension, - }) - if err != nil { - fmt.Printf("Failed to export snapshot %s: %v\n", snapshot.Name, err) - - continue - } - - fmt.Printf("Successfully started export of %s to %s/%s\n", snap.Task.ID, os.Getenv(envBucket), snap.Task.Description) - } else { - fmt.Printf("Skipping snapshot %s (ID: %s) - status is %s, not available\n", snapshot.Name, snapshot.ID, snapshot.State.String()) - } - } -} - -// Check for mandatory variables before starting to work. -func init() { - mandatoryVariables := [...]string{ - envOrgID, - envAccessKey, - envSecretKey, - envZone, - envProjectID, - envBucket, - envBucketEndpoint, + slog.Error("Failed to list bucket files", "error", err) + os.Exit(1) } - for idx := range mandatoryVariables { - if os.Getenv(mandatoryVariables[idx]) == "" { - panic("missing environment variable " + mandatoryVariables[idx]) - } - } + processSnapshots(instanceAPI, cfg, snapList.Snapshots, filesInBucket) } -func listBucketFiles() ([]string, error) { - // Retrieve S3-compatible endpoint and credentials from environment - endpoint := os.Getenv(envBucketEndpoint) - accessKeyID := os.Getenv(envAccessKey) - secretAccessKey := os.Getenv(envSecretKey) - - // Create new MinIO client - minioClient, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), +func listBucketFiles(cfg *Config) ([]string, error) { + minioClient, err := minio.New(cfg.BucketEndpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), Secure: true, }) if err != nil { return nil, err } - // Set up context and result slice ctx := context.Background() var files []string - // Channel to signal listing completion - doneCh := make(chan struct{}) - defer close(doneCh) - // List all objects in the bucket - for object := range minioClient.ListObjects(ctx, os.Getenv(envBucket), minio.ListObjectsOptions{ + // Use a closed channel to signal cancellation if needed (not used here but good practice) + // minioClient.ListObjects usually takes a channel for cancellation if needed, but here we pass context + opts := minio.ListObjectsOptions{ Recursive: false, WithMetadata: true, - }) { + } + + for object := range minioClient.ListObjects(ctx, cfg.BucketName, opts) { if object.Err != nil { return nil, object.Err } - files = append(files, object.Key) } return files, nil } + +func processSnapshots(api *instance.API, cfg *Config, snapshots []*instance.Snapshot, filesInBucket []string) { + for _, snapshot := range snapshots { + logger := slog.With("snapshot_id", snapshot.ID, "snapshot_name", snapshot.Name) + + logger.Info("Checking snapshot") + + if snapshot.State != instance.SnapshotStateAvailable { + logger.Info("Skipping snapshot (not available)", "status", snapshot.State.String()) + continue + } + + filename := snapshot.Name + snapshotExtension + + if slices.Contains(filesInBucket, filename) { + logger.Info("File already exists in bucket, deleting local snapshot") + if err := api.DeleteSnapshot(&instance.DeleteSnapshotRequest{SnapshotID: snapshot.ID}); err != nil { + logger.Error("Failed to delete snapshot", "error", err) + } + continue + } + + logger.Info("Exporting snapshot to bucket") + snap, err := api.ExportSnapshot(&instance.ExportSnapshotRequest{ + SnapshotID: snapshot.ID, + Bucket: cfg.BucketName, + Key: filename, + }) + if err != nil { + logger.Error("Failed to export snapshot", "error", err) + continue + } + + logger.Info("Successfully started export", "task_id", snap.Task.ID, "bucket", cfg.BucketName, "description", snap.Task.Description) + } +} From 3e7546741aa1b0af43dacfb67d05bca0a548b8e4 Mon Sep 17 00:00:00 2001 From: Thomas Tacquet Date: Mon, 29 Dec 2025 17:29:03 +0100 Subject: [PATCH 4/4] update readme --- README.md | 1 + jobs/snapshot-s3-archiver/README.md | 71 ++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ded4547..4f09498 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Table of Contents: | **[Instance Snapshot Cleaner](jobs/instances-snapshot-cleaner/README.md)**
Use Serverless Jobs to clean old instances snapshots | Go | [Console] | | **[Registry Tag Cleaner](jobs/registry-version-based-retention/README.md)**
Use Serverless Jobs to keep a desired amount of tags for each image | Go | [Console] | | **[Registry Empty Image Cleaner](jobs/registry-empty-ressource-cleaner/README.md)**
Use Serverless Jobs to clean container registry empty namespaces and images | Go | [Console] | +| **[Instance snapshots to S3](jobs/block-snapshot-s3-archiver/README.md)**
Use Serverless Jobs to move Instances snapshots to S3 Object Storage | Go | [Console] | ### ๐Ÿ’ฌ Messaging and Queueing diff --git a/jobs/snapshot-s3-archiver/README.md b/jobs/snapshot-s3-archiver/README.md index 7d261a8..b09e677 100644 --- a/jobs/snapshot-s3-archiver/README.md +++ b/jobs/snapshot-s3-archiver/README.md @@ -1,6 +1,18 @@ # Scaleway Instance Snapshot Archiver -Automated serverless job to archive Scaleway Instance snapshots to Object Storage S3. +Automated Serverless Job to archive Scaleway Instance snapshots to Object Storage S3. + +## Overview + +This tool automatically finds available snapshots of Scaleway Instances volumes, exports them to a specified S3 bucket in `.qcow2` format, and deletes the original snapshot to optimize storage costs. It's designed to run as a Serverless Job on Scaleway and skips snapshots that have already been archived. + +The main logic is implemented in `main.go`, which: +1. Loads configuration from environment variables. +2. Connects to Scaleway APIs using the Scaleway SDK. +3. Lists all available snapshots in the project. +4. Checks the target S3 bucket for already-archived snapshots. +5. Exports new snapshots to the bucket. +6. Deletes successfully exported snapshots to reduce storage costs. ## Features @@ -9,13 +21,40 @@ Automated serverless job to archive Scaleway Instance snapshots to Object Storag - **Idempotent**: Skips snapshots that are already archived in the bucket. - **Serverless Ready**: Designed for [Scaleway Serverless Jobs](https://www.scaleway.com/en/serverless-jobs/). -## Configuration +## Step 1 : Build and push to Container registry + +Serverless Jobs, like Serverless Containers (which are suited for HTTP applications), works +with containers. So first, use your terminal reach this folder and run the following commands: -Configure the job using environment variables: +```shell +# First command is to login to container registry, you can find it in Scaleway console +docker login rg.fr-par.scw.cloud/snapshot-s3-archiver -u nologin --password-stdin <<< "$SCW_SECRET_KEY" + +# Here we build the image to push +docker buildx build --platform linux/amd64 -t rg.fr-par.scw.cloud/snapshot-s3-archiver/snapshot-s3-archiver:v1 . + +# Push the image online to be used on Serverless Jobs +docker push rg.fr-par.scw.cloud/snapshot-s3-archiver/snapshot-s3-archiver:v1 +``` +> [!TIP] +> As we do not expose a web server and we do not require features such as auto-scaling, Serverless Jobs are perfect for this use case. +To check if everyting is ok, on the Scaleway Console you can verify if your tag is present in Container Registry. +## Step 2: Creating the Job Definition + +On Scaleway Console on the following link you can create a new Job Definition: https://console.scaleway.com/serverless-jobs/jobs/create?region=fr-par + +1. On Container image, select the image you created in the step before. +2. You can set the job definition name name to something clear. +3. Regarding the resources you can keep the default values, this job is fast and do not require specific compute power or memory. +4. To schedule your job for example every night at 2am, you can set the cron to `0 2 * * *`. +5. Important: advanced option, you need to set the following environment variables: + +> [!TIP] +> For sensitive data like `SCW_ACCESS_KEY` and `SCW_SECRET_KEY` we recommend to inject them via Secret Manager, [more info here](https://www.scaleway.com/en/docs/serverless/jobs/how-to/reference-secret-in-job/). | Variable | Description | |---|---| -| `SCW_DEFAULT_ORGANIZATION_ID` | Organization ID (Legacy). | +| `SCW_DEFAULT_ORGANIZATION_ID` | Organization ID . | | `SCW_DEFAULT_PROJECT_ID` | Project ID (Recommended resource grouping). | | `SCW_ACCESS_KEY` | IAM Access Key. | | `SCW_SECRET_KEY` | IAM Secret Key. | @@ -23,22 +62,20 @@ Configure the job using environment variables: | `SCW_BUCKET_NAME` | S3 Bucket name for archives. | | `SCW_BUCKET_ENDPOINT` | S3 Endpoint (e.g., `s3.fr-par.scw.cloud`). | -## Usage +* Then click "create job" -### 1. Build +## Step 3: Run the job -```bash -docker build -t snapshot-archiver . -``` - -### 2. Run Locally +On your created Job Definition, just click the button "Run Job" and within seconds it should be successful. -Ensure all environment variables are set, then run: +## Troubleshooting -```bash -go run . -``` +If your Job Run state goes in error, you can use the "Logs" tab in Scaleway Console to get more informations about the error. -### 3. Deploy +# Additional content -Push the image to your container registry and create a Serverless Job definition pointing to it with the required environment variables. \ No newline at end of file +- [Jobs Documentation](https://www.scaleway.com/en/docs/serverless/jobs/how-to/create-job-from-scaleway-registry/) +- [Other methods to deploy Jobs](https://www.scaleway.com/en/docs/serverless/jobs/reference-content/deploy-job/) +- [Secret key / access key doc](https://www.scaleway.com/en/docs/identity-and-access-management/iam/how-to/create-api-keys/) +- [CRON schedule help](https://www.scaleway.com/en/docs/serverless/jobs/reference-content/cron-schedules/) +- \ No newline at end of file