Feature: HTTP exponential backoff

.github/workflows/build.yml

@@ -7,25 +7,25 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Lint
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@v9
         with:
-          version: v1.64
+          version: v2.7
           only-new-issues: true
 
   test:
     runs-on: ubuntu-latest
     env:
       GORACE: history_size=4
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
-          go-version: '1.23'
+          go-version: '1.25'
 
       - name: Test
         run: |

.golangci.yml

@@ -1,25 +1,35 @@
+version: "2"
 run:
-  timeout: 240s
   tests: true
-
 linters:
-  disable-all: true
+  default: none
   enable:
+    - errcheck
+    - gocritic
     - govet
+    - ineffassign
+    - lll
+    - misspell
+    - nakedret
     - revive
-    - unused
-    - errcheck
     - staticcheck
-    - ineffassign
     - unconvert
+    - unused
+  settings:
+    revive:
+      rules:
+        - name: var-naming
+          disabled: true
+    staticcheck:
+      # Disable SA1019 (using deprecated code) as the existing use of CFB is
+      # required for backwards compatibility.
+      checks: ["all", "-SA1019"]
+    lll:
+      line-length: 240
+  exclusions:
+    generated: lax
+formatters:
+  enable:
     - goimports
-    - misspell
-    - lll
-    - nakedret
-    - gocritic
-
-linters-settings:
-  lll:
-    line-length: 240 # Exceptional case its usually 120
-issues:
-  exclude-use-default: false
+  exclusions:
+    generated: lax

go.mod

@@ -1,18 +1,18 @@
 module github.com/teamwork/utils/v2
 
-go 1.20
+go 1.25
 
 require (
-	github.com/mattn/goveralls v0.0.11
+	github.com/mattn/goveralls v0.0.12
 	github.com/pkg/errors v0.9.1
 	github.com/teamwork/test v0.0.0-20200108114543-02621bae84ad
-	golang.org/x/tools v0.8.0
+	golang.org/x/tools v0.39.0
 )
 
 require (
 	github.com/Strum355/go-difflib v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/teamwork/utils v1.0.0 // indirect
-	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
+	golang.org/x/mod v0.30.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
 )

go.sum

@@ -3,8 +3,10 @@ github.com/Strum355/go-difflib v1.1.0/go.mod h1:r1cVg1JkGsTWkaR7At56v7hfuMgiUL8m
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/mattn/goveralls v0.0.11 h1:eJXea6R6IFlL1QMKNMzDvvHv/hwGrnvyig4N+0+XiMM=
-github.com/mattn/goveralls v0.0.11/go.mod h1:gU8SyhNswsJKchEV93xRQxX6X3Ei4PJdQk/6ZHvrvRk=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/mattn/goveralls v0.0.12 h1:PEEeF0k1SsTjOBQ8FOmrOAoCu4ytuMaWCnWe94zxbCg=
+github.com/mattn/goveralls v0.0.12/go.mod h1:44ImGEUfmqH8bBtaMrYKsM65LXfNLWmwaxFGjZwgMSQ=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -13,10 +15,45 @@ github.com/teamwork/test v0.0.0-20200108114543-02621bae84ad h1:25sEr0awm0ZPancg5
 github.com/teamwork/test v0.0.0-20200108114543-02621bae84ad/go.mod h1:TIbx7tx6WHBjQeLRM4eWQZBL7kmBZ7/KI4x4v7Y5YmA=
 github.com/teamwork/utils v1.0.0 h1:30WqhSbZ9nFhaJSx9HH+yFLiQvL64nqAOyyl5IxoYlY=
 github.com/teamwork/utils v1.0.0/go.mod h1:3Fn0qxFeRNpvsg/9T1+btOOOKkd1qG2nPYKKcOmNpcs=
-golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
+golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
+golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

goutil/goutil_test.go

@@ -40,10 +40,10 @@ func TestExpand(t *testing.T) {
 			[]string{"net", "net/http", "net/http/cgi", "net/http/cookiejar",
 				"net/http/fcgi", "net/http/httptest", "net/http/httptrace",
 				"net/http/httputil", "net/http/internal", "net/http/internal/ascii",
-				"net/http/internal/testcert", "net/http/pprof",
-				"net/internal/cgotest", "net/internal/socktest", "net/mail",
-				"net/netip", "net/rpc", "net/rpc/jsonrpc", "net/smtp", "net/textproto",
-				"net/url",
+				"net/http/internal/httpcommon", "net/http/internal/testcert",
+				"net/http/pprof", "net/internal/cgotest", "net/internal/socktest",
+				"net/mail", "net/netip", "net/rpc", "net/rpc/jsonrpc", "net/smtp",
+				"net/textproto", "net/url",
 			},
 			"",
 		},

httputilx/httputilx.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"net/http/httputil"
 	"os"
@@ -145,3 +146,129 @@ func Save(url string, dir string, filename string) (string, error) {
 
 	return path, nil
 }
+
+// ExponentialBackoffOptions contains options for the exponential backoff retry
+// mechanism.
+type ExponentialBackoffOptions struct {
+	client            *http.Client
+	maxRetries        int
+	initialBackoff    time.Duration
+	maxBackoff        time.Duration
+	backoffMultiplier float64
+	shouldRetry       func(resp *http.Response, err error) bool
+	logger            *slog.Logger
+}
+
+// ExponentialBackoffOption is a function that configures
+// ExponentialBackoffOptions.
+type ExponentialBackoffOption func(*ExponentialBackoffOptions)
+
+// ExponentialBackoffWithClient sets the HTTP client to be used when sending the
+// API requests. By default, http.DefaultClient is used.
+func ExponentialBackoffWithClient(client *http.Client) ExponentialBackoffOption {
+	return func(o *ExponentialBackoffOptions) {
+		o.client = client
+	}
+}
+
+// ExponentialBackoffWithConfig sets the configuration for the exponential
+// backoff retry mechanism. By default, it will retry up to 3 times, starting
+// with a 100ms backoff, doubling each time up to a maximum of 5s.
+func ExponentialBackoffWithConfig(
+	maxRetries int,
+	initialBackoff, maxBackoff time.Duration,
+	backoffMultiplier float64,
+) ExponentialBackoffOption {
+	return func(o *ExponentialBackoffOptions) {
+		o.maxRetries = maxRetries
+		o.initialBackoff = initialBackoff
+		o.maxBackoff = maxBackoff
+		o.backoffMultiplier = backoffMultiplier
+	}
+}
+
+// ExponentialBackoffWithShouldRetry sets the function to determine whether a
+// request should be retried based on the response and error. By default, it
+// retries on any error, as well as on HTTP 5xx and 429 status codes.
+func ExponentialBackoffWithShouldRetry(
+	shouldRetry func(resp *http.Response, err error) bool,
+) ExponentialBackoffOption {
+	return func(o *ExponentialBackoffOptions) {
+		o.shouldRetry = shouldRetry
+	}
+}
+
+// ExponentialBackoffWithLogger sets the logger to be used for logging retry
+// attempts. By default, a no-op logger is used.
+func ExponentialBackoffWithLogger(logger *slog.Logger) ExponentialBackoffOption {
+	return func(o *ExponentialBackoffOptions) {
+		o.logger = logger
+	}
+}
+
+// DoExponentialBackoff will send an API request using exponential backoff until
+// it either succeeds or the maximum number of retries is reached.
+func DoExponentialBackoff(req *http.Request, options ...ExponentialBackoffOption) (*http.Response, error) {
+	o := ExponentialBackoffOptions{
+		client:            http.DefaultClient,
+		maxRetries:        3,
+		initialBackoff:    100 * time.Millisecond,
+		maxBackoff:        5 * time.Second,
+		backoffMultiplier: 2.0,
+		shouldRetry: func(resp *http.Response, err error) bool {
+			if err != nil {
+				return true
+			}
+			if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 {
+				return true
+			}
+			return false
+		},
+		logger: slog.New(slog.DiscardHandler),
+	}
+	for _, option := range options {
+		option(&o)
+	}
+
+	backoff := o.initialBackoff
+
+	for attempt := 0; attempt <= o.maxRetries; attempt++ {
+		reqClone := req.Clone(req.Context())
+		if req.Body != nil {
+			if seeker, ok := req.Body.(interface {
+				Seek(int64, int) (int64, error)
+			}); ok {
+				_, _ = seeker.Seek(0, 0)
+			}
+			reqClone.Body = req.Body
+		}
+
+		resp, err := o.client.Do(reqClone)
+		if !o.shouldRetry(resp, err) || attempt >= o.maxRetries {
+			return resp, nil
+		}
+
+		logArgs := []any{
+			slog.Int("attempt", attempt+1),
+			slog.Duration("backoff", backoff),
+		}
+		if err != nil {
+			logArgs = append(logArgs, slog.String("error", err.Error()))
+		}
+		if resp != nil {
+			if err := resp.Body.Close(); err != nil {
+				o.logger.Error("failed to close response body",
+					slog.Int("attempt", attempt+1),
+					slog.String("error", err.Error()),
+				)
+			}
+			logArgs = append(logArgs, slog.Int("status_code", resp.StatusCode))
+		}
+
+		o.logger.Debug("request failed", logArgs...)
+		time.Sleep(backoff)
+		backoff = min(time.Duration(float64(backoff)*o.backoffMultiplier), o.maxBackoff)
+	}
+
+	return nil, fmt.Errorf("request failed after %d attempts", o.maxRetries+1)
+}