From 1af313e11822e7b7fe95a15661120066d419f673 Mon Sep 17 00:00:00 2001 From: Frost Wong Date: Fri, 20 Dec 2019 16:44:55 +0800 Subject: [PATCH 1/9] DisableCAS support --- memcache/memcache.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/memcache/memcache.go b/memcache/memcache.go index 545a3e79..edecc8b3 100644 --- a/memcache/memcache.go +++ b/memcache/memcache.go @@ -126,7 +126,7 @@ func New(server ...string) *Client { // NewFromSelector returns a new Client using the provided ServerSelector. func NewFromSelector(ss ServerSelector) *Client { - return &Client{selector: ss} + return &Client{selector: ss, DisableCAS: false} } // Client is a memcache client. @@ -148,6 +148,10 @@ type Client struct { lk sync.Mutex freeconn map[string][]*conn + + // When true, get is sent instead of gets when Client.Get() is called. + // Default false. + DisableCAS bool } // Item is an item to be got or stored in a memcached server. @@ -362,8 +366,13 @@ func (c *Client) withKeyRw(key string, fn func(*bufio.ReadWriter) error) error { } func (c *Client) getFromAddr(addr net.Addr, keys []string, cb func(*Item)) error { + cmd := "gets" + if c.DisableCAS { + cmd = "get" + } + return c.withAddrRw(addr, func(rw *bufio.ReadWriter) error { - if _, err := fmt.Fprintf(rw, "gets %s\r\n", strings.Join(keys, " ")); err != nil { + if _, err := fmt.Fprintf(rw, "%s %s\r\n", cmd, strings.Join(keys, " ")); err != nil { return err } if err := rw.Flush(); err != nil { From 7982c31abcfd29eb2d4c4b383f1ddc9be32c35cc Mon Sep 17 00:00:00 2001 From: Frost Wong Date: Wed, 16 Feb 2022 23:27:27 +0800 Subject: [PATCH 2/9] provide roundrobin server selector for some non-standard memcache server users --- .gitignore | 1 + memcache/roundrobin_selector.go | 95 ++++++++++++++++++++++++++++ memcache/roundrobin_selector_test.go | 23 +++++++ 3 files changed, 119 insertions(+) create mode 100644 memcache/roundrobin_selector.go create mode 100644 memcache/roundrobin_selector_test.go diff --git a/.gitignore b/.gitignore index 02c604d7..4d0a2a9c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ _* *.out *~ +.idea/ \ No newline at end of file diff --git a/memcache/roundrobin_selector.go b/memcache/roundrobin_selector.go new file mode 100644 index 00000000..e726687f --- /dev/null +++ b/memcache/roundrobin_selector.go @@ -0,0 +1,95 @@ +package memcache + +import ( + "net" + "strings" + "sync" +) + +// NewRoundRobin returns a memcache client using the provided server(s) +// with equal weight. If a server is listed multiple times, +// it gets a proportional amount of weight. +func NewRoundRobin(server ...string) *Client { + ss := new(RoundRobinServerList) + err := ss.SetServers(server...) + if err != nil { + return nil + } + + return NewFromRoundRobinSelector(ss) +} + +// NewFromRoundRobinSelector returns a new Client using the provided RoundRobinServerSelector. +func NewFromRoundRobinSelector(ss *RoundRobinServerList) *Client { + return &Client{ + selector: ss, + DisableCAS: false, + } +} + +// RoundRobinServerList is a simple ServerSelector. Its zero value is usable. +type RoundRobinServerList struct { + mu sync.RWMutex + addrs []net.Addr + next int +} + +// SetServers changes a RoundRobinServerList's set of servers at runtime and is +// safe for concurrent use by multiple goroutines. +// +// Each server is given equal weight. A server is given more weight +// if it's listed multiple times. +// +// SetServers returns an error if any of the server names fail to +// resolve. No attempt is made to connect to the server. If any error +// is returned, no changes are made to the RoundRobinServerList. +func (ss *RoundRobinServerList) SetServers(servers ...string) error { + naddr := make([]net.Addr, len(servers)) + for i, server := range servers { + if strings.Contains(server, "/") { + addr, err := net.ResolveUnixAddr("unix", server) + if err != nil { + return err + } + naddr[i] = newStaticAddr(addr) + } else { + tcpaddr, err := net.ResolveTCPAddr("tcp", server) + if err != nil { + return err + } + naddr[i] = newStaticAddr(tcpaddr) + } + } + + ss.mu.Lock() + defer ss.mu.Unlock() + ss.addrs = naddr + return nil +} + +// Each iterates over each server calling the given function +func (ss *RoundRobinServerList) Each(f func(net.Addr) error) error { + ss.mu.RLock() + defer ss.mu.RUnlock() + for _, a := range ss.addrs { + if err := f(a); nil != err { + return err + } + } + return nil +} + +func (ss *RoundRobinServerList) PickServer(key string) (net.Addr, error) { + ss.mu.RLock() + defer ss.mu.RUnlock() + if len(ss.addrs) == 0 { + return nil, ErrNoServers + } + if len(ss.addrs) == 1 { + return ss.addrs[0], nil + } + + ss.next = (ss.next + 1) % len(ss.addrs) + + return ss.addrs[ss.next], nil +} diff --git a/memcache/roundrobin_selector_test.go b/memcache/roundrobin_selector_test.go new file mode 100644 index 00000000..791b8b5e --- /dev/null +++ b/memcache/roundrobin_selector_test.go @@ -0,0 +1,23 @@ +package memcache + +import "testing" + +func BenchmarkPickRoundRobinServer(b *testing.B) { + // at least two to avoid 0 and 1 special cases: + benchPickRoundRobinServer(b, "127.0.0.1:1234", "127.0.0.1:1235") +} + +func BenchmarkPickRoundRobinServer_Single(b *testing.B) { + benchPickRoundRobinServer(b, "127.0.0.1:1234") +} + +func benchPickRoundRobinServer(b *testing.B, servers ...string) { + b.ReportAllocs() + var ss RoundRobinServerList + ss.SetServers(servers...) + for i := 0; i < b.N; i++ { + if _, err := ss.PickServer("some key"); err != nil { + b.Fatal(err) + } + } +} From 3f0e2e4b87363573579d5eba0254ef0b07d05f91 Mon Sep 17 00:00:00 2001 From: qingchun3 Date: Thu, 17 Feb 2022 12:53:47 +0800 Subject: [PATCH 3/9] version v3 --- go.mod | 2 +- v3/go.mod | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 v3/go.mod diff --git a/go.mod b/go.mod index 0d0eed2c..bc1ef8c0 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ -module github.com/bradfitz/gomemcache +module github.com/lovelock/gomemcache go 1.12 diff --git a/v3/go.mod b/v3/go.mod new file mode 100644 index 00000000..bc1ef8c0 --- /dev/null +++ b/v3/go.mod @@ -0,0 +1,3 @@ +module github.com/lovelock/gomemcache + +go 1.12 From 21e132d1530e40800f0a365154ae0c3add6cc86d Mon Sep 17 00:00:00 2001 From: qingchun3 Date: Thu, 17 Feb 2022 13:45:15 +0800 Subject: [PATCH 4/9] version v3.0.2 --- v3/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v3/go.mod b/v3/go.mod index bc1ef8c0..eee1f52c 100644 --- a/v3/go.mod +++ b/v3/go.mod @@ -1,3 +1,3 @@ -module github.com/lovelock/gomemcache +module github.com/lovelock/gomemcache/v3 go 1.12 From 51ed3498943c092f5cff769c77868c62df25b89b Mon Sep 17 00:00:00 2001 From: qingchun3 Date: Thu, 17 Feb 2022 17:37:34 +0800 Subject: [PATCH 5/9] version upgrade --- go.mod | 2 +- v3/go.mod | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 v3/go.mod diff --git a/go.mod b/go.mod index bc1ef8c0..eee1f52c 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ -module github.com/lovelock/gomemcache +module github.com/lovelock/gomemcache/v3 go 1.12 diff --git a/v3/go.mod b/v3/go.mod deleted file mode 100644 index eee1f52c..00000000 --- a/v3/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/lovelock/gomemcache/v3 - -go 1.12 From 7f1081bba09f953b9e58fceea630a6a941d4e2bc Mon Sep 17 00:00:00 2001 From: Frost Wong Date: Thu, 17 Feb 2022 23:08:54 +0800 Subject: [PATCH 6/9] document more --- README.md | 81 +++++++++++++++++++++++++++++++++++--------- memcache/memcache.go | 2 +- 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f987363c..dc0093d6 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,86 @@ ## About -This is a memcache client library for the Go programming language +This is a memcache client library derived from [gomemcache](https://github.com/bradfitz/gomemcache) for the Go programming language (http://golang.org/). +## Why this project? + +The version is bumped to v3 to indicate it's has something incompatible with the vanilla one. + +### `get` vs `gets` + +There are many derivate servers which implement **incomplete** memcache prototol, e.g. only `get` and `set` are implemented. + +The thing is, the original repository of [gomemcache](https://github.com/bradfitz/gomemcache) has something confusing when it comes to [`get` command](https://github.com/bradfitz/gomemcache/blob/fb4bf637b56d66a1925c1bb0780b27dd714ec380/memcache/memcache.go#L361). + +```go +if _, err := fmt.Fprintf(rw, "gets %s\r\n", strings.Join(keys, " ")); err != nil { + return err +} +``` + +It means when you call `get`, the `gets` command is executed and [`casid` is always returned](https://github.com/bradfitz/gomemcache/blob/fb4bf637b56d66a1925c1bb0780b27dd714ec380/memcache/memcache.go#L523). + +```go +dest := []interface{}{&it.Key, &it.Flags, &size, &it.casid} +``` + +I've talked to bradfitz and got a lot of important advises from him. Truly all the things I mentioned above are all because of the **incomplete implementation** and have nothing to do with the client. What I stand for is `get` means `get` and `gets` means `gets`. + +### `RoundRobinServerSelector` + +Vanilla memcache server use crc32 to pick which server to store a key, for servers those distribute keys equally to all nodes the crc32 is not what is wanted. Thanks for bradfitz's brilliant work I can implement a `RoundRobinServerSelector` painlessly. + ## Installing -### Using *go get* +### Using _go get_ - $ go get github.com/bradfitz/gomemcache/memcache +`$ go get -u github.com/lovelock/gomemcache/v3/memcache` -After this command *gomemcache* is ready to use. Its source will be in: +After this command _gomemcache_ is ready to use. Its source will be in: - $GOPATH/src/github.com/bradfitz/gomemcache/memcache +`$GOPATH/src/github.com/lovelock/gomemcache/memcache` ## Example - import ( - "github.com/bradfitz/gomemcache/memcache" - ) +### For vanilla memcached server + +```go +import ( + "github.com/lovelock/gomemcache/v3/memcache" +) + +func main() { + mc := memcache.New("10.0.0.1:11211", "10.0.0.2:11211", "10.0.0.3:11212") + mc.Set(&memcache.Item{Key: "foo", Value: []byte("my value")}) + + it, err := mc.Get("foo") + ... +} +``` - func main() { - mc := memcache.New("10.0.0.1:11211", "10.0.0.2:11211", "10.0.0.3:11212") - mc.Set(&memcache.Item{Key: "foo", Value: []byte("my value")}) +### For other derivatives - it, err := mc.Get("foo") - ... +```go +import ( + "github.com/lovelock/gomemcache/v3/memcache" +) + +func main() { + mc := memcache.NewRoundRobin("10.0.0.1:11211", "10.0.0.2:11211", "10.0.0.3:11212") + mc.DisableCAS = true // don't want get casid + mc.Set(&memcache.Item{Key: "foo", Value: []byte("my value")}) + + it, err := mc.Get("foo") + ... } +``` + ## Full docs, see: -See https://godoc.org/github.com/bradfitz/gomemcache/memcache +See https://godoc.org/github.com/lovelock/gomemcache/v3/memcache Or run: - $ godoc github.com/bradfitz/gomemcache/memcache - +`$ godoc github.com/lovelock/gomemcache/v3/memcache` diff --git a/memcache/memcache.go b/memcache/memcache.go index 44246fba..c165f8e2 100644 --- a/memcache/memcache.go +++ b/memcache/memcache.go @@ -126,7 +126,7 @@ func New(server ...string) *Client { // NewFromSelector returns a new Client using the provided ServerSelector. func NewFromSelector(ss ServerSelector) *Client { - return &Client{selector: ss, DisableCAS: false} + return &Client{selector: ss} } // Client is a memcache client. From 7cb416ea0631eda111accb6330994c4a10aebb93 Mon Sep 17 00:00:00 2001 From: Frost Wong Date: Thu, 17 Feb 2022 23:14:53 +0800 Subject: [PATCH 7/9] fix typo in doc --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dc0093d6..f045eb6e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This is a memcache client library derived from [gomemcache](https://github.com/b ## Why this project? -The version is bumped to v3 to indicate it's has something incompatible with the vanilla one. +The version is bumped to v3 to indicate that it has something incompatible with the vanilla one. ### `get` vs `gets` @@ -73,8 +73,7 @@ func main() { it, err := mc.Get("foo") ... - } - +} ``` ## Full docs, see: From bdca2e8e46f9b77bc0b95177872ced39d2e6144b Mon Sep 17 00:00:00 2001 From: Frost Wong Date: Thu, 17 Feb 2022 23:40:23 +0800 Subject: [PATCH 8/9] modify vulnerable statements according to SA4011 --- memcache/memcache.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/memcache/memcache.go b/memcache/memcache.go index c165f8e2..e2900322 100644 --- a/memcache/memcache.go +++ b/memcache/memcache.go @@ -442,7 +442,7 @@ func (c *Client) touchFromAddr(addr net.Addr, keys []string, expiration int32) e } switch { case bytes.Equal(line, resultTouched): - break + continue case bytes.Equal(line, resultNotFound): return ErrCacheMiss default: @@ -486,7 +486,7 @@ func (c *Client) GetMulti(keys []string) (map[string]*Item, error) { } var err error - for _ = range keyMap { + for range keyMap { if ge := <-ch; ge != nil { err = ge } From 4dfff16f82c91fdc39356d448e716cb13d973091 Mon Sep 17 00:00:00 2001 From: Frost Wong Date: Sat, 26 Feb 2022 00:15:16 +0800 Subject: [PATCH 9/9] use write lock instead of read lock --- memcache/roundrobin_selector.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/memcache/roundrobin_selector.go b/memcache/roundrobin_selector.go index e726687f..a32cc3d8 100644 --- a/memcache/roundrobin_selector.go +++ b/memcache/roundrobin_selector.go @@ -29,7 +29,7 @@ func NewFromRoundRobinSelector(ss *RoundRobinServerList) *Client { // RoundRobinServerList is a simple ServerSelector. Its zero value is usable. type RoundRobinServerList struct { - mu sync.RWMutex + mu sync.Mutex addrs []net.Addr next int } @@ -69,8 +69,8 @@ func (ss *RoundRobinServerList) SetServers(servers ...string) error { // Each iterates over each server calling the given function func (ss *RoundRobinServerList) Each(f func(net.Addr) error) error { - ss.mu.RLock() - defer ss.mu.RUnlock() + ss.mu.Lock() + defer ss.mu.Unlock() for _, a := range ss.addrs { if err := f(a); nil != err { return err @@ -80,8 +80,8 @@ func (ss *RoundRobinServerList) Each(f func(net.Addr) error) error { } func (ss *RoundRobinServerList) PickServer(key string) (net.Addr, error) { - ss.mu.RLock() - defer ss.mu.RUnlock() + ss.mu.Lock() + defer ss.mu.Unlock() if len(ss.addrs) == 0 { return nil, ErrNoServers }