pax_global_header00006660000000000000000000000064145145024260014515gustar00rootroot0000000000000052 comment=96a8e4929843cf552851775857f81b0aae41b410 mna-redisc-96a8e49/000077500000000000000000000000001451450242600141125ustar00rootroot00000000000000mna-redisc-96a8e49/.github/000077500000000000000000000000001451450242600154525ustar00rootroot00000000000000mna-redisc-96a8e49/.github/FUNDING.yml000066400000000000000000000000731451450242600172670ustar00rootroot00000000000000github: [mna] custom: ["https://www.buymeacoffee.com/mna"] mna-redisc-96a8e49/.github/workflows/000077500000000000000000000000001451450242600175075ustar00rootroot00000000000000mna-redisc-96a8e49/.github/workflows/test.yml000066400000000000000000000017171451450242600212170ustar00rootroot00000000000000name: test on: [push, pull_request] env: GOPROXY: https://proxy.golang.org,direct jobs: test: strategy: matrix: go-version: [1.20.x, 1.21.x] redis-version: [5.x, 6.x] os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - name: Install Go uses: actions/setup-go@v4 with: go-version: ${{ matrix.go-version }} - name: Install Redis uses: shogo82148/actions-setup-redis@v1 with: redis-version: ${{ matrix.redis-version }} auto-start: "false" - name: Checkout code uses: actions/checkout@v4 - name: Test run: go test ./... -v -cover - name: Data Race run: go test ./... -race golangci: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Lint uses: golangci/golangci-lint-action@v3 with: version: v1.54 mna-redisc-96a8e49/.gitignore000066400000000000000000000000751451450242600161040ustar00rootroot00000000000000# cover/profile output files *.out # test binaries *.test mna-redisc-96a8e49/.golangci.yml000066400000000000000000000021361451450242600165000ustar00rootroot00000000000000linters: disable-all: true enable: - errcheck - gci - gochecknoinits - gofmt - gosec - gosimple - govet - importas - ineffassign - misspell - nakedret - prealloc - revive - staticcheck - typecheck - unconvert - unparam - unused linters-settings: revive: ignoreGeneratedHeader: false severity: "warning" confidence: 0.8 errorCode: 0 warningCode: 0 rules: - name: blank-imports - name: context-as-argument - name: context-keys-type - name: dot-imports - name: error-return - name: error-strings - name: error-naming - name: exported - name: increment-decrement - name: var-naming - name: var-declaration - name: package-comments - name: range - name: receiver-naming - name: time-naming - name: unexported-return - name: indent-error-flow - name: errorf - name: empty-block - name: superfluous-else - name: unused-parameter - name: unreachable-code - name: redefines-builtin-id mna-redisc-96a8e49/LICENSE000066400000000000000000000026771451450242600151330ustar00rootroot00000000000000Copyright (c) 2016, Martin Angers All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. mna-redisc-96a8e49/README.md000066400000000000000000000146701451450242600154010ustar00rootroot00000000000000[![Go Reference](https://pkg.go.dev/badge/github.com/mna/redisc.svg)](https://pkg.go.dev/github.com/mna/redisc) [![Build Status](https://github.com/mna/redisc/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/mna/redisc/actions) # redisc Package redisc implements a redis cluster client built on top of the [redigo package][redigo]. See the [documentation][godoc] for details. ## Installation $ go get [-u] [-t] github.com/mna/redisc ## Releases * **v1.4.0** : Improve the reliability of the refresh of cluster mapping for edge cases where no nodes are known anymore (thanks to [@ljfuyuan][ljfuyuan]). * **v1.3.2** : Export the `HashSlots` constant to make it nicer to write the `Cluster.LayoutRefresh` function signature. * **v1.3.1** : Fix closing/releasing of connections used in `Cluster.EachNode`. * **v1.3.0** : Add `Cluster.EachNode` to call a function with a connection for each known node in the cluster (e.g. to run diagnostics commands on each node or to collect all keys in a cluster); add optional Cluster function field `BgError` to receive notification of errors happening in background topology refreshes and on closing of `RetryConn` after following a redirection to a new connection; add optional Cluster function field `LayoutRefresh` to receive the old and new cluster slot mappings to server address(es); prevent unnecessary cluster layout refreshes when the internal mapping is the same as the redirection error; better handling of closed Cluster; move CI to Github Actions; drop support for old Go versions (currently tested on 1.15+); enable more static analysis/linters; refactor tests to create less separate clusters and run faster. * **v1.2.0** : Use Go modules, fix a failing test due to changed error message on Redis 6. * **v1.1.7** : Do not bind to a random node if `Do` is called without a command and the connection is not already bound (thanks to [@tysonmote][tysonmote]). * **v1.1.6** : Append the actual error messages when a refresh returns "all nodes failed" error. * **v1.1.5** : Add `Cluster.PoolWaitTime` to configure the time to wait on a connection from a pool with `MaxActive` > 0 and `Wait` set to true (thanks to [@iwanbk][iwanbk]). * **v1.1.4** : Add `Conn.DoWithTimeout` and `Conn.ReceiveWithTimeout` to match redigo's `ConnWithTimeout` interface (thanks to [@letsfire][letsfire]). * **v1.1.3** : Fix handling of `ASK` replies in `RetryConn`. * **v1.1.2** : Remove mention that `StartupNodes` in `Cluster` struct needs to be master nodes (it can be replicas). Add supporting test. * **v1.1.1** : Fix CI tests. * **v1.1.0** : This release builds with the `github.com/gomodule/redigo` package (the new import path of `redigo`, which also has a breaking change in its `v2.0.0`, the `PMessage` type has been removed and consolidated into `Message`). * **v1.0.0** : This release builds with the `github.com/garyburd/redigo` package, which - according to its [readme][oldredigo] - will not be maintained anymore, having moved to [`github.com/gomodule/redigo`][redigo] for future development. As such, `redisc` will not be updated with the old redigo package, this version was created only to avoid causing issues to users of redisc. ## Documentation The [code documentation][godoc] is the canonical source for documentation. The design goal of redisc is to be as compatible as possible with the [redigo][] package. As such, the `Cluster` type can be used as a drop-in replacement to a `redis.Pool` when moving from a standalone Redis to a Redis Cluster setup, and the connections returned by the cluster implement redigo's `redis.Conn` interface. The package offers additional features specific to dealing with a cluster that may be needed for more advanced scenarios. The main features are: * Drop-in replacement for `redis.Pool` (the `Cluster` type implements the same `Get` and `Close` method signatures). * Connections are `redis.Conn` interfaces and use the `redigo` package to execute commands, `redisc` only handles the cluster part. * Support for all cluster-supported commands including scripting, transactions and pub-sub (within the limitations imposed by Redis Cluster). * Support for READONLY/READWRITE commands to allow reading data from replicas. * Client-side smart routing, automatically keeps track of which node holds which key slots. * Automatic retry of MOVED, ASK and TRYAGAIN errors when desired, via `RetryConn`. * Manual handling of redirections and retries when desired, via `IsTryAgain` and `ParseRedir`. * Automatic detection of the node to call based on the command's first parameter (assumed to be the key). * Explicit selection of the node to call via `BindConn` when needed. * Support for optimal batch calls via `SplitBySlot`. Note that to make efficient use of Redis Cluster, some upfront work is usually required. A good understanding of Redis Cluster is highly recommended and the official Redis website has [good documentation that covers this](https://redis.io/topics/cluster-spec). In particular, [Migrating to Redis Cluster](https://redis.io/topics/cluster-tutorial#migrating-to-redis-cluster) will help understand how straightforward (or not) the migration may be for your specific case. ## Alternatives * [redis-go-cluster][rgc]. * [radix v1][radix1] provides a cluster package. * [radix v2][radix2] provides a cluster package. ## Support There are a number of ways you can support the project: * Use it, star it, build something with it, spread the word! * Raise issues to improve the project (note: doc typos and clarifications are issues too!) - Please search existing issues before opening a new one - it may have already been adressed. * Pull requests: please discuss new code in an issue first, unless the fix is really trivial. - Make sure new code is tested. - Be mindful of existing code - PRs that break existing code have a high probability of being declined, unless it fixes a serious issue. * Sponsor the developer - See the Github Sponsor button at the top of the repo on github ## License The [BSD 3-Clause license][bsd]. [bsd]: http://opensource.org/licenses/BSD-3-Clause [godoc]: https://pkg.go.dev/github.com/mna/redisc [redigo]: https://github.com/gomodule/redigo [oldredigo]: https://github.com/garyburd/redigo [rgc]: https://github.com/chasex/redis-go-cluster [radix1]: https://github.com/fzzy/radix [radix2]: https://github.com/mediocregopher/radix.v2 [letsfire]: https://github.com/letsfire [iwanbk]: https://github.com/iwanbk [tysonmote]: https://github.com/tysonmote [ljfuyuan]: https://github.com/ljfuyuan mna-redisc-96a8e49/ccheck/000077500000000000000000000000001451450242600153325ustar00rootroot00000000000000mna-redisc-96a8e49/ccheck/README.md000066400000000000000000000017551451450242600166210ustar00rootroot00000000000000# Consistency Checker This folder implements a consistency checker as described in the [redis cluster tutorial documentation][tut] and implemented in the [reference redis cluster client ruby repository][rubycluster]. The `redis-trib.rb` and `create-cluster` scripts are from the [redis repository][redis] and copied here only for convenience, with the redis copyright added to the top of the files. The `create-cluster` script has been adjusted to start the $PATH-installed redis-server and redis-cli binaries, and the `redis-trib.rb` script in the current directory. All scripts should be executed in this directory. To run the cluster: 1. create-cluster start 2. create-cluster create (type yes when prompted) 3. execute the ccheck go client program 4. play with the cluster, trigger failovers, etc. 5. create-cluster stop 6. create-cluster clean [tut]: http://redis.io/topics/cluster-tutorial [rubycluster]: https://github.com/antirez/redis-rb-cluster [redis]: https://github.com/antirez/redis mna-redisc-96a8e49/ccheck/ccheck.go000066400000000000000000000113051451450242600171010ustar00rootroot00000000000000// Command ccheck implements the consistency checker redis cluster client // as described in http://redis.io/topics/cluster-tutorial. It is used // to test the redisc package with real cluster failover and resharding // situations. package main import ( "flag" "fmt" "log" "math/rand" "net" "strconv" "sync" "time" "github.com/gomodule/redigo/redis" "github.com/mna/redisc" ) var ( addrFlag = flag.String("addr", "localhost:7000", "Redis server `address`.") connTimeoutFlag = flag.Duration("c", time.Second, "Connection `timeout`.") delayFlag = flag.Duration("d", 0, "Delay `duration` between INCR calls.") idleTimeoutFlag = flag.Duration("i", 30*time.Second, "Pooled connection idle `timeout`.") readTimeoutFlag = flag.Duration("r", 100*time.Millisecond, "Read `timeout`.") writeTimeoutFlag = flag.Duration("w", 100*time.Millisecond, "Write `timeout`.") refreshFlag = flag.Bool("f", false, "Perform a cluster refresh before starting.") retryConnFlag = flag.Bool("R", false, "Use a single retry connection.") disablePoolFlag = flag.Bool("P", false, "Disable connection pooling.") maxIdleFlag = flag.Int("max-idle", 10, "Maximum idle `connections` per pool.") maxActiveFlag = flag.Int("max-active", 100, "Maximum active `connections` per pool.") ) const ( workingSet = 1000 keySpace = 10000 ) var ( mu sync.Mutex writes, reads int failedWrites, failedReads int lostWrites, noAckWrites int ) func main() { flag.Parse() rand.Seed(time.Now().UnixNano()) cluster := &redisc.Cluster{ StartupNodes: []string{*addrFlag}, DialOptions: []redis.DialOption{ redis.DialConnectTimeout(*connTimeoutFlag), redis.DialReadTimeout(*readTimeoutFlag), redis.DialWriteTimeout(*writeTimeoutFlag), }, CreatePool: createPool, } defer cluster.Close() if *disablePoolFlag { cluster.CreatePool = nil } if *refreshFlag { if err := cluster.Refresh(); err != nil { log.Fatalf("failed to refresh cluster: %v", err) } } errCh := make(chan error, 1) go printStats() go printErr(errCh) runChecks(cluster, errCh, *delayFlag, *retryConnFlag) } func getRetryConn(cluster *redisc.Cluster) redis.Conn { c := cluster.Get() c, _ = redisc.RetryConn(c, 4, 100*time.Millisecond) if err := c.Err(); err != nil { log.Fatalf("failed to get a connection: %v", err) } return c } func runChecks(cluster *redisc.Cluster, errCh chan<- error, delay time.Duration, useRetryConn bool) { var c redis.Conn cache := make(map[string]int, workingSet) for { var r, w, fr, fw, lw, naw int if useRetryConn && c == nil { c = getRetryConn(cluster) } else { c = cluster.Get() } key := genKey() // read only if we know what that key should be exp, ok := cache[key] if ok { v, err := redis.Int(c.Do("GET", key)) if err != nil { if isNetOpError(err) { c.Close() c = nil continue } select { case errCh <- fmt.Errorf("read from slot %d failed: %v", redisc.Slot(key), err): default: } fr = 1 } else { r = 1 if exp > v { lw = exp - v } else if exp < v { naw = v - exp } } } // write v, err := redis.Int(c.Do("INCR", key)) if err != nil { if isNetOpError(err) { c.Close() c = nil continue } select { case errCh <- fmt.Errorf("write to slot %d failed: %v", redisc.Slot(key), err): default: } fw = 1 } else { w = 1 cache[key] = v } if !useRetryConn { c.Close() } updateStats(w, r, fw, fr, lw, naw) time.Sleep(delay) } } func isNetOpError(err error) bool { _, ok := err.(*net.OpError) return ok } func updateStats(deltas ...int) { mu.Lock() writes += deltas[0] reads += deltas[1] failedWrites += deltas[2] failedReads += deltas[3] lostWrites += deltas[4] noAckWrites += deltas[5] mu.Unlock() } func printErr(errCh <-chan error) { for err := range errCh { fmt.Println(err) time.Sleep(time.Second) } } // each second, print stats func printStats() { for range time.Tick(time.Second) { mu.Lock() w, r := writes, reads fw, fr := failedWrites, failedReads lw, naw := lostWrites, noAckWrites mu.Unlock() fmt.Printf("%d R (%d err) | %d W (%d err) | %d lost | %d noack\n", r, fr, w, fw, lw, naw) } } func genKey() string { ks := workingSet //nolint:gosec if rand.Float64() > 0.5 { ks = keySpace } return "key_" + strconv.Itoa(rand.Intn(ks)) //nolint:gosec } func createPool(addr string, opts ...redis.DialOption) (*redis.Pool, error) { return &redis.Pool{ Dial: func() (redis.Conn, error) { return redis.Dial("tcp", addr, opts...) }, TestOnBorrow: func(c redis.Conn, t time.Time) error { _, err := c.Do("PING") return err }, MaxActive: *maxActiveFlag, MaxIdle: *maxIdleFlag, IdleTimeout: *idleTimeoutFlag, }, nil } mna-redisc-96a8e49/ccheck/create-cluster000077500000000000000000000071711451450242600202100ustar00rootroot00000000000000#!/bin/bash # Copyright (c) 2006-2015, Salvatore Sanfilippo # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided with the distribution. # * Neither the name of Redis nor the names of its contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER # IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Settings PORT=30000 TIMEOUT=2000 NODES=6 REPLICAS=1 # You may want to put the above config parameters into config.sh in order to # override the defaults without modifying this script. if [ -a config.sh ] then source "config.sh" fi # Computed vars ENDPORT=$((PORT+NODES)) if [ "$1" == "start" ] then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" redis-server --port $PORT --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes done exit 0 fi if [ "$1" == "create" ] then HOSTS="" while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) HOSTS="$HOSTS 127.0.0.1:$PORT" done ./redis-trib.rb create --replicas $REPLICAS $HOSTS exit 0 fi if [ "$1" == "stop" ] then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Stopping $PORT" redis-cli -p $PORT shutdown nosave done exit 0 fi if [ "$1" == "watch" ] then PORT=$((PORT+1)) while [ 1 ]; do clear date redis-cli -p $PORT cluster nodes | head -30 sleep 1 done exit 0 fi if [ "$1" == "tail" ] then INSTANCE=$2 PORT=$((PORT+INSTANCE)) tail -f ${PORT}.log exit 0 fi if [ "$1" == "call" ] then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9 done exit 0 fi if [ "$1" == "clean" ] then rm -rf *.log rm -rf appendonly*.aof rm -rf dump*.rdb rm -rf nodes*.conf exit 0 fi echo "Usage: $0 [start|create|stop|watch|tail|clean]" echo "start -- Launch Redis Cluster instances." echo "create -- Create a cluster using redis-trib create." echo "stop -- Stop Redis Cluster instances." echo "watch -- Show CLUSTER NODES output (first 30 lines) of first node." echo "tail -- Run tail -f of instance at base port + ID." echo "clean -- Remove all instances data, logs, configs." mna-redisc-96a8e49/ccheck/redis-trib.rb000077500000000000000000001712101451450242600177300ustar00rootroot00000000000000#!/usr/bin/env ruby # Copyright (c) 2006-2015, Salvatore Sanfilippo # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided with the distribution. # * Neither the name of Redis nor the names of its contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER # IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # TODO (temporary here, we'll move this into the Github issues once # redis-trib initial implementation is completed). # # - Make sure that if the rehashing fails in the middle redis-trib will try # to recover. # - When redis-trib performs a cluster check, if it detects a slot move in # progress it should prompt the user to continue the move from where it # stopped. # - Gracefully handle Ctrl+C in move_slot to prompt the user if really stop # while rehashing, and performing the best cleanup possible if the user # forces the quit. # - When doing "fix" set a global Fix to true, and prompt the user to # fix the problem if automatically fixable every time there is something # to fix. For instance: # 1) If there is a node that pretend to receive a slot, or to migrate a # slot, but has no entries in that slot, fix it. # 2) If there is a node having keys in slots that are not owned by it # fix this condition moving the entries in the same node. # 3) Perform more possibly slow tests about the state of the cluster. # 4) When aborted slot migration is detected, fix it. require 'rubygems' require 'redis' ClusterHashSlots = 16384 MigrateDefaultTimeout = 60000 MigrateDefaultPipeline = 10 RebalanceDefaultThreshold = 2 $verbose = false def xputs(s) case s[0..2] when ">>>" color="29;1" when "[ER" color="31;1" when "[WA" color="31;1" when "[OK" color="32" when "[FA","***" color="33" else color=nil end color = nil if ENV['TERM'] != "xterm" print "\033[#{color}m" if color print s print "\033[0m" if color print "\n" end class ClusterNode def initialize(addr) s = addr.split("@")[0].split(":") if s.length < 2 puts "Invalid IP or Port (given as #{addr}) - use IP:Port format" exit 1 end port = s.pop # removes port from split array ip = s.join(":") # if s.length > 1 here, it's IPv6, so restore address @r = nil @info = {} @info[:host] = ip @info[:port] = port @info[:slots] = {} @info[:migrating] = {} @info[:importing] = {} @info[:replicate] = false @dirty = false # True if we need to flush slots info into node. @friends = [] end def friends @friends end def slots @info[:slots] end def has_flag?(flag) @info[:flags].index(flag) end def to_s "#{@info[:host]}:#{@info[:port]}" end def connect(o={}) return if @r print "Connecting to node #{self}: " if $verbose STDOUT.flush begin @r = Redis.new(:host => @info[:host], :port => @info[:port], :timeout => 60) @r.ping rescue xputs "[ERR] Sorry, can't connect to node #{self}" exit 1 if o[:abort] @r = nil end xputs "OK" if $verbose end def assert_cluster info = @r.info if !info["cluster_enabled"] || info["cluster_enabled"].to_i == 0 xputs "[ERR] Node #{self} is not configured as a cluster node." exit 1 end end def assert_empty if !(@r.cluster("info").split("\r\n").index("cluster_known_nodes:1")) || (@r.info['db0']) xputs "[ERR] Node #{self} is not empty. Either the node already knows other nodes (check with CLUSTER NODES) or contains some key in database 0." exit 1 end end def load_info(o={}) self.connect nodes = @r.cluster("nodes").split("\n") nodes.each{|n| # name addr flags role ping_sent ping_recv link_status slots split = n.split name,addr,flags,master_id,ping_sent,ping_recv,config_epoch,link_status = split[0..6] slots = split[8..-1] info = { :name => name, :addr => addr, :flags => flags.split(","), :replicate => master_id, :ping_sent => ping_sent.to_i, :ping_recv => ping_recv.to_i, :link_status => link_status } info[:replicate] = false if master_id == "-" if info[:flags].index("myself") @info = @info.merge(info) @info[:slots] = {} slots.each{|s| if s[0..0] == '[' if s.index("->-") # Migrating slot,dst = s[1..-1].split("->-") @info[:migrating][slot.to_i] = dst elsif s.index("-<-") # Importing slot,src = s[1..-1].split("-<-") @info[:importing][slot.to_i] = src end elsif s.index("-") start,stop = s.split("-") self.add_slots((start.to_i)..(stop.to_i)) else self.add_slots((s.to_i)..(s.to_i)) end } if slots @dirty = false @r.cluster("info").split("\n").each{|e| k,v=e.split(":") k = k.to_sym v.chop! if k != :cluster_state @info[k] = v.to_i else @info[k] = v end } elsif o[:getfriends] @friends << info end } end def add_slots(slots) slots.each{|s| @info[:slots][s] = :new } @dirty = true end def set_as_replica(node_id) @info[:replicate] = node_id @dirty = true end def flush_node_config return if !@dirty if @info[:replicate] begin @r.cluster("replicate",@info[:replicate]) rescue # If the cluster did not already joined it is possible that # the slave does not know the master node yet. So on errors # we return ASAP leaving the dirty flag set, to flush the # config later. return end else new = [] @info[:slots].each{|s,val| if val == :new new << s @info[:slots][s] = true end } @r.cluster("addslots",*new) end @dirty = false end def info_string # We want to display the hash slots assigned to this node # as ranges, like in: "1-5,8-9,20-25,30" # # Note: this could be easily written without side effects, # we use 'slots' just to split the computation into steps. # First step: we want an increasing array of integers # for instance: [1,2,3,4,5,8,9,20,21,22,23,24,25,30] slots = @info[:slots].keys.sort # As we want to aggregate adjacent slots we convert all the # slot integers into ranges (with just one element) # So we have something like [1..1,2..2, ... and so forth. slots.map!{|x| x..x} # Finally we group ranges with adjacent elements. slots = slots.reduce([]) {|a,b| if !a.empty? && b.first == (a[-1].last)+1 a[0..-2] + [(a[-1].first)..(b.last)] else a + [b] end } # Now our task is easy, we just convert ranges with just one # element into a number, and a real range into a start-end format. # Finally we join the array using the comma as separator. slots = slots.map{|x| x.count == 1 ? x.first.to_s : "#{x.first}-#{x.last}" }.join(",") role = self.has_flag?("master") ? "M" : "S" if self.info[:replicate] and @dirty is = "S: #{self.info[:name]} #{self.to_s}" else is = "#{role}: #{self.info[:name]} #{self.to_s}\n"+ " slots:#{slots} (#{self.slots.length} slots) "+ "#{(self.info[:flags]-["myself"]).join(",")}" end if self.info[:replicate] is += "\n replicates #{info[:replicate]}" elsif self.has_flag?("master") && self.info[:replicas] is += "\n #{info[:replicas].length} additional replica(s)" end is end # Return a single string representing nodes and associated slots. # TODO: remove slaves from config when slaves will be handled # by Redis Cluster. def get_config_signature config = [] @r.cluster("nodes").each_line{|l| s = l.split slots = s[8..-1].select {|x| x[0..0] != "["} next if slots.length == 0 config << s[0]+":"+(slots.sort.join(",")) } config.sort.join("|") end def info @info end def is_dirty? @dirty end def r @r end end class RedisTrib def initialize @nodes = [] @fix = false @errors = [] @timeout = MigrateDefaultTimeout end def check_arity(req_args, num_args) if ((req_args > 0 and num_args != req_args) || (req_args < 0 and num_args < req_args.abs)) xputs "[ERR] Wrong number of arguments for specified sub command" exit 1 end end def add_node(node) @nodes << node end def reset_nodes @nodes = [] end def cluster_error(msg) @errors << msg xputs msg end # Return the node with the specified ID or Nil. def get_node_by_name(name) @nodes.each{|n| return n if n.info[:name] == name.downcase } return nil end # Like get_node_by_name but the specified name can be just the first # part of the node ID as long as the prefix in unique across the # cluster. def get_node_by_abbreviated_name(name) l = name.length candidates = [] @nodes.each{|n| if n.info[:name][0...l] == name.downcase candidates << n end } return nil if candidates.length != 1 candidates[0] end # This function returns the master that has the least number of replicas # in the cluster. If there are multiple masters with the same smaller # number of replicas, one at random is returned. def get_master_with_least_replicas masters = @nodes.select{|n| n.has_flag? "master"} sorted = masters.sort{|a,b| a.info[:replicas].length <=> b.info[:replicas].length } sorted[0] end def check_cluster(opt={}) xputs ">>> Performing Cluster Check (using node #{@nodes[0]})" show_nodes if !opt[:quiet] check_config_consistency check_open_slots check_slots_coverage end def show_cluster_info masters = 0 keys = 0 @nodes.each{|n| if n.has_flag?("master") puts "#{n} (#{n.info[:name][0...8]}...) -> #{n.r.dbsize} keys | #{n.slots.length} slots | "+ "#{n.info[:replicas].length} slaves." masters += 1 keys += n.r.dbsize end } xputs "[OK] #{keys} keys in #{masters} masters." keys_per_slot = sprintf("%.2f",keys/16384.0) puts "#{keys_per_slot} keys per slot on average." end # Merge slots of every known node. If the resulting slots are equal # to ClusterHashSlots, then all slots are served. def covered_slots slots = {} @nodes.each{|n| slots = slots.merge(n.slots) } slots end def check_slots_coverage xputs ">>> Check slots coverage..." slots = covered_slots if slots.length == ClusterHashSlots xputs "[OK] All #{ClusterHashSlots} slots covered." else cluster_error \ "[ERR] Not all #{ClusterHashSlots} slots are covered by nodes." fix_slots_coverage if @fix end end def check_open_slots xputs ">>> Check for open slots..." open_slots = [] @nodes.each{|n| if n.info[:migrating].size > 0 cluster_error \ "[WARNING] Node #{n} has slots in migrating state (#{n.info[:migrating].keys.join(",")})." open_slots += n.info[:migrating].keys elsif n.info[:importing].size > 0 cluster_error \ "[WARNING] Node #{n} has slots in importing state (#{n.info[:importing].keys.join(",")})." open_slots += n.info[:importing].keys end } open_slots.uniq! if open_slots.length > 0 xputs "[WARNING] The following slots are open: #{open_slots.join(",")}" end if @fix open_slots.each{|slot| fix_open_slot slot} end end def nodes_with_keys_in_slot(slot) nodes = [] @nodes.each{|n| next if n.has_flag?("slave") nodes << n if n.r.cluster("getkeysinslot",slot,1).length > 0 } nodes end def fix_slots_coverage not_covered = (0...ClusterHashSlots).to_a - covered_slots.keys xputs ">>> Fixing slots coverage..." xputs "List of not covered slots: " + not_covered.join(",") # For every slot, take action depending on the actual condition: # 1) No node has keys for this slot. # 2) A single node has keys for this slot. # 3) Multiple nodes have keys for this slot. slots = {} not_covered.each{|slot| nodes = nodes_with_keys_in_slot(slot) slots[slot] = nodes xputs "Slot #{slot} has keys in #{nodes.length} nodes: #{nodes.join(", ")}" } none = slots.select {|k,v| v.length == 0} single = slots.select {|k,v| v.length == 1} multi = slots.select {|k,v| v.length > 1} # Handle case "1": keys in no node. if none.length > 0 xputs "The folowing uncovered slots have no keys across the cluster:" xputs none.keys.join(",") yes_or_die "Fix these slots by covering with a random node?" none.each{|slot,nodes| node = @nodes.sample xputs ">>> Covering slot #{slot} with #{node}" node.r.cluster("addslots",slot) } end # Handle case "2": keys only in one node. if single.length > 0 xputs "The folowing uncovered slots have keys in just one node:" puts single.keys.join(",") yes_or_die "Fix these slots by covering with those nodes?" single.each{|slot,nodes| xputs ">>> Covering slot #{slot} with #{nodes[0]}" nodes[0].r.cluster("addslots",slot) } end # Handle case "3": keys in multiple nodes. if multi.length > 0 xputs "The folowing uncovered slots have keys in multiple nodes:" xputs multi.keys.join(",") yes_or_die "Fix these slots by moving keys into a single node?" multi.each{|slot,nodes| target = get_node_with_most_keys_in_slot(nodes,slot) xputs ">>> Covering slot #{slot} moving keys to #{target}" target.r.cluster('addslots',slot) target.r.cluster('setslot',slot,'stable') nodes.each{|src| next if src == target # Set the source node in 'importing' state (even if we will # actually migrate keys away) in order to avoid receiving # redirections for MIGRATE. src.r.cluster('setslot',slot,'importing',target.info[:name]) move_slot(src,target,slot,:dots=>true,:fix=>true,:cold=>true) src.r.cluster('setslot',slot,'stable') } } end end # Return the owner of the specified slot def get_slot_owners(slot) owners = [] @nodes.each{|n| next if n.has_flag?("slave") n.slots.each{|s,_| owners << n if s == slot } } owners end # Return the node, among 'nodes' with the greatest number of keys # in the specified slot. def get_node_with_most_keys_in_slot(nodes,slot) best = nil best_numkeys = 0 @nodes.each{|n| next if n.has_flag?("slave") numkeys = n.r.cluster("countkeysinslot",slot) if numkeys > best_numkeys || best == nil best = n best_numkeys = numkeys end } return best end # Slot 'slot' was found to be in importing or migrating state in one or # more nodes. This function fixes this condition by migrating keys where # it seems more sensible. def fix_open_slot(slot) puts ">>> Fixing open slot #{slot}" # Try to obtain the current slot owner, according to the current # nodes configuration. owners = get_slot_owners(slot) owner = owners[0] if owners.length == 1 migrating = [] importing = [] @nodes.each{|n| next if n.has_flag? "slave" if n.info[:migrating][slot] migrating << n elsif n.info[:importing][slot] importing << n elsif n.r.cluster("countkeysinslot",slot) > 0 && n != owner xputs "*** Found keys about slot #{slot} in node #{n}!" importing << n end } puts "Set as migrating in: #{migrating.join(",")}" puts "Set as importing in: #{importing.join(",")}" # If there is no slot owner, set as owner the slot with the biggest # number of keys, among the set of migrating / importing nodes. if !owner xputs ">>> Nobody claims ownership, selecting an owner..." owner = get_node_with_most_keys_in_slot(@nodes,slot) # If we still don't have an owner, we can't fix it. if !owner xputs "[ERR] Can't select a slot owner. Impossible to fix." exit 1 end # Use ADDSLOTS to assign the slot. puts "*** Configuring #{owner} as the slot owner" n.r.cluster("setslot",slot,"stable") n.r.cluster("addslot",slot) # Make sure this information will propagate. Not strictly needed # since there is no past owner, so all the other nodes will accept # whatever epoch this node will claim the slot with. n.r.cluster("bumpepoch") # Remove the owner from the list of migrating/importing # nodes. migrating.delete(n) importing.delete(n) end # If there are multiple owners of the slot, we need to fix it # so that a single node is the owner and all the other nodes # are in importing state. Later the fix can be handled by one # of the base cases above. # # Note that this case also covers multiple nodes having the slot # in migrating state, since migrating is a valid state only for # slot owners. if owners.length > 1 owner = get_node_with_most_keys_in_slot(owners,slot) owners.each{|n| next if n == owner n.r.cluster('delslots',slot) n.r.cluster('setslot',slot,'importing',owner.info[:name]) importing.delete(n) # Avoid duplciates importing << n } owner.r.cluster('bumpepoch') end # Case 1: The slot is in migrating state in one slot, and in # importing state in 1 slot. That's trivial to address. if migrating.length == 1 && importing.length == 1 move_slot(migrating[0],importing[0],slot,:dots=>true,:fix=>true) # Case 2: There are multiple nodes that claim the slot as importing, # they probably got keys about the slot after a restart so opened # the slot. In this case we just move all the keys to the owner # according to the configuration. elsif migrating.length == 0 && importing.length > 0 xputs ">>> Moving all the #{slot} slot keys to its owner #{owner}" importing.each {|node| next if node == owner move_slot(node,owner,slot,:dots=>true,:fix=>true,:cold=>true) xputs ">>> Setting #{slot} as STABLE in #{node}" node.r.cluster("setslot",slot,"stable") } # Case 3: There are no slots claiming to be in importing state, but # there is a migrating node that actually don't have any key. We # can just close the slot, probably a reshard interrupted in the middle. elsif importing.length == 0 && migrating.length == 1 && migrating[0].r.cluster("getkeysinslot",slot,10).length == 0 migrating[0].r.cluster("setslot",slot,"stable") else xputs "[ERR] Sorry, Redis-trib can't fix this slot yet (work in progress). Slot is set as migrating in #{migrating.join(",")}, as importing in #{importing.join(",")}, owner is #{owner}" end end # Check if all the nodes agree about the cluster configuration def check_config_consistency if !is_config_consistent? cluster_error "[ERR] Nodes don't agree about configuration!" else xputs "[OK] All nodes agree about slots configuration." end end def is_config_consistent? signatures=[] @nodes.each{|n| signatures << n.get_config_signature } return signatures.uniq.length == 1 end def wait_cluster_join print "Waiting for the cluster to join" while !is_config_consistent? print "." STDOUT.flush sleep 1 end print "\n" end def alloc_slots nodes_count = @nodes.length masters_count = @nodes.length / (@replicas+1) masters = [] # The first step is to split instances by IP. This is useful as # we'll try to allocate master nodes in different physical machines # (as much as possible) and to allocate slaves of a given master in # different physical machines as well. # # This code assumes just that if the IP is different, than it is more # likely that the instance is running in a different physical host # or at least a different virtual machine. ips = {} @nodes.each{|n| ips[n.info[:host]] = [] if !ips[n.info[:host]] ips[n.info[:host]] << n } # Select master instances puts "Using #{masters_count} masters:" interleaved = [] stop = false while not stop do # Take one node from each IP until we run out of nodes # across every IP. ips.each do |ip,nodes| if nodes.empty? # if this IP has no remaining nodes, check for termination if interleaved.length == nodes_count # stop when 'interleaved' has accumulated all nodes stop = true next end else # else, move one node from this IP to 'interleaved' interleaved.push nodes.shift end end end masters = interleaved.slice!(0, masters_count) nodes_count -= masters.length masters.each{|m| puts m} # Alloc slots on masters slots_per_node = ClusterHashSlots.to_f / masters_count first = 0 cursor = 0.0 masters.each_with_index{|n,masternum| last = (cursor+slots_per_node-1).round if last > ClusterHashSlots || masternum == masters.length-1 last = ClusterHashSlots-1 end last = first if last < first # Min step is 1. n.add_slots first..last first = last+1 cursor += slots_per_node } # Select N replicas for every master. # We try to split the replicas among all the IPs with spare nodes # trying to avoid the host where the master is running, if possible. # # Note we loop two times. The first loop assigns the requested # number of replicas to each master. The second loop assigns any # remaining instances as extra replicas to masters. Some masters # may end up with more than their requested number of replicas, but # all nodes will be used. assignment_verbose = false [:requested,:unused].each do |assign| masters.each do |m| assigned_replicas = 0 while assigned_replicas < @replicas break if nodes_count == 0 if assignment_verbose if assign == :requested puts "Requesting total of #{@replicas} replicas " \ "(#{assigned_replicas} replicas assigned " \ "so far with #{nodes_count} total remaining)." elsif assign == :unused puts "Assigning extra instance to replication " \ "role too (#{nodes_count} remaining)." end end # Return the first node not matching our current master node = interleaved.find{|n| n.info[:host] != m.info[:host]} # If we found a node, use it as a best-first match. # Otherwise, we didn't find a node on a different IP, so we # go ahead and use a same-IP replica. if node slave = node interleaved.delete node else slave = interleaved.shift end slave.set_as_replica(m.info[:name]) nodes_count -= 1 assigned_replicas += 1 puts "Adding replica #{slave} to #{m}" # If we are in the "assign extra nodes" loop, # we want to assign one extra replica to each # master before repeating masters. # This break lets us assign extra replicas to masters # in a round-robin way. break if assign == :unused end end end end def flush_nodes_config @nodes.each{|n| n.flush_node_config } end def show_nodes @nodes.each{|n| xputs n.info_string } end # Redis Cluster config epoch collision resolution code is able to eventually # set a different epoch to each node after a new cluster is created, but # it is slow compared to assign a progressive config epoch to each node # before joining the cluster. However we do just a best-effort try here # since if we fail is not a problem. def assign_config_epoch config_epoch = 1 @nodes.each{|n| begin n.r.cluster("set-config-epoch",config_epoch) rescue end config_epoch += 1 } end def join_cluster # We use a brute force approach to make sure the node will meet # each other, that is, sending CLUSTER MEET messages to all the nodes # about the very same node. # Thanks to gossip this information should propagate across all the # cluster in a matter of seconds. first = false @nodes.each{|n| if !first then first = n.info; next; end # Skip the first node n.r.cluster("meet",first[:host],first[:port]) } end def yes_or_die(msg) print "#{msg} (type 'yes' to accept): " STDOUT.flush if !(STDIN.gets.chomp.downcase == "yes") xputs "*** Aborting..." exit 1 end end def load_cluster_info_from_node(nodeaddr) node = ClusterNode.new(nodeaddr) node.connect(:abort => true) node.assert_cluster node.load_info(:getfriends => true) add_node(node) node.friends.each{|f| next if f[:flags].index("noaddr") || f[:flags].index("disconnected") || f[:flags].index("fail") fnode = ClusterNode.new(f[:addr]) fnode.connect() next if !fnode.r begin fnode.load_info() add_node(fnode) rescue => e xputs "[ERR] Unable to load info for node #{fnode}" end } populate_nodes_replicas_info end # This function is called by load_cluster_info_from_node in order to # add additional information to every node as a list of replicas. def populate_nodes_replicas_info # Start adding the new field to every node. @nodes.each{|n| n.info[:replicas] = [] } # Populate the replicas field using the replicate field of slave # nodes. @nodes.each{|n| if n.info[:replicate] master = get_node_by_name(n.info[:replicate]) if !master xputs "*** WARNING: #{n} claims to be slave of unknown node ID #{n.info[:replicate]}." else master.info[:replicas] << n end end } end # Given a list of source nodes return a "resharding plan" # with what slots to move in order to move "numslots" slots to another # instance. def compute_reshard_table(sources,numslots) moved = [] # Sort from bigger to smaller instance, for two reasons: # 1) If we take less slots than instances it is better to start # getting from the biggest instances. # 2) We take one slot more from the first instance in the case of not # perfect divisibility. Like we have 3 nodes and need to get 10 # slots, we take 4 from the first, and 3 from the rest. So the # biggest is always the first. sources = sources.sort{|a,b| b.slots.length <=> a.slots.length} source_tot_slots = sources.inject(0) {|sum,source| sum+source.slots.length } sources.each_with_index{|s,i| # Every node will provide a number of slots proportional to the # slots it has assigned. n = (numslots.to_f/source_tot_slots*s.slots.length) if i == 0 n = n.ceil else n = n.floor end s.slots.keys.sort[(0...n)].each{|slot| if moved.length < numslots moved << {:source => s, :slot => slot} end } } return moved end def show_reshard_table(table) table.each{|e| puts " Moving slot #{e[:slot]} from #{e[:source].info[:name]}" } end # Move slots between source and target nodes using MIGRATE. # # Options: # :verbose -- Print a dot for every moved key. # :fix -- We are moving in the context of a fix. Use REPLACE. # :cold -- Move keys without opening slots / reconfiguring the nodes. # :update -- Update nodes.info[:slots] for source/target nodes. # :quiet -- Don't print info messages. def move_slot(source,target,slot,o={}) o = {:pipeline => MigrateDefaultPipeline}.merge(o) # We start marking the slot as importing in the destination node, # and the slot as migrating in the target host. Note that the order of # the operations is important, as otherwise a client may be redirected # to the target node that does not yet know it is importing this slot. if !o[:quiet] print "Moving slot #{slot} from #{source} to #{target}: " STDOUT.flush end if !o[:cold] target.r.cluster("setslot",slot,"importing",source.info[:name]) source.r.cluster("setslot",slot,"migrating",target.info[:name]) end # Migrate all the keys from source to target using the MIGRATE command while true keys = source.r.cluster("getkeysinslot",slot,o[:pipeline]) break if keys.length == 0 begin source.r.client.call(["migrate",target.info[:host],target.info[:port],"",0,@timeout,:keys,*keys]) rescue => e if o[:fix] && e.to_s =~ /BUSYKEY/ xputs "*** Target key exists. Replacing it for FIX." source.r.client.call(["migrate",target.info[:host],target.info[:port],"",0,@timeout,:replace,:keys,*keys]) else puts "" xputs "[ERR] #{e}" exit 1 end end print "."*keys.length if o[:dots] STDOUT.flush end puts if !o[:quiet] # Set the new node as the owner of the slot in all the known nodes. if !o[:cold] @nodes.each{|n| next if n.has_flag?("slave") n.r.cluster("setslot",slot,"node",target.info[:name]) } end # Update the node logical config if o[:update] then source.info[:slots].delete(slot) target.info[:slots][slot] = true end end # redis-trib subcommands implementations. def check_cluster_cmd(argv,opt) load_cluster_info_from_node(argv[0]) check_cluster end def info_cluster_cmd(argv,opt) load_cluster_info_from_node(argv[0]) show_cluster_info end def rebalance_cluster_cmd(argv,opt) opt = { 'pipeline' => MigrateDefaultPipeline, 'threshold' => RebalanceDefaultThreshold }.merge(opt) # Load nodes info before parsing options, otherwise we can't # handle --weight. load_cluster_info_from_node(argv[0]) # Options parsing threshold = opt['threshold'].to_i autoweights = opt['auto-weights'] weights = {} opt['weight'].each{|w| fields = w.split("=") node = get_node_by_abbreviated_name(fields[0]) if !node || !node.has_flag?("master") puts "*** No such master node #{fields[0]}" exit 1 end weights[node.info[:name]] = fields[1].to_f } if opt['weight'] useempty = opt['use-empty-masters'] # Assign a weight to each node, and compute the total cluster weight. total_weight = 0 nodes_involved = 0 @nodes.each{|n| if n.has_flag?("master") next if !useempty && n.slots.length == 0 n.info[:w] = weights[n.info[:name]] ? weights[n.info[:name]] : 1 total_weight += n.info[:w] nodes_involved += 1 end } # Check cluster, only proceed if it looks sane. check_cluster(:quiet => true) if @errors.length != 0 puts "*** Please fix your cluster problems before rebalancing" exit 1 end # Calculate the slots balance for each node. It's the number of # slots the node should lose (if positive) or gain (if negative) # in order to be balanced. threshold = opt['threshold'].to_f threshold_reached = false @nodes.each{|n| if n.has_flag?("master") next if !n.info[:w] expected = ((ClusterHashSlots.to_f / total_weight) * n.info[:w]).to_i n.info[:balance] = n.slots.length - expected # Compute the percentage of difference between the # expected number of slots and the real one, to see # if it's over the threshold specified by the user. over_threshold = false if threshold > 0 if n.slots.length > 0 err_perc = (100-(100.0*expected/n.slots.length)).abs over_threshold = true if err_perc > threshold elsif expected > 0 over_threshold = true end end threshold_reached = true if over_threshold end } if !threshold_reached xputs "*** No rebalancing needed! All nodes are within the #{threshold}% threshold." return end # Only consider nodes we want to change sn = @nodes.select{|n| n.has_flag?("master") && n.info[:w] } # Because of rounding, it is possible that the balance of all nodes # summed does not give 0. Make sure that nodes that have to provide # slots are always matched by nodes receiving slots. total_balance = sn.map{|x| x.info[:balance]}.reduce{|a,b| a+b} while total_balance > 0 sn.each{|n| if n.info[:balance] < 0 && total_balance > 0 n.info[:balance] -= 1 total_balance -= 1 end } end # Sort nodes by their slots balance. sn = sn.sort{|a,b| a.info[:balance] <=> b.info[:balance] } xputs ">>> Rebalancing across #{nodes_involved} nodes. Total weight = #{total_weight}" if $verbose sn.each{|n| puts "#{n} balance is #{n.info[:balance]} slots" } end # Now we have at the start of the 'sn' array nodes that should get # slots, at the end nodes that must give slots. # We take two indexes, one at the start, and one at the end, # incrementing or decrementing the indexes accordingly til we # find nodes that need to get/provide slots. dst_idx = 0 src_idx = sn.length - 1 while dst_idx < src_idx dst = sn[dst_idx] src = sn[src_idx] numslots = [dst.info[:balance],src.info[:balance]].map{|n| n.abs }.min if numslots > 0 puts "Moving #{numslots} slots from #{src} to #{dst}" # Actaully move the slots. reshard_table = compute_reshard_table([src],numslots) if reshard_table.length != numslots xputs "*** Assertio failed: Reshard table != number of slots" exit 1 end if opt['simulate'] print "#"*reshard_table.length else reshard_table.each{|e| move_slot(e[:source],dst,e[:slot], :quiet=>true, :dots=>false, :update=>true, :pipeline=>opt['pipeline']) print "#" STDOUT.flush } end puts end # Update nodes balance. dst.info[:balance] += numslots src.info[:balance] -= numslots dst_idx += 1 if dst.info[:balance] == 0 src_idx -= 1 if src.info[:balance] == 0 end end def fix_cluster_cmd(argv,opt) @fix = true @timeout = opt['timeout'].to_i if opt['timeout'] load_cluster_info_from_node(argv[0]) check_cluster end def reshard_cluster_cmd(argv,opt) opt = {'pipeline' => MigrateDefaultPipeline}.merge(opt) load_cluster_info_from_node(argv[0]) check_cluster if @errors.length != 0 puts "*** Please fix your cluster problems before resharding" exit 1 end @timeout = opt['timeout'].to_i if opt['timeout'].to_i # Get number of slots if opt['slots'] numslots = opt['slots'].to_i else numslots = 0 while numslots <= 0 or numslots > ClusterHashSlots print "How many slots do you want to move (from 1 to #{ClusterHashSlots})? " numslots = STDIN.gets.to_i end end # Get the target instance if opt['to'] target = get_node_by_name(opt['to']) if !target || target.has_flag?("slave") xputs "*** The specified node is not known or not a master, please retry." exit 1 end else target = nil while not target print "What is the receiving node ID? " target = get_node_by_name(STDIN.gets.chop) if !target || target.has_flag?("slave") xputs "*** The specified node is not known or not a master, please retry." target = nil end end end # Get the source instances sources = [] if opt['from'] opt['from'].split(',').each{|node_id| if node_id == "all" sources = "all" break end src = get_node_by_name(node_id) if !src || src.has_flag?("slave") xputs "*** The specified node is not known or is not a master, please retry." exit 1 end sources << src } else xputs "Please enter all the source node IDs." xputs " Type 'all' to use all the nodes as source nodes for the hash slots." xputs " Type 'done' once you entered all the source nodes IDs." while true print "Source node ##{sources.length+1}:" line = STDIN.gets.chop src = get_node_by_name(line) if line == "done" break elsif line == "all" sources = "all" break elsif !src || src.has_flag?("slave") xputs "*** The specified node is not known or is not a master, please retry." elsif src.info[:name] == target.info[:name] xputs "*** It is not possible to use the target node as source node." else sources << src end end end if sources.length == 0 puts "*** No source nodes given, operation aborted" exit 1 end # Handle soures == all. if sources == "all" sources = [] @nodes.each{|n| next if n.info[:name] == target.info[:name] next if n.has_flag?("slave") sources << n } end # Check if the destination node is the same of any source nodes. if sources.index(target) xputs "*** Target node is also listed among the source nodes!" exit 1 end puts "\nReady to move #{numslots} slots." puts " Source nodes:" sources.each{|s| puts " "+s.info_string} puts " Destination node:" puts " #{target.info_string}" reshard_table = compute_reshard_table(sources,numslots) puts " Resharding plan:" show_reshard_table(reshard_table) if !opt['yes'] print "Do you want to proceed with the proposed reshard plan (yes/no)? " yesno = STDIN.gets.chop exit(1) if (yesno != "yes") end reshard_table.each{|e| move_slot(e[:source],target,e[:slot], :dots=>true, :pipeline=>opt['pipeline']) } end # This is an helper function for create_cluster_cmd that verifies if # the number of nodes and the specified replicas have a valid configuration # where there are at least three master nodes and enough replicas per node. def check_create_parameters masters = @nodes.length/(@replicas+1) if masters < 3 puts "*** ERROR: Invalid configuration for cluster creation." puts "*** Redis Cluster requires at least 3 master nodes." puts "*** This is not possible with #{@nodes.length} nodes and #{@replicas} replicas per node." puts "*** At least #{3*(@replicas+1)} nodes are required." exit 1 end end def create_cluster_cmd(argv,opt) opt = {'replicas' => 0}.merge(opt) @replicas = opt['replicas'].to_i xputs ">>> Creating cluster" argv[0..-1].each{|n| node = ClusterNode.new(n) node.connect(:abort => true) node.assert_cluster node.load_info node.assert_empty add_node(node) } check_create_parameters xputs ">>> Performing hash slots allocation on #{@nodes.length} nodes..." alloc_slots show_nodes yes_or_die "Can I set the above configuration?" flush_nodes_config xputs ">>> Nodes configuration updated" xputs ">>> Assign a different config epoch to each node" assign_config_epoch xputs ">>> Sending CLUSTER MEET messages to join the cluster" join_cluster # Give one second for the join to start, in order to avoid that # wait_cluster_join will find all the nodes agree about the config as # they are still empty with unassigned slots. sleep 1 wait_cluster_join flush_nodes_config # Useful for the replicas check_cluster end def addnode_cluster_cmd(argv,opt) xputs ">>> Adding node #{argv[0]} to cluster #{argv[1]}" # Check the existing cluster load_cluster_info_from_node(argv[1]) check_cluster # If --master-id was specified, try to resolve it now so that we # abort before starting with the node configuration. if opt['slave'] if opt['master-id'] master = get_node_by_name(opt['master-id']) if !master xputs "[ERR] No such master ID #{opt['master-id']}" end else master = get_master_with_least_replicas xputs "Automatically selected master #{master}" end end # Add the new node new = ClusterNode.new(argv[0]) new.connect(:abort => true) new.assert_cluster new.load_info new.assert_empty first = @nodes.first.info add_node(new) # Send CLUSTER MEET command to the new node xputs ">>> Send CLUSTER MEET to node #{new} to make it join the cluster." new.r.cluster("meet",first[:host],first[:port]) # Additional configuration is needed if the node is added as # a slave. if opt['slave'] wait_cluster_join xputs ">>> Configure node as replica of #{master}." new.r.cluster("replicate",master.info[:name]) end xputs "[OK] New node added correctly." end def delnode_cluster_cmd(argv,opt) id = argv[1].downcase xputs ">>> Removing node #{id} from cluster #{argv[0]}" # Load cluster information load_cluster_info_from_node(argv[0]) # Check if the node exists and is not empty node = get_node_by_name(id) if !node xputs "[ERR] No such node ID #{id}" exit 1 end if node.slots.length != 0 xputs "[ERR] Node #{node} is not empty! Reshard data away and try again." exit 1 end # Send CLUSTER FORGET to all the nodes but the node to remove xputs ">>> Sending CLUSTER FORGET messages to the cluster..." @nodes.each{|n| next if n == node if n.info[:replicate] && n.info[:replicate].downcase == id # Reconfigure the slave to replicate with some other node master = get_master_with_least_replicas xputs ">>> #{n} as replica of #{master}" n.r.cluster("replicate",master.info[:name]) end n.r.cluster("forget",argv[1]) } # Finally shutdown the node xputs ">>> SHUTDOWN the node." node.r.shutdown end def set_timeout_cluster_cmd(argv,opt) timeout = argv[1].to_i if timeout < 100 puts "Setting a node timeout of less than 100 milliseconds is a bad idea." exit 1 end # Load cluster information load_cluster_info_from_node(argv[0]) ok_count = 0 err_count = 0 # Send CLUSTER FORGET to all the nodes but the node to remove xputs ">>> Reconfiguring node timeout in every cluster node..." @nodes.each{|n| begin n.r.config("set","cluster-node-timeout",timeout) n.r.config("rewrite") ok_count += 1 xputs "*** New timeout set for #{n}" rescue => e puts "ERR setting node-timeot for #{n}: #{e}" err_count += 1 end } xputs ">>> New node timeout set. #{ok_count} OK, #{err_count} ERR." end def call_cluster_cmd(argv,opt) cmd = argv[1..-1] cmd[0] = cmd[0].upcase # Load cluster information load_cluster_info_from_node(argv[0]) xputs ">>> Calling #{cmd.join(" ")}" @nodes.each{|n| begin res = n.r.send(*cmd) puts "#{n}: #{res}" rescue => e puts "#{n}: #{e}" end } end def import_cluster_cmd(argv,opt) source_addr = opt['from'] xputs ">>> Importing data from #{source_addr} to cluster #{argv[1]}" use_copy = opt['copy'] use_replace = opt['replace'] # Check the existing cluster. load_cluster_info_from_node(argv[0]) check_cluster # Connect to the source node. xputs ">>> Connecting to the source Redis instance" src_host,src_port = source_addr.split(":") source = Redis.new(:host =>src_host, :port =>src_port) if source.info['cluster_enabled'].to_i == 1 xputs "[ERR] The source node should not be a cluster node." end xputs "*** Importing #{source.dbsize} keys from DB 0" # Build a slot -> node map slots = {} @nodes.each{|n| n.slots.each{|s,_| slots[s] = n } } # Use SCAN to iterate over the keys, migrating to the # right node as needed. cursor = nil while cursor != 0 cursor,keys = source.scan(cursor, :count => 1000) cursor = cursor.to_i keys.each{|k| # Migrate keys using the MIGRATE command. slot = key_to_slot(k) target = slots[slot] print "Migrating #{k} to #{target}: " STDOUT.flush begin cmd = ["migrate",target.info[:host],target.info[:port],k,0,@timeout] cmd << :copy if use_copy cmd << :replace if use_replace source.client.call(cmd) rescue => e puts e else puts "OK" end } end end def help_cluster_cmd(argv,opt) show_help exit 0 end # Parse the options for the specific command "cmd". # Returns an hash populate with option => value pairs, and the index of # the first non-option argument in ARGV. def parse_options(cmd) idx = 1 ; # Current index into ARGV options={} while idx < ARGV.length && ARGV[idx][0..1] == '--' if ARGV[idx][0..1] == "--" option = ARGV[idx][2..-1] idx += 1 # --verbose is a global option if option == "verbose" $verbose = true next end if ALLOWED_OPTIONS[cmd] == nil || ALLOWED_OPTIONS[cmd][option] == nil puts "Unknown option '#{option}' for command '#{cmd}'" exit 1 end if ALLOWED_OPTIONS[cmd][option] != false value = ARGV[idx] idx += 1 else value = true end # If the option is set to [], it's a multiple arguments # option. We just queue every new value into an array. if ALLOWED_OPTIONS[cmd][option] == [] options[option] = [] if !options[option] options[option] << value else options[option] = value end else # Remaining arguments are not options. break end end # Enforce mandatory options if ALLOWED_OPTIONS[cmd] ALLOWED_OPTIONS[cmd].each {|option,val| if !options[option] && val == :required puts "Option '--#{option}' is required "+ \ "for subcommand '#{cmd}'" exit 1 end } end return options,idx end end ################################################################################# # Libraries # # We try to don't depend on external libs since this is a critical part # of Redis Cluster. ################################################################################# # This is the CRC16 algorithm used by Redis Cluster to hash keys. # Implementation according to CCITT standards. # # This is actually the XMODEM CRC 16 algorithm, using the # following parameters: # # Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" # Width : 16 bit # Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) # Initialization : 0000 # Reflect Input byte : False # Reflect Output CRC : False # Xor constant to output CRC : 0000 # Output for "123456789" : 31C3 module RedisClusterCRC16 def RedisClusterCRC16.crc16(bytes) crc = 0 bytes.each_byte{|b| crc = ((crc<<8) & 0xffff) ^ XMODEMCRC16Lookup[((crc>>8)^b) & 0xff] } crc end private XMODEMCRC16Lookup = [ 0x0000,0x1021,0x2042,0x3063,0x4084,0x50a5,0x60c6,0x70e7, 0x8108,0x9129,0xa14a,0xb16b,0xc18c,0xd1ad,0xe1ce,0xf1ef, 0x1231,0x0210,0x3273,0x2252,0x52b5,0x4294,0x72f7,0x62d6, 0x9339,0x8318,0xb37b,0xa35a,0xd3bd,0xc39c,0xf3ff,0xe3de, 0x2462,0x3443,0x0420,0x1401,0x64e6,0x74c7,0x44a4,0x5485, 0xa56a,0xb54b,0x8528,0x9509,0xe5ee,0xf5cf,0xc5ac,0xd58d, 0x3653,0x2672,0x1611,0x0630,0x76d7,0x66f6,0x5695,0x46b4, 0xb75b,0xa77a,0x9719,0x8738,0xf7df,0xe7fe,0xd79d,0xc7bc, 0x48c4,0x58e5,0x6886,0x78a7,0x0840,0x1861,0x2802,0x3823, 0xc9cc,0xd9ed,0xe98e,0xf9af,0x8948,0x9969,0xa90a,0xb92b, 0x5af5,0x4ad4,0x7ab7,0x6a96,0x1a71,0x0a50,0x3a33,0x2a12, 0xdbfd,0xcbdc,0xfbbf,0xeb9e,0x9b79,0x8b58,0xbb3b,0xab1a, 0x6ca6,0x7c87,0x4ce4,0x5cc5,0x2c22,0x3c03,0x0c60,0x1c41, 0xedae,0xfd8f,0xcdec,0xddcd,0xad2a,0xbd0b,0x8d68,0x9d49, 0x7e97,0x6eb6,0x5ed5,0x4ef4,0x3e13,0x2e32,0x1e51,0x0e70, 0xff9f,0xefbe,0xdfdd,0xcffc,0xbf1b,0xaf3a,0x9f59,0x8f78, 0x9188,0x81a9,0xb1ca,0xa1eb,0xd10c,0xc12d,0xf14e,0xe16f, 0x1080,0x00a1,0x30c2,0x20e3,0x5004,0x4025,0x7046,0x6067, 0x83b9,0x9398,0xa3fb,0xb3da,0xc33d,0xd31c,0xe37f,0xf35e, 0x02b1,0x1290,0x22f3,0x32d2,0x4235,0x5214,0x6277,0x7256, 0xb5ea,0xa5cb,0x95a8,0x8589,0xf56e,0xe54f,0xd52c,0xc50d, 0x34e2,0x24c3,0x14a0,0x0481,0x7466,0x6447,0x5424,0x4405, 0xa7db,0xb7fa,0x8799,0x97b8,0xe75f,0xf77e,0xc71d,0xd73c, 0x26d3,0x36f2,0x0691,0x16b0,0x6657,0x7676,0x4615,0x5634, 0xd94c,0xc96d,0xf90e,0xe92f,0x99c8,0x89e9,0xb98a,0xa9ab, 0x5844,0x4865,0x7806,0x6827,0x18c0,0x08e1,0x3882,0x28a3, 0xcb7d,0xdb5c,0xeb3f,0xfb1e,0x8bf9,0x9bd8,0xabbb,0xbb9a, 0x4a75,0x5a54,0x6a37,0x7a16,0x0af1,0x1ad0,0x2ab3,0x3a92, 0xfd2e,0xed0f,0xdd6c,0xcd4d,0xbdaa,0xad8b,0x9de8,0x8dc9, 0x7c26,0x6c07,0x5c64,0x4c45,0x3ca2,0x2c83,0x1ce0,0x0cc1, 0xef1f,0xff3e,0xcf5d,0xdf7c,0xaf9b,0xbfba,0x8fd9,0x9ff8, 0x6e17,0x7e36,0x4e55,0x5e74,0x2e93,0x3eb2,0x0ed1,0x1ef0 ] end # Turn a key name into the corrisponding Redis Cluster slot. def key_to_slot(key) # Only hash what is inside {...} if there is such a pattern in the key. # Note that the specification requires the content that is between # the first { and the first } after the first {. If we found {} without # nothing in the middle, the whole key is hashed as usually. s = key.index "{" if s e = key.index "}",s+1 if e && e != s+1 key = key[s+1..e-1] end end RedisClusterCRC16.crc16(key) % 16384 end ################################################################################# # Definition of commands ################################################################################# COMMANDS={ "create" => ["create_cluster_cmd", -2, "host1:port1 ... hostN:portN"], "check" => ["check_cluster_cmd", 2, "host:port"], "info" => ["info_cluster_cmd", 2, "host:port"], "fix" => ["fix_cluster_cmd", 2, "host:port"], "reshard" => ["reshard_cluster_cmd", 2, "host:port"], "rebalance" => ["rebalance_cluster_cmd", -2, "host:port"], "add-node" => ["addnode_cluster_cmd", 3, "new_host:new_port existing_host:existing_port"], "del-node" => ["delnode_cluster_cmd", 3, "host:port node_id"], "set-timeout" => ["set_timeout_cluster_cmd", 3, "host:port milliseconds"], "call" => ["call_cluster_cmd", -3, "host:port command arg arg .. arg"], "import" => ["import_cluster_cmd", 2, "host:port"], "help" => ["help_cluster_cmd", 1, "(show this help)"] } ALLOWED_OPTIONS={ "create" => {"replicas" => true}, "add-node" => {"slave" => false, "master-id" => true}, "import" => {"from" => :required, "copy" => false, "replace" => false}, "reshard" => {"from" => true, "to" => true, "slots" => true, "yes" => false, "timeout" => true, "pipeline" => true}, "rebalance" => {"weight" => [], "auto-weights" => false, "use-empty-masters" => false, "timeout" => true, "simulate" => false, "pipeline" => true, "threshold" => true}, "fix" => {"timeout" => MigrateDefaultTimeout}, } def show_help puts "Usage: redis-trib \n\n" COMMANDS.each{|k,v| o = "" puts " #{k.ljust(15)} #{v[2]}" if ALLOWED_OPTIONS[k] ALLOWED_OPTIONS[k].each{|optname,has_arg| puts " --#{optname}" + (has_arg ? " " : "") } end } puts "\nFor check, fix, reshard, del-node, set-timeout you can specify the host and port of any working node in the cluster.\n" end # Sanity check if ARGV.length == 0 show_help exit 1 end rt = RedisTrib.new cmd_spec = COMMANDS[ARGV[0].downcase] if !cmd_spec puts "Unknown redis-trib subcommand '#{ARGV[0]}'" exit 1 end # Parse options cmd_options,first_non_option = rt.parse_options(ARGV[0].downcase) rt.check_arity(cmd_spec[1],ARGV.length-(first_non_option-1)) # Dispatch rt.send(cmd_spec[0],ARGV[first_non_option..-1],cmd_options) mna-redisc-96a8e49/cluster.go000066400000000000000000000373301451450242600161300ustar00rootroot00000000000000package redisc import ( "context" "errors" "math/rand" "strconv" "strings" "sync" "time" "github.com/gomodule/redigo/redis" ) // HashSlots is the number of slots supported by redis cluster. const HashSlots = 16384 // BgErrorSrc identifies the origin of a background error as reported by calls // to Cluster.BgError, when set. type BgErrorSrc uint // List of possible BgErrorSrc values. const ( // ClusterRefresh indicates the error comes from a background refresh of // cluster slots mapping, e.g. following reception of a MOVED error. ClusterRefresh BgErrorSrc = iota // RetryCloseConn indicates the error comes from the call to Close for a // previous connection, before retrying a command with a new one. RetryCloseConn ) // A Cluster manages a redis cluster. If the CreatePool field is not nil, a // redis.Pool is used for each node in the cluster to get connections via Get. // If it is nil or if Dial is called, redis.Dial is used to get the connection. // // All fields must be set prior to using the Cluster value, and must not be // changed afterwards, as that could be a data race. type Cluster struct { // StartupNodes is the list of initial nodes that make up the cluster. The // values are expected as "address:port" (e.g.: "127.0.0.1:6379"). StartupNodes []string // DialOptions is the list of options to set on each new connection. DialOptions []redis.DialOption // CreatePool is the function to call to create a redis.Pool for the // specified TCP address, using the provided options as set in DialOptions. // If this field is not nil, a redis.Pool is created for each node in the // cluster and the pool is used to manage the connections returned by Get. CreatePool func(address string, options ...redis.DialOption) (*redis.Pool, error) // PoolWaitTime is the time to wait when getting a connection from a pool // configured with MaxActive > 0 and Wait set to true, and MaxActive // connections are already in use. // // If <= 0 (or with Go < 1.7), there is no wait timeout, it will wait // indefinitely if Pool.Wait is true. PoolWaitTime time.Duration // BgError is an optional function to call when a background error occurs // that would otherwise go unnoticed. The source of the error is indicated // by the parameter of type BgErrorSrc, see the list of BgErrorSrc values // for possible error sources. The function may be called in a distinct // goroutine, it should not access shared values that are not meant to be // used concurrently. BgError func(BgErrorSrc, error) // LayoutRefresh is an optional function that is called each time a cluster // refresh is successfully executed, either by an explicit call to // Cluster.Refresh or e.g. as required following a MOVED error. Note that // even though it is unlikely, the old and new mappings could be identical. // The function may be called in a separate goroutine, it should not access // shared values that are not meant to be used concurrently. LayoutRefresh func(old, new [HashSlots][]string) mu sync.RWMutex // protects following fields err error // closed cluster error pools map[string]*redis.Pool // created pools per node address masters map[string]bool // set of known active master nodes addresses, kept up-to-date replicas map[string]bool // set of known active replica nodes addresses, kept up-to-date mapping [HashSlots][]string // hash slot number to master and replica(s) addresses, master is always at [0] refreshing bool // indicates if there's a refresh in progress } // Refresh updates the cluster's internal mapping of hash slots to redis node. // It calls CLUSTER SLOTS on each known node until one of them succeeds. // // It should typically be called after creating the Cluster and before using // it. The cluster automatically keeps its mapping up-to-date afterwards, based // on the redis commands' MOVED responses. func (c *Cluster) Refresh() error { c.mu.Lock() err := c.err if err == nil { c.refreshing = true } c.mu.Unlock() if err != nil { return err } return c.refresh(false) } func (c *Cluster) refresh(bg bool) error { var errMsgs []string var oldm, newm [HashSlots][]string // get master nodes first addrs, _ := c.getNodeAddrs(false) if len(addrs) == 0 { // when master nodes cannot be obtained, try to get all replicas if addrs, _ = c.getNodeAddrs(true); len(addrs) == 0 { // when there is no node information, StartupNodes is always used to populate addrs = c.StartupNodes } } for _, addr := range addrs { m, err := c.getClusterSlots(addr) if err != nil { errMsgs = append(errMsgs, err.Error()) continue } // succeeded, save as mapping c.mu.Lock() oldm = c.mapping // mark all current nodes as false for k := range c.masters { c.masters[k] = false } for k := range c.replicas { c.replicas[k] = false } for _, sm := range m { for i, node := range sm.nodes { if node != "" { target := c.masters if i > 0 { target = c.replicas } target[node] = true } } for ix := sm.start; ix <= sm.end; ix++ { c.mapping[ix] = sm.nodes } } // remove all nodes that are gone from the cluster for _, nodes := range []map[string]bool{c.masters, c.replicas} { for k, ok := range nodes { if !ok { delete(nodes, k) // close and remove all existing pools for removed nodes if p := c.pools[k]; p != nil { // Pool.Close always returns nil p.Close() delete(c.pools, k) } } } } // mark that no refresh is needed until another MOVED c.refreshing = false newm = c.mapping c.mu.Unlock() if c.LayoutRefresh != nil { c.LayoutRefresh(oldm, newm) } return nil } // reset the refreshing flag c.mu.Lock() c.refreshing = false c.mu.Unlock() msg := "redisc: all nodes failed\n" msg += strings.Join(errMsgs, "\n") err := errors.New(msg) if bg && c.BgError != nil { // in bg mode, this is already called in a distinct goroutine, so do not // call BgError in a distinct one. c.BgError(ClusterRefresh, err) } return err } // needsRefresh handles automatic update of the mapping, either because no node // was found for the slot, or because a MOVED error was received. func (c *Cluster) needsRefresh(re *RedirError) { c.mu.Lock() if re != nil { // update the mapping only if the address has changed, so that if a // READONLY replica read returns a MOVED to a master, it doesn't overwrite // that slot's replicas by setting just the master (i.e. this is not a // MOVED because the cluster is updating, it is a MOVED because the replica // cannot serve that key). Same goes for a request to a random connection // that gets a MOVED, should not overwrite the moved-to slot's // configuration if the master's address is the same. if current := c.mapping[re.NewSlot]; len(current) == 0 || current[0] != re.Addr { c.mapping[re.NewSlot] = []string{re.Addr} } else { // no refresh needed, the mapping already points to this address c.mu.Unlock() return } } if !c.refreshing { // refreshing is reset only once the goroutine has finished updating the // mapping, so a new refresh goroutine will only be started if none is // running. c.refreshing = true go c.refresh(true) //nolint:errcheck } c.mu.Unlock() } type slotMapping struct { start, end int nodes []string // master is always at [0] } func (c *Cluster) getClusterSlots(addr string) ([]slotMapping, error) { conn, err := c.getConnForAddr(addr, false) if err != nil { return nil, err } defer conn.Close() vals, err := redis.Values(conn.Do("CLUSTER", "SLOTS")) if err != nil { return nil, err } m := make([]slotMapping, 0, len(vals)) for len(vals) > 0 { var slotRange []interface{} vals, err = redis.Scan(vals, &slotRange) if err != nil { return nil, err } var start, end int slotRange, err = redis.Scan(slotRange, &start, &end) if err != nil { return nil, err } sm := slotMapping{start: start, end: end} // store the master address and all replicas for len(slotRange) > 0 { var nodes []interface{} slotRange, err = redis.Scan(slotRange, &nodes) if err != nil { return nil, err } var addr string var port int if _, err = redis.Scan(nodes, &addr, &port); err != nil { return nil, err } sm.nodes = append(sm.nodes, addr+":"+strconv.Itoa(port)) } m = append(m, sm) } return m, nil } func (c *Cluster) getConnForAddr(addr string, forceDial bool) (redis.Conn, error) { c.mu.Lock() if err := c.err; err != nil { c.mu.Unlock() return nil, err } if c.CreatePool == nil || forceDial { c.mu.Unlock() return redis.Dial("tcp", addr, c.DialOptions...) } p := c.pools[addr] if p == nil { c.mu.Unlock() pool, err := c.CreatePool(addr, c.DialOptions...) if err != nil { return nil, err } c.mu.Lock() // check again, concurrent request may have set the pool in the meantime if p = c.pools[addr]; p == nil { if c.pools == nil { c.pools = make(map[string]*redis.Pool, len(c.StartupNodes)) } c.pools[addr] = pool p = pool } else { // Don't assume CreatePool just returned the pool struct, it may have // used a connection or something - always match CreatePool with Close. // Do it in a defer to keep lock time short. Pool.Close always returns // nil. defer pool.Close() } } c.mu.Unlock() return c.getFromPool(p) } // get connection from the pool. // use GetContext if PoolWaitTime > 0 func (c *Cluster) getFromPool(p *redis.Pool) (redis.Conn, error) { if c.PoolWaitTime <= 0 { conn := p.Get() return conn, conn.Err() } ctx, cancel := context.WithTimeout(context.Background(), c.PoolWaitTime) defer cancel() return p.GetContext(ctx) } var errNoNodeForSlot = errors.New("redisc: no node for slot") func (c *Cluster) getConnForSlot(slot int, forceDial, readOnly bool) (redis.Conn, string, error) { c.mu.Lock() addrs := c.mapping[slot] c.mu.Unlock() if len(addrs) == 0 { return nil, "", errNoNodeForSlot } // mapping slices are never altered, they are replaced when refreshing // or on a MOVED response, so it's non-racy to read them outside the lock. addr := addrs[0] if readOnly && len(addrs) > 1 { // get the address of a replica if len(addrs) == 2 { addr = addrs[1] } else { rnd.Lock() ix := rnd.Intn(len(addrs) - 1) rnd.Unlock() addr = addrs[ix+1] // +1 because 0 is the master } } else { readOnly = false } conn, err := c.getConnForAddr(addr, forceDial) if err == nil && readOnly { _, _ = conn.Do("READONLY") } return conn, addr, err } // a *rand.Rand is not safe for concurrent access var rnd = struct { sync.Mutex *rand.Rand }{Rand: rand.New(rand.NewSource(time.Now().UnixNano()))} //nolint:gosec func (c *Cluster) getRandomConn(forceDial, readOnly bool) (redis.Conn, string, error) { addrs, _ := c.getNodeAddrs(readOnly) rnd.Lock() perms := rnd.Perm(len(addrs)) rnd.Unlock() var errMsgs []string //nolint:prealloc for _, ix := range perms { addr := addrs[ix] conn, err := c.getConnForAddr(addr, forceDial) if err == nil { if readOnly { _, _ = conn.Do("READONLY") } return conn, addr, nil } errMsgs = append(errMsgs, err.Error()) } msg := "redisc: failed to get a connection" if len(errMsgs) > 0 { msg += "\n" msg += strings.Join(errMsgs, "\n") } return nil, "", errors.New(msg) } func (c *Cluster) getConn(preferredSlot int, forceDial, readOnly bool) (conn redis.Conn, addr string, err error) { if preferredSlot >= 0 { conn, addr, err = c.getConnForSlot(preferredSlot, forceDial, readOnly) if err == errNoNodeForSlot { c.needsRefresh(nil) } } if preferredSlot < 0 || err != nil { conn, addr, err = c.getRandomConn(forceDial, readOnly) } return conn, addr, err } func (c *Cluster) getNodeAddrs(preferReplicas bool) (addrs []string, replicas bool) { c.mu.Lock() // populate nodes lazily, only once if c.masters == nil { c.masters = make(map[string]bool, len(c.StartupNodes)) c.replicas = make(map[string]bool) // StartupNodes should be masters for _, n := range c.StartupNodes { c.masters[n] = true } } from := c.masters if preferReplicas && len(c.replicas) > 0 { from = c.replicas replicas = true } // grab a slice of addresses addrs = make([]string, 0, len(from)) for addr := range from { addrs = append(addrs, addr) } c.mu.Unlock() return addrs, replicas } // Dial returns a connection the same way as Get, but it guarantees that the // connection will not be managed by the pool, even if CreatePool is set. The // actual returned type is *Conn, see its documentation for details. func (c *Cluster) Dial() (redis.Conn, error) { c.mu.Lock() err := c.err c.mu.Unlock() if err != nil { return nil, err } return &Conn{ cluster: c, forceDial: true, }, nil } // Get returns a redis.Conn interface that can be used to call redis commands // on the cluster. The application must close the returned connection. The // actual returned type is *Conn, see its documentation for details. func (c *Cluster) Get() redis.Conn { c.mu.Lock() err := c.err c.mu.Unlock() return &Conn{ cluster: c, err: err, } } // EachNode calls fn for each node in the cluster, with a connection bound to // that node. The connection is automatically closed (and potentially returned // to the pool if Cluster.CreatePool is set) after the function executes. Note // that conn is not a RetryConn and using one is inappropriate, as the goal of // EachNode is to connect to specific nodes, not to target specific keys. The // visited nodes are those that are known at the time of the call - it does not // force a refresh of the cluster layout. If no nodes are known, it returns an // error. // // If fn returns an error, no more nodes are visited and that error is returned // by EachNode. If replicas is true, it will visit each replica node instead, // otherwise the primary nodes are visited. Keep in mind that if replicas is // true, it will visit all known replicas - which is great e.g. to run // diagnostics on each node, but can be surprising if the goal is e.g. to // collect all keys, as it is possible that more than one node is acting as // replica for the same primary, meaning that the same keys could be seen // multiple times - you should be prepared to handle this scenario. The // connection provided to fn is not a ReadOnly connection (conn.ReadOnly hasn't // been called on it), it is up to fn to execute the READONLY redis command if // required. func (c *Cluster) EachNode(replicas bool, fn func(addr string, conn redis.Conn) error) error { addrs, ok := c.getNodeAddrs(replicas) if len(addrs) == 0 || replicas && !ok { return errors.New("redisc: no known node address") } for _, addr := range addrs { conn, err := c.getConnForAddr(addr, false) cconn := &Conn{ cluster: c, boundAddr: addr, rc: conn, // in case of error, create a failed connection and still call fn, so // that it can decide whether or not to keep visiting nodes. err: err, } err = func() error { defer cconn.Close() return fn(addr, cconn) }() if err != nil { return err } } return nil } // Close releases the resources used by the cluster. It closes all the pools // that were created, if any. func (c *Cluster) Close() error { c.mu.Lock() err := c.err if err == nil { c.err = errors.New("redisc: closed") for _, p := range c.pools { if e := p.Close(); e != nil && err == nil { // note that Pool.Close always returns nil. err = e } } // keep c.pools around so that Stats can still be called after Close } c.mu.Unlock() return err } // Stats returns the current statistics for all pools. Keys are node's // addresses. func (c *Cluster) Stats() map[string]redis.PoolStats { c.mu.RLock() defer c.mu.RUnlock() stats := make(map[string]redis.PoolStats, len(c.pools)) for address, pool := range c.pools { stats[address] = pool.Stats() } return stats } mna-redisc-96a8e49/cluster_test.go000066400000000000000000000774761451450242600172060ustar00rootroot00000000000000package redisc import ( "context" "strings" "sync" "sync/atomic" "testing" "time" "github.com/gomodule/redigo/redis" "github.com/mna/redisc/redistest" "github.com/mna/redisc/redistest/resp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestStandaloneRedis(t *testing.T) { cmd, port := redistest.StartServer(t, nil, "") defer cmd.Process.Kill() //nolint:errcheck port = ":" + port t.Run("refresh", func(t *testing.T) { c := &Cluster{ StartupNodes: []string{port}, } err := c.Refresh() if assert.Error(t, err, "Refresh") { assert.Contains(t, err.Error(), "redisc: all nodes failed", "expected redisc error message") assert.Contains(t, err.Error(), "cluster support disabled", "expected redis error message") } }) } func TestClusterRedis(t *testing.T) { fn, ports := redistest.StartCluster(t, nil) defer fn() for i, p := range ports { ports[i] = ":" + p } t.Run("refresh", func(t *testing.T) { testClusterRefresh(t, ports) }) t.Run("needs refresh", func(t *testing.T) { testClusterNeedsRefresh(t, ports) }) t.Run("close", func(t *testing.T) { testClusterClose(t, ports) }) t.Run("closed refresh", func(t *testing.T) { testClusterClosedRefresh(t, ports) }) t.Run("conn readonly no replica", func(t *testing.T) { testConnReadOnlyNoReplica(t, ports) }) t.Run("conn bind", func(t *testing.T) { testConnBind(t, ports) }) t.Run("conn blank do", func(t *testing.T) { testConnBlankDo(t, ports) }) t.Run("conn with timeout", func(t *testing.T) { testConnWithTimeout(t, ports) }) t.Run("retry conn too many attempts", func(t *testing.T) { testRetryConnTooManyAttempts(t, ports) }) t.Run("retry conn moved", func(t *testing.T) { testRetryConnMoved(t, ports) }) t.Run("each node none", func(t *testing.T) { testEachNodeNone(t, ports) }) t.Run("each node some", func(t *testing.T) { testEachNodeSome(t, ports) }) t.Run("retry conn trigger refresh", func(t *testing.T) { testRetryConnTriggerRefreshes(t, ports) }) } func TestClusterRedisWithReplica(t *testing.T) { fn, ports := redistest.StartClusterWithReplicas(t, nil) defer fn() for i, p := range ports { ports[i] = ":" + p } t.Run("refresh startup nodes a replica", func(t *testing.T) { testClusterRefreshStartWithReplica(t, ports) }) t.Run("conn readonly", func(t *testing.T) { testConnReadOnlyWithReplicas(t, ports) }) t.Run("each node some", func(t *testing.T) { testEachNodeSomeWithReplica(t, ports) }) t.Run("each node scan keys", func(t *testing.T) { testEachNodeScanKeysWithReplica(t, ports) }) t.Run("layout refresh", func(t *testing.T) { testLayoutRefreshWithReplica(t, ports) }) t.Run("layout moved", func(t *testing.T) { testLayoutMovedWithReplica(t, ports) }) } func assertMapping(t *testing.T, mapping [HashSlots][]string, masterPorts, replicaPorts []string) { expectedMappingNodes := 1 // at least a master node if len(replicaPorts) > 0 { // if there are replicas, then we expected 2 mapping nodes (master+replica) expectedMappingNodes = 2 } for _, maps := range mapping { if assert.Equal(t, expectedMappingNodes, len(maps), "Mapping has %d node(s)", expectedMappingNodes) { if assert.NotEmpty(t, maps[0]) { split := strings.Index(maps[0], ":") assert.Contains(t, masterPorts, maps[0][split:], "expected master") } if len(maps) > 1 && assert.NotEmpty(t, maps[1]) { split := strings.Index(maps[1], ":") assert.Contains(t, replicaPorts, maps[1][split:], "expected replica") } } } } func testEachNodeNone(t *testing.T, _ []string) { c := &Cluster{} defer c.Close() // no known node var count int err := c.EachNode(false, func(addr string, conn redis.Conn) error { count++ return nil }) if assert.Error(t, err) { assert.Contains(t, err.Error(), "no known node") } assert.Equal(t, 0, count) // no known replica count = 0 err = c.EachNode(true, func(addr string, conn redis.Conn) error { count++ return nil }) if assert.Error(t, err) { assert.Contains(t, err.Error(), "no known node") } assert.Equal(t, 0, count) } func assertNodeIdentity(t *testing.T, conn redis.Conn, gotAddr, wantPort, wantRole string) { assertNodeIdentityIn(t, conn, gotAddr, wantRole, map[string]bool{wantPort: true}) } func assertNodeIdentityIn(t *testing.T, conn redis.Conn, gotAddr, wantRole string, portIn map[string]bool) { var foundPort string for port := range portIn { if strings.HasSuffix(gotAddr, port) { foundPort = port delete(portIn, port) } } assert.NotEmpty(t, foundPort, "address not in %#v", portIn) vs, err := redis.Values(conn.Do("ROLE")) require.NoError(t, err) var role string _, err = redis.Scan(vs, &role) require.NoError(t, err) assert.Equal(t, wantRole, role) info, err := redis.String(conn.Do("INFO", "server")) require.NoError(t, err) assert.Contains(t, info, "tcp_port"+foundPort) } func testEachNodeSome(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: []string{ports[0]}, } defer c.Close() // only the single startup node at the moment var count int err := c.EachNode(false, func(addr string, conn redis.Conn) error { count++ assertNodeIdentity(t, conn, addr, ports[0], "master") return nil }) assert.NoError(t, err) assert.Equal(t, 1, count) // no known replica count = 0 err = c.EachNode(true, func(addr string, conn redis.Conn) error { count++ return nil }) if assert.Error(t, err) { assert.Contains(t, err.Error(), "no known node") } assert.Equal(t, 0, count) require.NoError(t, c.Refresh()) portsIn := make(map[string]bool, len(ports)) for _, port := range ports { portsIn[port] = true } count = 0 err = c.EachNode(false, func(addr string, conn redis.Conn) error { count++ assertNodeIdentityIn(t, conn, addr, "master", portsIn) return nil }) assert.NoError(t, err) assert.Equal(t, len(ports), count) // no known replica count = 0 err = c.EachNode(true, func(addr string, conn redis.Conn) error { count++ return nil }) if assert.Error(t, err) { assert.Contains(t, err.Error(), "no known node") } assert.Equal(t, 0, count) } func testEachNodeSomeWithReplica(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: []string{ports[0]}, } defer c.Close() // only the single startup node at the moment var count int err := c.EachNode(false, func(addr string, conn redis.Conn) error { count++ assertNodeIdentity(t, conn, addr, ports[0], "master") return nil }) assert.NoError(t, err) assert.Equal(t, 1, count) // no known replica count = 0 err = c.EachNode(true, func(addr string, conn redis.Conn) error { count++ return nil }) if assert.Error(t, err) { assert.Contains(t, err.Error(), "no known node") } assert.Equal(t, 0, count) require.NoError(t, c.Refresh()) // visit each primary primaries, replicas := ports[:redistest.NumClusterNodes], ports[redistest.NumClusterNodes:] portsIn := make(map[string]bool, len(primaries)) for _, port := range primaries { portsIn[port] = true } count = 0 err = c.EachNode(false, func(addr string, conn redis.Conn) error { count++ assertNodeIdentityIn(t, conn, addr, "master", portsIn) return nil }) assert.NoError(t, err) assert.Equal(t, len(primaries), count) // visit each replica portsIn = make(map[string]bool, len(replicas)) for _, port := range replicas { portsIn[port] = true } count = 0 err = c.EachNode(true, func(addr string, conn redis.Conn) error { count++ assertNodeIdentityIn(t, conn, addr, "slave", portsIn) return nil }) assert.NoError(t, err) assert.Equal(t, len(replicas), count) } func testEachNodeScanKeysWithReplica(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: []string{"127.0.0.1" + ports[0]}, CreatePool: createPool, } defer c.Close() require.NoError(t, c.Refresh()) conn := c.Get() conn, _ = RetryConn(conn, 3, 100*time.Millisecond) defer conn.Close() const prefix = "eachnode:" keys := []string{"a", "b", "c", "d", "e"} for i, k := range keys { k = prefix + "{" + k + "}" keys[i] = k _, err := conn.Do("SET", k, i) require.NoError(t, err) } conn.Close() // close it now so it does not show up as in use in stats // collect from primaries var gotKeys []string err := c.EachNode(false, func(addr string, conn redis.Conn) error { var cursor int for { var keyList []string vs, err := redis.Values(conn.Do("SCAN", cursor, "MATCH", prefix+"*")) require.NoError(t, err) _, err = redis.Scan(vs, &cursor, &keyList) require.NoError(t, err) gotKeys = append(gotKeys, keyList...) if cursor == 0 { return nil } } }) require.NoError(t, err) assert.ElementsMatch(t, keys, gotKeys) // collect from replicas gotKeys = nil err = c.EachNode(true, func(addr string, conn redis.Conn) error { var cursor int for { var keyList []string vs, err := redis.Values(conn.Do("SCAN", cursor, "MATCH", prefix+"*")) require.NoError(t, err) _, err = redis.Scan(vs, &cursor, &keyList) require.NoError(t, err) gotKeys = append(gotKeys, keyList...) if cursor == 0 { return nil } } }) require.NoError(t, err) assert.ElementsMatch(t, keys, gotKeys) var inuse, idle int stats := c.Stats() for _, st := range stats { inuse += st.ActiveCount - st.IdleCount idle += st.IdleCount } assert.Equal(t, 0, inuse) // all connections were closed/returned to the pool assert.Equal(t, len(ports), idle) // one for each node, primary + replica } func testLayoutRefreshWithReplica(t *testing.T, ports []string) { var count int c := &Cluster{ StartupNodes: []string{ports[0]}, LayoutRefresh: func(old, new [HashSlots][]string) { for slot, maps := range old { assert.Len(t, maps, 0, "slot %d", slot) } for slot, maps := range new { assert.Len(t, maps, 2, "slot %d", slot) } count++ }, } defer c.Close() // LayoutRefresh is called synchronously when Refresh call is explicit require.NoError(t, c.Refresh()) require.Equal(t, count, 1) } func testLayoutMovedWithReplica(t *testing.T, ports []string) { var count int64 done := make(chan bool, 1) c := &Cluster{ StartupNodes: []string{ports[0]}, LayoutRefresh: func(old, new [HashSlots][]string) { for slot, maps := range old { if slot == 15495 { // slot of key "a" assert.Len(t, maps, 1, "slot %d", slot) continue } assert.Len(t, maps, 0, "slot %d", slot) } for slot, maps := range new { assert.Len(t, maps, 2, "slot %d", slot) } atomic.AddInt64(&count, 1) done <- true }, } defer c.Close() conn := c.Get() defer conn.Close() // to trigger this properly, first do EachNode (which only knows about the // current node, which serves the bottom tier slots), and request key "a" // which hashes to a high slot. This will result in a MOVED error that will // update the single mapping and trigger a full refresh. var eachCalls int _ = c.EachNode(false, func(_ string, conn redis.Conn) error { eachCalls++ _, err := conn.Do("GET", "a") if assert.Error(t, err) { assert.Contains(t, err.Error(), "MOVED") } return nil }) assert.Equal(t, 1, eachCalls) // LayoutRefresh call might not have completed yet, so wait for the channel // receive, or fail after a second. waitForClusterRefresh(c, nil) select { case <-time.After(time.Second): require.Fail(t, "LayoutRefresh call not done after timeout") case <-done: count := atomic.LoadInt64(&count) require.Equal(t, int(count), 1) } } func testClusterRefresh(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: []string{ports[0]}, } defer c.Close() err := c.Refresh() if assert.NoError(t, err, "Refresh") { assertMapping(t, c.mapping, ports, nil) } } func testClusterRefreshStartWithReplica(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: []string{ports[len(ports)-1]}, // last port is a replica } defer c.Close() err := c.Refresh() if assert.NoError(t, err, "Refresh") { assertMapping(t, c.mapping, ports[:redistest.NumClusterNodes], ports[redistest.NumClusterNodes:]) } } func TestClusterRefreshAllFail(t *testing.T) { s := redistest.StartMockServer(t, func(cmd string, args ...string) interface{} { return resp.Error("nope") }) defer s.Close() c := &Cluster{ StartupNodes: []string{s.Addr}, } defer c.Close() if err := c.Refresh(); assert.Error(t, err, "Refresh") { assert.Contains(t, err.Error(), "all nodes failed", "expected message") assert.Contains(t, err.Error(), "nope", "expected server message") } require.NoError(t, c.Close(), "Close") } func TestClusterNoNode(t *testing.T) { c := &Cluster{} defer c.Close() conn := c.Get() _, err := conn.Do("A") if assert.Error(t, err, "Do") { assert.Contains(t, err.Error(), "failed to get a connection", "expected message") } if err := BindConn(conn); assert.Error(t, err, "Bind without key") { assert.Contains(t, err.Error(), "failed to get a connection", "expected message") } if err := BindConn(conn, "A"); assert.Error(t, err, "Bind with key") { assert.Contains(t, err.Error(), "failed to get a connection", "expected message") } } func testClusterNeedsRefresh(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: ports, } defer c.Close() conn := c.Get().(*Conn) defer conn.Close() // at this point, no mapping is stored c.mu.Lock() for i, v := range c.mapping { if !assert.Empty(t, v, "No addr for %d", i) { break } } c.mu.Unlock() // calling Do may or may not generate a MOVED error (it will get a // random node, because no mapping is known yet) _, _ = conn.Do("GET", "b") waitForClusterRefresh(c, func() { for i, v := range c.mapping { if !assert.NotEmpty(t, v, "Addr for %d", i) { break } } }) } func testClusterClose(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: []string{ports[0]}, DialOptions: []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)}, CreatePool: createPool, } defer c.Close() require.NoError(t, c.Refresh()) // get some connections before closing connUnbound := c.Get() defer connUnbound.Close() connBound := c.Get() defer connBound.Close() _ = BindConn(connBound, "b") connRetry := c.Get() defer connRetry.Close() connRetry, _ = RetryConn(connRetry, 3, time.Millisecond) // close the cluster and check that all API works as expected assert.NoError(t, c.Close(), "Close") if err := c.Close(); assert.Error(t, err, "Close after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if conn := c.Get(); assert.Error(t, conn.Err(), "Get after Close") { assert.Contains(t, conn.Err().Error(), "redisc: closed", "expected message") } if _, err := c.Dial(); assert.Error(t, err, "Dial after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if err := c.Refresh(); assert.Error(t, err, "Refresh after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if err := c.EachNode(false, func(addr string, c redis.Conn) error { return c.Err() }); assert.Error(t, err, "EachNode after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if _, err := connUnbound.Do("SET", "a", 1); assert.Error(t, err, "unbound connection Do") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } // connection was bound pre-cluster-close, so it already has a valid connection if _, err := connBound.Do("SET", "b", 1); assert.NoError(t, err, "bound connection Do") { err = connBound.Close() assert.NoError(t, err) } if _, err := connRetry.Do("GET", "a"); assert.Error(t, err, "retry connection Do") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } // Stats still works after Close stats := c.Stats() assert.True(t, len(stats) > 0) } func testClusterClosedRefresh(t *testing.T, ports []string) { var clusterRefreshCount int64 var clusterRefreshErr atomic.Value done := make(chan bool) c := &Cluster{ StartupNodes: []string{ports[0]}, DialOptions: []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)}, CreatePool: createPool, BgError: func(src BgErrorSrc, err error) { if src == ClusterRefresh { atomic.AddInt64(&clusterRefreshCount, 1) clusterRefreshErr.Store(err) done <- true } }, } defer c.Close() conn := c.Get() defer conn.Close() // close the cluster and check that all API works as expected assert.NoError(t, c.Close(), "Close") if _, err := conn.Do("SET", "a", 1); assert.Error(t, err, "connection Do") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } waitForClusterRefresh(c, nil) // BgError call might not have completed yet, so wait for the channel // receive, or fail after a second. select { case <-time.After(time.Second): require.Fail(t, "BgError call not done after timeout") case <-done: count := atomic.LoadInt64(&clusterRefreshCount) require.Equal(t, int(count), 1) if err := clusterRefreshErr.Load().(error); assert.Error(t, err, "refresh error") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } } } // TestGetPoolTimedOut test case where we can't get the connection because the pool // is full func TestGetPoolTimedOut(t *testing.T) { s := redistest.StartMockServer(t, func(cmd string, args ...string) interface{} { return nil }) defer s.Close() p := &redis.Pool{ MaxActive: 1, Dial: func() (redis.Conn, error) { return redis.Dial("tcp", s.Addr) }, Wait: true, } c := Cluster{ PoolWaitTime: 100 * time.Millisecond, } defer c.Close() conn, err := c.getFromPool(p) if assert.NoError(t, err) { defer conn.Close() } // second connection should be failed because we only have 1 MaxActive start := time.Now() _, err = c.getFromPool(p) if assert.Error(t, err) { assert.Equal(t, context.DeadlineExceeded, err) assert.True(t, time.Since(start) >= 100*time.Millisecond) } } // TestGetPoolWaitOnFull test that we could get the connection when the pool // is full and we can wait for it func TestGetPoolWaitOnFull(t *testing.T) { s := redistest.StartMockServer(t, func(cmd string, args ...string) interface{} { return nil }) defer s.Close() var ( usageTime = 100 * time.Millisecond // how long the connection will be used waitTime = 3 * usageTime // how long we want to wait ) p := &redis.Pool{ MaxActive: 1, Dial: func() (redis.Conn, error) { return redis.Dial("tcp", s.Addr) }, Wait: true, } c := Cluster{ PoolWaitTime: waitTime, } defer c.Close() // first connection OK conn, err := c.getFromPool(p) assert.NoError(t, err) // second connection should be failed because we only have 1 MaxActive start := time.Now() _, err = c.getFromPool(p) if assert.Error(t, err) { assert.Equal(t, context.DeadlineExceeded, err) assert.True(t, time.Since(start) >= waitTime) } go func() { time.Sleep(usageTime) // sleep before close, to simulate waiting for connection conn.Close() }() start = time.Now() conn2, err := c.getFromPool(p) if assert.NoError(t, err) { assert.True(t, time.Since(start) >= usageTime) } conn2.Close() } func createPool(addr string, opts ...redis.DialOption) (*redis.Pool, error) { return &redis.Pool{ MaxIdle: 5, MaxActive: 10, IdleTimeout: time.Minute, Dial: func() (redis.Conn, error) { return redis.Dial("tcp", addr, opts...) }, TestOnBorrow: func(c redis.Conn, t time.Time) error { _, err := c.Do("PING") return err }, }, nil } // waits for a running Cluster.refresh call to complete before calling fn. // Note that fn is called while the Cluster's lock is held - to just wait // for refresh to complete and continue without holding the lock, simply // pass nil as fn - the lock is released before this call returns. func waitForClusterRefresh(cluster *Cluster, fn func()) { // wait for refreshing to become false again cluster.mu.Lock() for cluster.refreshing { cluster.mu.Unlock() time.Sleep(100 * time.Millisecond) cluster.mu.Lock() } if fn != nil { fn() } cluster.mu.Unlock() } type redisCmd struct { name string args redis.Args resp interface{} // if resp is of type lenResult, asserts that there is a result at least this long errMsg string } type lenResult int func TestCommands(t *testing.T) { cmdsPerGroup := map[string][]redisCmd{ "cluster": { {"CLUSTER", redis.Args{"INFO"}, lenResult(10), ""}, {"READONLY", nil, "OK", ""}, {"READWRITE", nil, "OK", ""}, {"CLUSTER", redis.Args{"COUNTKEYSINSLOT", 12345}, int64(0), ""}, {"CLUSTER", redis.Args{"KEYSLOT", "a"}, int64(15495), ""}, {"CLUSTER", redis.Args{"NODES"}, lenResult(100), ""}, }, "connection": { {"AUTH", redis.Args{"pwd"}, nil, "AUTH"}, {"ECHO", redis.Args{"a"}, []byte("a"), ""}, {"PING", nil, "PONG", ""}, {"SELECT", redis.Args{1}, nil, "ERR SELECT is not allowed in cluster mode"}, {"QUIT", nil, "OK", ""}, }, "hashes": { {"HSET", redis.Args{"ha", "f1", "1"}, int64(1), ""}, {"HLEN", redis.Args{"ha"}, int64(1), ""}, {"HEXISTS", redis.Args{"ha", "f1"}, int64(1), ""}, {"HDEL", redis.Args{"ha", "f1", "f2"}, int64(1), ""}, {"HINCRBY", redis.Args{"hb", "f1", "1"}, int64(1), ""}, {"HINCRBYFLOAT", redis.Args{"hb", "f2", "0.5"}, []byte("0.5"), ""}, {"HKEYS", redis.Args{"hb"}, []interface{}{[]byte("f1"), []byte("f2")}, ""}, {"HMGET", redis.Args{"hb", "f1", "f2"}, []interface{}{[]byte("1"), []byte("0.5")}, ""}, {"HMSET", redis.Args{"hc", "f1", "a", "f2", "b"}, "OK", ""}, {"HSET", redis.Args{"ha", "f1", "2"}, int64(1), ""}, {"HGET", redis.Args{"ha", "f1"}, []byte("2"), ""}, {"HGETALL", redis.Args{"ha"}, []interface{}{[]byte("f1"), []byte("2")}, ""}, {"HSETNX", redis.Args{"ha", "f2", "3"}, int64(1), ""}, //{"HSTRLEN", redis.Args{"hb", "f2"}, int64(3), ""}, // redis 3.2 only {"HVALS", redis.Args{"hb"}, []interface{}{[]byte("1"), []byte("0.5")}, ""}, {"HSCAN", redis.Args{"hb", 0}, lenResult(2), ""}, }, "hyperloglog": { {"PFADD", redis.Args{"hll", "a", "b", "c"}, int64(1), ""}, {"PFCOUNT", redis.Args{"hll"}, int64(3), ""}, {"PFADD", redis.Args{"hll2", "d"}, int64(1), ""}, {"PFMERGE", redis.Args{"hll", "hll2"}, nil, "CROSSSLOT"}, }, "keys": { // connection will bind to the node that serves slot of "k1" {"SET", redis.Args{"k1", "z"}, "OK", ""}, {"EXISTS", redis.Args{"k1"}, int64(1), ""}, {"DUMP", redis.Args{"k1"}, lenResult(10), ""}, {"EXPIRE", redis.Args{"k1", 10}, int64(1), ""}, {"EXPIREAT", redis.Args{"k1", time.Now().Add(time.Hour).Unix()}, int64(1), ""}, {"KEYS", redis.Args{"z*"}, []interface{}{}, ""}, // KEYS is supported, but uses a random node and returns keys from that node (undeterministic) {"MOVE", redis.Args{"k1", 2}, nil, "ERR MOVE is not allowed in cluster mode"}, {"PERSIST", redis.Args{"k1"}, int64(1), ""}, {"PEXPIRE", redis.Args{"k1", 10000}, int64(1), ""}, {"PEXPIREAT", redis.Args{"k1", time.Now().Add(time.Hour).UnixNano() / int64(time.Millisecond)}, int64(1), ""}, {"PTTL", redis.Args{"k1"}, lenResult(3500000), ""}, // RANDOMKEY is not deterministic {"RENAME", redis.Args{"k1", "k2"}, nil, "CROSSSLOT"}, {"RENAMENX", redis.Args{"k1", "k2"}, nil, "CROSSSLOT"}, {"SCAN", redis.Args{0}, lenResult(2), ""}, // works, but only for the keys on that random node {"TTL", redis.Args{"k1"}, lenResult(3000), ""}, {"TYPE", redis.Args{"k1"}, "string", ""}, {"DEL", redis.Args{"k1"}, int64(1), ""}, {"SADD", redis.Args{"k1", "a", "z", "d"}, int64(3), ""}, {"SORT", redis.Args{"k1", "ALPHA"}, []interface{}{[]byte("a"), []byte("d"), []byte("z")}, ""}, {"DEL", redis.Args{"a", "b"}, nil, "CROSSSLOT"}, }, "lists": { {"LPUSH", redis.Args{"l1", "a", "b", "c"}, int64(3), ""}, {"LINDEX", redis.Args{"l1", 1}, []byte("b"), ""}, {"LINSERT", redis.Args{"l1", "BEFORE", "b", "d"}, int64(4), ""}, {"LLEN", redis.Args{"l1"}, int64(4), ""}, {"LPOP", redis.Args{"l1"}, []byte("c"), ""}, {"LPUSHX", redis.Args{"l1", "e"}, int64(4), ""}, {"LRANGE", redis.Args{"l1", 0, 1}, []interface{}{[]byte("e"), []byte("d")}, ""}, {"LREM", redis.Args{"l1", 0, "d"}, int64(1), ""}, {"LSET", redis.Args{"l1", 0, "f"}, "OK", ""}, {"LTRIM", redis.Args{"l1", 0, 3}, "OK", ""}, {"RPOP", redis.Args{"l1"}, []byte("a"), ""}, {"RPOPLPUSH", redis.Args{"l1", "l2"}, nil, "CROSSSLOT"}, {"RPUSH", redis.Args{"l1", "g"}, int64(3), ""}, {"RPUSH", redis.Args{"l1", "h"}, int64(4), ""}, {"BLPOP", redis.Args{"l1", 1}, lenResult(2), ""}, {"BRPOP", redis.Args{"l1", 1}, lenResult(2), ""}, {"BRPOPLPUSH", redis.Args{"l1", "l2", 1}, nil, "CROSSSLOT"}, }, "pubsub": { {"PUBSUB", redis.Args{"NUMPAT"}, lenResult(0), ""}, {"PUBLISH", redis.Args{"ev1", "a"}, lenResult(0), ""}, // to actually subscribe to events, only Send must be called, and Receive to listen (or redis.PubSubConn must be used) }, "scripting": { {"SCRIPT", redis.Args{"FLUSH"}, "OK", ""}, {"SCRIPT", redis.Args{"EXISTS", "return GET x"}, []interface{}{int64(0)}, ""}, // to actually use scripts with keys, conn.Bind must be called to select the right node }, "server": { {"CLIENT", redis.Args{"LIST"}, lenResult(10), ""}, {"COMMAND", nil, lenResult(50), ""}, {"INFO", nil, lenResult(100), ""}, {"TIME", nil, lenResult(2), ""}, }, "sets": { {"SADD", redis.Args{"t1", "a", "b"}, int64(2), ""}, {"SADD", redis.Args{"{t1}.b", "c", "b"}, int64(2), ""}, {"SCARD", redis.Args{"t1"}, int64(2), ""}, {"SDIFF", redis.Args{"t1", "t2"}, nil, "CROSSSLOT"}, {"SDIFFSTORE", redis.Args{"{t1}.3", "t1", "{t1}.2"}, int64(2), ""}, {"SINTER", redis.Args{"t1", "{t1}.b"}, []interface{}{[]byte("b")}, ""}, {"SINTERSTORE", redis.Args{"{t1}.c", "t1", "{t1}.b"}, int64(1), ""}, {"SISMEMBER", redis.Args{"t1", "a"}, int64(1), ""}, {"SMEMBERS", redis.Args{"t1"}, lenResult(2), ""}, // order is not deterministic {"SMOVE", redis.Args{"t1", "{t1}.c", "a"}, int64(1), ""}, {"SPOP", redis.Args{"t3{t1}"}, nil, ""}, {"SRANDMEMBER", redis.Args{"t3{t1}"}, nil, ""}, {"SREM", redis.Args{"t1", "b"}, int64(1), ""}, {"SSCAN", redis.Args{"{t1}.b", 0}, lenResult(2), ""}, {"SUNION", redis.Args{"{t1}.b", "{t1}.c"}, lenResult(3), ""}, {"SUNIONSTORE", redis.Args{"{t1}.d", "{t1}.b", "{t1}.c"}, int64(3), ""}, }, "sortedsets": { {"ZADD", redis.Args{"z1", 1, "m1", 2, "m2", 3, "m3"}, int64(3), ""}, {"ZCARD", redis.Args{"z1"}, int64(3), ""}, {"ZCOUNT", redis.Args{"z1", "(1", "3"}, int64(2), ""}, {"ZINCRBY", redis.Args{"z1", 1, "m1"}, []byte("2"), ""}, {"ZINTERSTORE", redis.Args{"z2", 1, "z1"}, nil, "CROSSSLOT"}, {"ZLEXCOUNT", redis.Args{"z1", "[m1", "[m2"}, int64(2), ""}, {"ZRANGE", redis.Args{"z1", 0, 0}, []interface{}{[]byte("m1")}, ""}, {"ZRANGEBYLEX", redis.Args{"z1", "[m1", "(m2"}, []interface{}{[]byte("m1")}, ""}, {"ZRANGEBYSCORE", redis.Args{"z1", "(2", "3"}, []interface{}{[]byte("m3")}, ""}, {"ZRANK", redis.Args{"z1", "m3"}, int64(2), ""}, {"ZREM", redis.Args{"z1", "m1"}, int64(1), ""}, // TODO : complete commands... {"ZSCORE", redis.Args{"z1", "m3"}, []byte("3"), ""}, }, "strings": { {"APPEND", redis.Args{"s1", "a"}, int64(1), ""}, {"BITCOUNT", redis.Args{"s1"}, int64(3), ""}, {"GET", redis.Args{"s1"}, []byte("a"), ""}, {"MSET", redis.Args{"s2", "b", "s3", "c"}, "", "CROSSSLOT"}, {"SET", redis.Args{"s{b}", "b"}, "OK", ""}, {"SET", redis.Args{"s{bcd}", "c"}, "OK", ""}, // keys "b" (3300) and "bcd" (1872) are both in a hash slot < 5000, so on same node for this test // yet it still fails with CROSSSLOT (i.e. redis does not accept multi-key commands that don't // strictly hash to the same slot, regardless of which host serves them). {"MGET", redis.Args{"s{b}", "s{bcd}"}, "", "CROSSSLOT"}, }, "transactions": { {"DISCARD", nil, "", "ERR DISCARD without MULTI"}, {"EXEC", nil, "", "ERR EXEC without MULTI"}, {"MULTI", nil, "OK", ""}, {"SET", redis.Args{"tr1", 1}, "OK", ""}, {"WATCH", redis.Args{"tr1"}, "OK", ""}, {"UNWATCH", nil, "OK", ""}, // to actually use transactions, conn.Bind must be called to select the right node }, } fn, ports := redistest.StartCluster(t, nil) defer fn() for i, p := range ports { ports[i] = ":" + p } c := &Cluster{ StartupNodes: ports, DialOptions: []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)}, CreatePool: createPool, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") var wg sync.WaitGroup // start a goroutine that subscribes and listens to events ok := make(chan int) done := make(chan int) go runPubSubCommands(t, c, ok, done) <-ok wg.Add(len(cmdsPerGroup)) for _, cmds := range cmdsPerGroup { go runCommands(t, c, cmds, &wg) } wg.Add(2) go runScriptCommands(t, c, &wg) go runTransactionsCommands(t, c, &wg) wg.Wait() close(done) <-ok assert.NoError(t, c.Close(), "Cluster Close") } func runTransactionsCommands(t *testing.T, c *Cluster, wg *sync.WaitGroup) { defer wg.Done() conn := c.Get() defer conn.Close() require.NoError(t, BindConn(conn, "tr{a}1", "tr{a}2"), "Bind") _, err := conn.Do("WATCH", "tr{a}1") assert.NoError(t, err, "WATCH") _, err = conn.Do("MULTI") assert.NoError(t, err, "MULTI") _, err = conn.Do("SET", "tr{a}1", "a") assert.NoError(t, err, "SET 1") _, err = conn.Do("SET", "tr{a}2", "b") assert.NoError(t, err, "SET 2") _, err = conn.Do("EXEC") assert.NoError(t, err, "EXEC") v, err := redis.Strings(conn.Do("MGET", "tr{a}1", "tr{a}2")) assert.NoError(t, err, "MGET") if assert.Equal(t, 2, len(v), "Number of MGET results") { assert.Equal(t, "a", v[0], "MGET[0]") assert.Equal(t, "b", v[1], "MGET[1]") } } func runPubSubCommands(t *testing.T, c *Cluster, steps, stop chan int) { conn, err := c.Dial() require.NoError(t, err, "Dial for PubSub") psc := redis.PubSubConn{Conn: conn} assert.NoError(t, psc.PSubscribe("ev*"), "PSubscribe") assert.NoError(t, psc.Subscribe("e1"), "Subscribe") // allow commands to start running steps <- 1 var received bool loop: for { select { case <-stop: break loop default: } v := psc.Receive() switch v := v.(type) { case redis.Message: if !assert.Equal(t, []byte("a"), v.Data, "Received value") { t.Logf("%T", v) } received = true break loop } } <-stop assert.NoError(t, psc.Unsubscribe("e1"), "Unsubscribe") assert.NoError(t, psc.PUnsubscribe("ev*"), "PUnsubscribe") assert.NoError(t, psc.Close(), "Close for PubSub") assert.True(t, received, "Did receive event") steps <- 1 } func runScriptCommands(t *testing.T, c *Cluster, wg *sync.WaitGroup) { defer wg.Done() var script = redis.NewScript(2, ` redis.call("SET", KEYS[1], ARGV[1]) redis.call("SET", KEYS[2], ARGV[2]) return 1 `) conn := c.Get() defer conn.Close() require.NoError(t, BindConn(conn, "scr{a}1", "src{a}2"), "Bind") // script.Do, send the whole script v, err := script.Do(conn, "scr{a}1", "scr{a}2", "x", "y") assert.NoError(t, err, "Do script") assert.Equal(t, int64(1), v, "Script result") // send only the hash, should work because the script is now loaded on this node assert.NoError(t, script.SendHash(conn, "scr{a}1", "scr{a}2", "x", "y"), "SendHash") assert.NoError(t, conn.Flush(), "Flush") v, err = conn.Receive() assert.NoError(t, err, "SendHash Receive") assert.Equal(t, int64(1), v, "SendHash Script result") // do with keys from different slots _, err = script.Do(conn, "scr{a}1", "scr{b}2", "x", "y") if assert.Error(t, err, "Do script invalid keys") { assert.Contains(t, err.Error(), "CROSSSLOT", "Do script invalid keys") } } func runCommands(t *testing.T, c *Cluster, cmds []redisCmd, wg *sync.WaitGroup) { defer wg.Done() for _, cmd := range cmds { conn := c.Get() res, err := conn.Do(cmd.name, cmd.args...) if cmd.errMsg != "" { if assert.Error(t, err, cmd.name) { assert.Contains(t, err.Error(), cmd.errMsg, cmd.name) } } else { assert.NoError(t, err, cmd.name) if lr, ok := cmd.resp.(lenResult); ok { switch res := res.(type) { case []byte: assert.True(t, len(res) >= int(lr), "result has at least %d bytes, has %d", lr, len(res)) case []interface{}: assert.True(t, len(res) >= int(lr), "result array has at least %d items, has %d", lr, len(res)) case int64: assert.True(t, res >= int64(lr), "result is at least %d, is %d", lr, res) default: t.Errorf("unexpected result type %T", res) } } else { if !assert.Equal(t, cmd.resp, res, cmd.name) { t.Logf("%T vs %T", cmd.resp, res) } } } require.NoError(t, conn.Close(), "Close") } } mna-redisc-96a8e49/conn.go000066400000000000000000000244321451450242600154030ustar00rootroot00000000000000package redisc import ( "errors" "fmt" "strconv" "strings" "sync" "time" "github.com/gomodule/redigo/redis" ) var _ redis.ConnWithTimeout = (*Conn)(nil) // Conn is a redis cluster connection. When returned by Get or Dial, it is not // yet bound to any node in the cluster. Only when a call to Do, Send, Receive // or Bind is made is a connection to a specific node established: // // - if Do or Send is called first, the command's first parameter is // assumed to be the key, and its slot is used to find the node // - if Receive is called first, or if Do or Send is called first but with // no parameter for the command (or no command), a random node is selected // in the cluster // - if Bind is called first, the node corresponding to the slot of the // specified key(s) is selected // // Because Get and Dial return a redis.Conn interface, a type assertion can be // used to call Bind or ReadOnly on this concrete Conn type: // // redisConn := cluster.Get() // if conn, ok := redisConn.(*redisc.Conn); ok { // if err := conn.Bind("my-key"); err != nil { // // handle error // } // } // // Alternatively, the package-level BindConn or ReadOnlyConn helper functions // may be used. type Conn struct { cluster *Cluster // immutable forceDial bool // immutable // redigo allows concurrent reader and writer (conn.Receive and // conn.Send/conn.Flush), a mutex is needed to protect concurrent accesses. mu sync.Mutex readOnly bool boundAddr string err error rc redis.Conn } // RedirError is a cluster redirection error. It indicates that the redis node // returned either a MOVED or an ASK error, as specified by the Type field. type RedirError struct { // Type indicates if the redirection is a MOVED or an ASK. Type string // NewSlot is the slot number of the redirection. NewSlot int // Addr is the node address to redirect to. Addr string raw string } // Error returns the error message of a RedirError. This is the message as // received from redis. func (e *RedirError) Error() string { return e.raw } func isRedisErr(err error, typ string) bool { re, ok := err.(redis.Error) if !ok { return false } parts := strings.Fields(re.Error()) return len(parts) > 0 && parts[0] == typ } // IsTryAgain returns true if the error is a redis cluster error of type // TRYAGAIN, meaning that the command is valid, but the cluster is in an // unstable state and it can't complete the request at the moment. func IsTryAgain(err error) bool { return isRedisErr(err, "TRYAGAIN") } // IsCrossSlot returns true if the error is a redis cluster error of type // CROSSSLOT, meaning that a command was sent with keys from different slots. func IsCrossSlot(err error) bool { return isRedisErr(err, "CROSSSLOT") } // ParseRedir parses err into a RedirError. If err is not a MOVED or ASK error // or if it is nil, it returns nil. func ParseRedir(err error) *RedirError { re, ok := err.(redis.Error) if !ok { return nil } parts := strings.Fields(re.Error()) if len(parts) != 3 || (parts[0] != "MOVED" && parts[0] != "ASK") { return nil } slot, err := strconv.Atoi(parts[1]) if err != nil { return nil } return &RedirError{ Type: parts[0], NewSlot: slot, Addr: parts[2], raw: re.Error(), } } // binds the connection to a specific node, the one holding the slot or a // random node if slot is -1, iff the connection is not broken and is not // already bound. It returns the redis conn, true if it successfully bound to // this slot, or any error. func (c *Conn) bind(slot int) (rc redis.Conn, ok bool, err error) { c.mu.Lock() rc, err = c.rc, c.err if err == nil { if rc == nil { conn, addr, err2 := c.cluster.getConn(slot, c.forceDial, c.readOnly) if err2 != nil { err = err2 } else { c.rc, rc = conn, conn c.boundAddr = addr ok = true } } } c.mu.Unlock() return rc, ok, err } func cmdSlot(_ string, args []interface{}) int { slot := -1 if len(args) > 0 { key := fmt.Sprintf("%s", args[0]) slot = Slot(key) } return slot } // BindConn is a convenience function that checks if c implements a Bind method // with the right signature such as the one for a *Conn, and calls that method. // If c doesn't implement that method, it returns an error. func BindConn(c redis.Conn, keys ...string) error { if cc, ok := c.(interface { Bind(...string) error }); ok { return cc.Bind(keys...) } return errors.New("redisc: no Bind method") } // Bind binds the connection to the cluster node corresponding to the slot of // the provided keys. If the keys don't belong to the same slot, an error is // returned and the connection is not bound. If the connection is already // bound, an error is returned. If no key is provided, it binds to a random // node. func (c *Conn) Bind(keys ...string) error { slot := -1 for _, k := range keys { ks := Slot(k) if slot != -1 && ks != slot { return errors.New("redisc: keys do not belong to the same slot") } slot = ks } _, ok, err := c.bind(slot) if err != nil { return err } if !ok { // was already bound return errors.New("redisc: connection already bound to a node") } return nil } // ReadOnlyConn is a convenience function that checks if c implements a // ReadOnly method with the right signature such as the one for a *Conn, and // calls that method. If c doesn't implement that method, it returns an error. func ReadOnlyConn(c redis.Conn) error { if cc, ok := c.(interface { ReadOnly() error }); ok { return cc.ReadOnly() } return errors.New("redisc: no ReadOnly method") } // ReadOnly marks the connection as read-only, meaning that when it is bound to // a cluster node, it will attempt to connect to a replica instead of the // master and will automatically emit a READONLY command so that the replica // agrees to serve read commands. Be aware that reading from a replica may // return stale data. Sending write commands on a read-only connection will // fail with a MOVED error. See http://redis.io/commands/readonly for more // details. // // If the connection is already bound to a node, either via a call to Do, Send, // Receive or Bind, ReadOnly returns an error. func (c *Conn) ReadOnly() error { c.mu.Lock() defer c.mu.Unlock() if c.err != nil { return c.err } if c.rc != nil { // was already bound return errors.New("redisc: connection already bound to a node") } c.readOnly = true return nil } // Do sends a command to the server and returns the received reply. If the // connection is not yet bound to a cluster node, it will be after this call, // based on the rules documented in the Conn type. func (c *Conn) Do(cmd string, args ...interface{}) (interface{}, error) { return c.DoWithTimeout(-1, cmd, args...) } // DoWithTimeout sends a command to the server and returns the received reply. // If the connection is not yet bound to a cluster node, it will be after this // call, based on the rules documented in the Conn type. // // The timeout overrides the read timeout set when dialing the connection (in // the DialOptions of the Cluster). func (c *Conn) DoWithTimeout(timeout time.Duration, cmd string, args ...interface{}) (v interface{}, err error) { // The blank command is a special redigo/redis command that flushes the // output buffer and receives all pending replies. This is used, for example, // when returning a Redis connection back to the pool. If we receive the // blank command, don't bind to a random node if this connection is not bound // yet. if cmd == "" && len(args) == 0 { c.mu.Lock() rc := c.rc c.mu.Unlock() if rc == nil { return nil, nil } } rc, _, err := c.bind(cmdSlot(cmd, args)) if err != nil { return nil, err } if timeout < 0 { v, err = rc.Do(cmd, args...) } else if rcwt, ok := rc.(redis.ConnWithTimeout); ok { v, err = rcwt.DoWithTimeout(timeout, cmd, args...) } else { return nil, errors.New("redisc: connection does not support ConnWithTimeout") } // handle redirections, if any if re := ParseRedir(err); re != nil { if re.Type == "MOVED" { c.cluster.needsRefresh(re) } } return v, err } // Send writes the command to the client's output buffer. If the connection is // not yet bound to a cluster node, it will be after this call, based on the // rules documented in the Conn type. func (c *Conn) Send(cmd string, args ...interface{}) error { rc, _, err := c.bind(cmdSlot(cmd, args)) if err != nil { return err } return rc.Send(cmd, args...) } // Receive receives a single reply from the server. If the connection is not // yet bound to a cluster node, it will be after this call, based on the rules // documented in the Conn type. func (c *Conn) Receive() (interface{}, error) { return c.ReceiveWithTimeout(-1) } // ReceiveWithTimeout receives a single reply from the Redis server. If the // connection is not yet bound to a cluster node, it will be after this call, // based on the rules documented in the Conn type. // // The timeout overrides the read timeout set when dialing the connection (in // the DialOptions of the Cluster). func (c *Conn) ReceiveWithTimeout(timeout time.Duration) (v interface{}, err error) { rc, _, err := c.bind(-1) if err != nil { return nil, err } if timeout < 0 { v, err = rc.Receive() } else if rcwt, ok := rc.(redis.ConnWithTimeout); ok { v, err = rcwt.ReceiveWithTimeout(timeout) } else { return nil, errors.New("redisc: connection does not support ConnWithTimeout") } // handle redirections, if any if re := ParseRedir(err); re != nil { if re.Type == "MOVED" { c.cluster.needsRefresh(re) } } return v, err } // Flush flushes the output buffer to the server. func (c *Conn) Flush() error { c.mu.Lock() err := c.err if err == nil && c.rc != nil { err = c.rc.Flush() } c.mu.Unlock() return err } // Err returns a non-nil value if the connection is broken. Applications // should close broken connections. func (c *Conn) Err() error { c.mu.Lock() err := c.err if err == nil && c.rc != nil { err = c.rc.Err() } c.mu.Unlock() return err } // Close closes the connection. func (c *Conn) Close() error { c.mu.Lock() err := c.err if err == nil { c.err = errors.New("redisc: closed") err = c.closeLocked() } c.mu.Unlock() return err } func (c *Conn) closeLocked() (err error) { if c.rc != nil { // this may be a pooled connection, so make sure the readOnly flag is reset if c.readOnly { _, _ = c.rc.Do("READWRITE") } err = c.rc.Close() } return err } mna-redisc-96a8e49/conn_test.go000066400000000000000000000257551451450242600164530ustar00rootroot00000000000000package redisc import ( "io" "net" "strings" "testing" "time" "github.com/gomodule/redigo/redis" "github.com/mna/redisc/redistest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // Test the conn.ReadOnly behaviour in a cluster setup with 1 replica per // node. func testConnReadOnlyWithReplicas(t *testing.T, ports []string) { t.Run("bind random without node", func(t *testing.T) { c := &Cluster{} defer c.Close() testWithReplicaBindRandomWithoutNode(t, c) }) t.Run("bind empty slot", func(t *testing.T) { c := &Cluster{StartupNodes: []string{ports[0]}} defer c.Close() testWithReplicaBindEmptySlot(t, c) }) t.Run("with refresh", func(t *testing.T) { c := &Cluster{StartupNodes: []string{ports[0]}} defer c.Close() testWithReplicaClusterRefresh(t, c, ports) // at this point the cluster has refreshed its mapping testReadWriteFromReplica(t, c, ports[redistest.NumClusterNodes:]) testReadOnlyWithRandomConn(t, c, ports[redistest.NumClusterNodes:]) testRetryReadOnlyConn(t, c, ports[:redistest.NumClusterNodes], ports[redistest.NumClusterNodes:]) }) } func testRetryReadOnlyConn(t *testing.T, c *Cluster, masters []string, replicas []string) { conn := c.Get().(*Conn) defer conn.Close() assert.NoError(t, ReadOnlyConn(conn), "ReadOnly") rc, _ := RetryConn(conn, 4, time.Second) // keys "a" and "b" are not in the same slot - bind to "a" and // then ask for "b" to force a redirect. assert.NoError(t, BindConn(conn, "a"), "Bind") addr1 := assertBoundTo(t, conn, replicas) if _, err := rc.Do("GET", "b"); assert.NoError(t, err, "GET b") { addr2 := assertBoundTo(t, conn, replicas) assert.NotEqual(t, addr1, addr2, "Bound to different replica") // conn is now bound to the node serving slot "b". Send a READWRITE // command and get "b" again, should re-bind to the same slot, but to // the master. _, err := rc.Do("READWRITE") assert.NoError(t, err, "READWRITE") if _, err := rc.Do("GET", "b"); assert.NoError(t, err, "GET b") { addr3 := assertBoundTo(t, conn, masters) assert.NotEqual(t, addr2, addr3, "Bound to the master") } } } // assert that conn is bound to one of the specified ports. func assertBoundTo(t *testing.T, conn *Conn, ports []string) string { conn.mu.Lock() addr := conn.boundAddr conn.mu.Unlock() found := false for _, port := range ports { if strings.HasSuffix(addr, port) { found = true break } } assert.True(t, found, "Bound address") return addr } func testReadOnlyWithRandomConn(t *testing.T, c *Cluster, replicas []string) { conn := c.Get().(*Conn) defer conn.Close() assert.NoError(t, ReadOnlyConn(conn), "ReadOnlyConn") assert.NoError(t, BindConn(conn), "BindConn") // it should now be bound to a random replica assertBoundTo(t, conn, replicas) } func testReadWriteFromReplica(t *testing.T, c *Cluster, replicas []string) { conn1 := c.Get() defer conn1.Close() _, err := conn1.Do("SET", "k1", "a") assert.NoError(t, err, "SET on master") conn2 := c.Get().(*Conn) defer conn2.Close() _ = ReadOnlyConn(conn2) // can read the key from the replica (may take a moment to replicate, // so retry a few times) var got string deadline := time.Now().Add(100 * time.Millisecond) for time.Now().Before(deadline) { got, err = redis.String(conn2.Do("GET", "k1")) if err != nil && got == "a" { break } time.Sleep(10 * time.Millisecond) } if assert.NoError(t, err, "GET from replica") { assert.Equal(t, "a", got, "expected value") } // bound address should be a replica assertBoundTo(t, conn2, replicas) // write command should fail with a MOVED if _, err = conn2.Do("SET", "k1", "b"); assert.Error(t, err, "SET on ReadOnly conn") { assert.Contains(t, err.Error(), "MOVED", "MOVED error") } // sending READWRITE switches the connection back to read from master _, err = conn2.Do("READWRITE") assert.NoError(t, err, "READWRITE") // now even a GET fails with a MOVED if _, err = conn2.Do("GET", "k1"); assert.Error(t, err, "GET on replica conn after READWRITE") { assert.Contains(t, err.Error(), "MOVED", "MOVED error") } } func testWithReplicaBindEmptySlot(t *testing.T, c *Cluster) { conn := c.Get() defer conn.Close() // key "a" is not in node at [0], so will generate a refresh and connect // to a random node (to node at [0]). assert.NoError(t, conn.(*Conn).Bind("a"), "Bind to missing slot") if _, err := conn.Do("GET", "a"); assert.Error(t, err, "GET") { assert.Contains(t, err.Error(), "MOVED", "MOVED error") } waitForClusterRefresh(c, func() { for i, v := range c.mapping { if !assert.NotEmpty(t, v, "Addr for %d", i) { break } } }) } func testWithReplicaBindRandomWithoutNode(t *testing.T, c *Cluster) { conn := c.Get() defer conn.Close() if err := conn.(*Conn).Bind(); assert.Error(t, err, "Bind fails") { assert.Contains(t, err.Error(), "failed to get a connection", "expected message") } } func testWithReplicaClusterRefresh(t *testing.T, c *Cluster, ports []string) { err := c.Refresh() if assert.NoError(t, err, "Refresh") { var prev string pix := -1 for ix, node := range c.mapping { if assert.Equal(t, 2, len(node), "Mapping for slot %d must have 2 nodes", ix) { if node[0] != prev || ix == len(c.mapping)-1 { prev = node[0] t.Logf("%5d: %s\n", ix, node[0]) pix++ } if assert.NotEmpty(t, node[0]) { split0, split1 := strings.Index(node[0], ":"), strings.Index(node[1], ":") assert.Contains(t, ports, node[0][split0:], "expected address") assert.Contains(t, ports, node[1][split1:], "expected address") } } else { break } } } } func testConnReadOnlyNoReplica(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: []string{ports[0]}, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") conn := c.Get() defer conn.Close() cc := conn.(*Conn) assert.NoError(t, cc.ReadOnly(), "ReadOnly") // both get and set work, because the connection is on a master _, err := cc.Do("SET", "b", 1) assert.NoError(t, err, "SET") v, err := redis.Int(cc.Do("GET", "b")) if assert.NoError(t, err, "GET") { assert.Equal(t, 1, v, "expected result") } conn2 := c.Get() defer conn2.Close() cc2 := conn2.(*Conn) assert.NoError(t, cc2.Bind(), "Bind") assert.Error(t, cc2.ReadOnly(), "ReadOnly after Bind") } func testConnBind(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: ports, DialOptions: []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)}, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") conn := c.Get() defer conn.Close() if err := BindConn(conn, "A", "B"); assert.Error(t, err, "Bind with different keys") { assert.Contains(t, err.Error(), "keys do not belong to the same slot", "expected message") } assert.NoError(t, BindConn(conn, "A"), "Bind") if err := BindConn(conn, "B"); assert.Error(t, err, "Bind after Bind") { assert.Contains(t, err.Error(), "connection already bound", "expected message") } conn2 := c.Get() defer conn2.Close() assert.NoError(t, BindConn(conn2), "Bind without key") } func testConnBlankDo(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: ports, DialOptions: []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)}, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") conn := c.Get() defer conn.Close() cconn := conn.(*Conn) _, err := conn.Do("") assert.NoError(t, err) assert.Nil(t, cconn.rc) err = BindConn(conn, "A") assert.NoError(t, err, "Bind") assert.NotNil(t, cconn.rc) } func testConnWithTimeout(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: []string{ports[0]}, DialOptions: []redis.DialOption{ redis.DialConnectTimeout(2 * time.Second), redis.DialReadTimeout(time.Second), }, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") testConnDoWithTimeout(t, c) testConnReceiveWithTimeout(t, c) } func testConnDoWithTimeout(t *testing.T, c *Cluster) { conn1 := c.Get().(*Conn) defer conn1.Close() // Do fails because the default timeout is 1s, but command blocks for 2s _, err1 := conn1.Do("BLPOP", "x", 2) if assert.Error(t, err1, "Do") { if assert.IsType(t, &net.OpError{}, err1) { oe := err1.(*net.OpError) assert.True(t, oe.Timeout(), "is timeout") } } conn2 := c.Get().(*Conn) defer conn2.Close() // DoWithTimeout succeeds because overrides timeout with 3s. v2, err2 := conn2.DoWithTimeout(time.Second*3, "BLPOP", "x", 2) assert.NoError(t, err2, "DoWithTimeout") assert.Equal(t, nil, v2, "expected result") } func testConnReceiveWithTimeout(t *testing.T, c *Cluster) { conn1 := c.Get().(*Conn) defer conn1.Close() assert.NoError(t, conn1.Send("BLPOP", "x", 2), "Send") assert.NoError(t, conn1.Flush(), "Flush") // Receive fails with its default timeout of 1s vs Block command for 2s _, err1 := conn1.Receive() if assert.Error(t, err1, "Receive") { if assert.IsType(t, &net.OpError{}, err1) { oe := err1.(*net.OpError) assert.True(t, oe.Timeout(), "is timeout") } } conn2 := c.Get().(*Conn) defer conn2.Close() // ReceiveWithTimeout succeeds with timeout of 3s vs Block command for 2s assert.NoError(t, conn2.Send("BLPOP", "x", 2), "Send") assert.NoError(t, conn2.Flush(), "Flush") v2, err2 := conn2.ReceiveWithTimeout(time.Second * 3) assert.NoError(t, err2, "ReceiveWithTimeout") assert.Equal(t, nil, v2, "expected result") } func TestConnClose(t *testing.T) { c := &Cluster{ StartupNodes: []string{":6379"}, } defer c.Close() conn := c.Get() require.NoError(t, conn.Close(), "Close") _, err := conn.Do("A") if assert.Error(t, err, "Do after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if assert.Error(t, conn.Err(), "Err after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if assert.Error(t, conn.Close(), "Close after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if assert.Error(t, conn.Flush(), "Flush after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if assert.Error(t, conn.Send("A"), "Send after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } _, err = conn.Receive() if assert.Error(t, err, "Receive after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } cc := conn.(*Conn) if assert.Error(t, cc.Bind("A"), "Bind after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } if assert.Error(t, cc.ReadOnly(), "ReadOnly after Close") { assert.Contains(t, err.Error(), "redisc: closed", "expected message") } } func TestIsRedisError(t *testing.T) { err := error(redis.Error("CROSSSLOT some message")) assert.True(t, IsCrossSlot(err), "CrossSlot") assert.False(t, IsTryAgain(err), "CrossSlot") err = redis.Error("TRYAGAIN some message") assert.False(t, IsCrossSlot(err), "TryAgain") assert.True(t, IsTryAgain(err), "TryAgain") err = io.EOF assert.False(t, IsCrossSlot(err), "EOF") assert.False(t, IsTryAgain(err), "EOF") err = redis.Error("ERR some error") assert.False(t, IsCrossSlot(err), "ERR") assert.False(t, IsTryAgain(err), "ERR") } mna-redisc-96a8e49/crc16.go000066400000000000000000000110011451450242600153500ustar00rootroot00000000000000/* * Copyright 2001-2010 Georges Menie (www.menie.org) * Copyright 2010-2012 Salvatore Sanfilippo (adapted to Redis coding style) * Copyright 2015 https://github.com/chasex (adapted to Go, Apache 2.0 license) * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the University of California, Berkeley nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* CRC16 implementation according to CCITT standards. * * Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the * following parameters: * * Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" * Width : 16 bit * Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) * Initialization : 0000 * Reflect Input byte : False * Reflect Output CRC : False * Xor constant to output CRC : 0000 * Output for "123456789" : 31C3 */ package redisc var crc16tab = [...]uint16{ 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, } func crc16(s string) uint16 { var crc uint16 for i := 0; i < len(s); i++ { b := s[i] crc = (crc << 8) ^ crc16tab[byte(crc>>8)^b] } return crc } mna-redisc-96a8e49/doc.go000066400000000000000000000176441451450242600152220ustar00rootroot00000000000000// Package redisc implements a redis cluster client on top of // the redigo client package. It supports all commands that can // be executed on a redis cluster, including pub-sub, scripts and // read-only connections to read data from replicas. // See http://redis.io/topics/cluster-spec for details. // // # Design // // The package defines two main types: Cluster and Conn. Both // are described in more details below, but the Cluster manages // the mapping of keys (or more exactly, hash slots computed from // keys) to a group of nodes that form a redis cluster, and a // Conn manages a connection to this cluster. // // The package is designed such that for simple uses, or when // keys have been carefully named to play well with a redis // cluster, a Cluster value can be used as a drop-in replacement // for a redis.Pool from the redigo package. // // Similarly, the Conn type implements redigo's redis.Conn // interface (and the augmented redis.ConnWithTimeout one too), // so the API to execute commands is the same - // in fact the redisc package uses the redigo package as its // only third-party dependency. // // When more control is needed, the package offers some // extra behaviour specific to working with a redis cluster: // // - Slot and SplitBySlot functions to compute the slot for // a given key and to split a list of keys into groups of // keys from the same slot, so that each group can safely be // handled using the same connection. // // - *Conn.Bind (or the BindConn package-level helper function) // to explicitly specify the keys that will be used with the // connection so that the right node is selected, instead of // relying on the automatic detection based on the first // parameter of the command. // // - *Conn.ReadOnly (or the ReadOnlyConn package-level helper // function) to mark a connection as read-only, allowing // commands to be served by a replica instead of the master. // // - RetryConn to wrap a connection into one that automatically // follows redirections when the cluster moves slots around. // // - Helper functions to deal with cluster-specific errors. // // # Cluster // // The Cluster type manages a redis cluster and offers an // interface compatible with redigo's redis.Pool: // // Get() redis.Conn // Close() error // // Along with some additional methods specific to a cluster: // // Dial() (redis.Conn, error) // EachNode(bool, func(string, redis.Conn) error) error // Refresh() error // Stats() map[string]redis.PoolStats // // If the CreatePool function field is set, then a // redis.Pool is created to manage connections to each of the // cluster's nodes. A call to Get returns a connection // from that pool. // // The Dial method, on the other hand, guarantees that // the returned connection will not be managed by a pool, even if // CreatePool is set. It calls redigo's redis.Dial function // to create the unpooled connection, passing along any DialOptions // set on the cluster. If the cluster's CreatePool field is nil, // Get behaves the same as Dial. // // The Refresh method refreshes the cluster's internal mapping of // hash slots to nodes. It should typically be called only once, // after the cluster is created and before it is used, so that // the first connections already benefit from smart routing. // It is automatically kept up-to-date based on the redis MOVED // responses afterwards. // // The EachNode method visits each node in the cluster and calls // the provided function with a connection to that node, which may // be useful to run diagnostics commands on each node or to collect // keys across the whole cluster. // // The Stats method returns the pool statistics for each node, with // the node's address as key of the map. // // A cluster must be closed once it is no longer used to release // its resources. // // # Connection // // The connection returned from Get or Dial is a redigo redis.Conn // interface (that also implements redis.ConnWithTimeout), // with a concrete type of *Conn. In addition to the // interface's required methods, *Conn adds the following methods: // // Bind(...string) error // ReadOnly() error // // The returned connection is not yet connected to any node; it is // "bound" to a specific node only when a call to Do, Send, Receive // or Bind is made. For Do, Send and Receive, the node selection is // implicit, it uses the first parameter of the command, and // computes the hash slot assuming that first parameter is a key. // It then binds the connection to the node corresponding to that // slot. If there are no parameters for the command, or if there is // no command (e.g. in a call to Receive), a random node is selected. // // Bind is explicit, it gives control to the caller over // which node to select by specifying a list of keys that the caller // wishes to handle with the connection. All keys must belong to the // same slot, and the connection must not already be bound to a node, // otherwise an error is returned. On success, the connection is // bound to the node holding the slot of the specified key(s). // // Because the connection is returned as a redis.Conn interface, a // type assertion must be used to access the underlying *Conn and // to be able to call Bind: // // redisConn := cluster.Get() // if conn, ok := redisConn.(*redisc.Conn); ok { // if err := conn.Bind("my-key"); err != nil { // // handle error // } // } // // The BindConn package-level function is provided as a helper for // this common use-case. // // The ReadOnly method marks the connection as read-only, meaning that // it will attempt to connect to a replica instead of the master node // for its slot. Once bound to a node, the READONLY redis command is // sent automatically, so it doesn't have to be sent explicitly before // use. ReadOnly must be called before the connection is bound to a // node, otherwise an error is returned. // // For the same reason as for Bind, a type assertion must be used to // call ReadOnly on a *Conn, so a package-level helper function is // also provided, ReadOnlyConn. // // There is no ReadWrite method, because it can be sent as a normal // redis command and will essentially end that connection (all commands // will now return MOVED errors). If the connection was wrapped in // a RetryConn call, then it will automatically follow the redirection // to the master node (see the Redirections section). // // The connection must be closed after use, to release the underlying // resources. // // # Redirections // // The redis cluster may return MOVED and ASK errors when the node // that received the command doesn't currently hold the slot corresponding // to the key. The package cannot reliably handle those redirections // automatically because the redirection error may be returned for // a pipeline of commands, some of which may have succeeded. // // However, a connection can be wrapped by a call to RetryConn, which // returns a redis.Conn interface where only calls to Do, Close and Err // can succeed. That means pipelining is not supported, and only a single // command can be executed at a time, but it will automatically handle // MOVED and ASK replies, as well as TRYAGAIN errors. // // Note that even if RetryConn is not used, the cluster always updates // its mapping of slots to nodes automatically by keeping track of // MOVED replies. // // # Concurrency // // The concurrency model is similar to that of the redigo package: // // - Cluster methods are safe to call concurrently (like redis.Pool). // // - Connections do not support concurrent calls to write methods // (Send, Flush) or concurrent calls to the read method (Receive). // // - Connections do allow a concurrent reader and writer. // // - Because the Do method combines the functionality of Send, Flush // and Receive, it cannot be called concurrently with other methods. // // - The Bind and ReadOnly methods are safe to call concurrently, but // there is not much point in doing so for as both will fail if // the connection is already bound. package redisc mna-redisc-96a8e49/example_test.go000066400000000000000000000077161451450242600171460ustar00rootroot00000000000000package redisc_test import ( "fmt" "log" "time" "github.com/gomodule/redigo/redis" "github.com/mna/redisc" ) // Create and use a cluster. func Example() { // create the cluster cluster := redisc.Cluster{ StartupNodes: []string{":7000", ":7001", ":7002"}, DialOptions: []redis.DialOption{redis.DialConnectTimeout(5 * time.Second)}, CreatePool: createPool, } defer cluster.Close() // initialize its mapping if err := cluster.Refresh(); err != nil { log.Fatalf("Refresh failed: %v", err) } // grab a connection from the pool conn := cluster.Get() defer conn.Close() // call commands on it s, err := redis.String(conn.Do("GET", "some-key")) if err != nil { log.Fatalf("GET failed: %v", err) } log.Println(s) // grab a non-pooled connection conn2, err := cluster.Dial() if err != nil { log.Fatalf("Dial failed: %v", err) } defer conn2.Close() // make it handle redirections automatically rc, err := redisc.RetryConn(conn2, 3, 100*time.Millisecond) if err != nil { log.Fatalf("RetryConn failed: %v", err) } _, err = rc.Do("SET", "some-key", 2) if err != nil { log.Fatalf("SET failed: %v", err) } } func createPool(addr string, opts ...redis.DialOption) (*redis.Pool, error) { return &redis.Pool{ MaxIdle: 5, MaxActive: 10, IdleTimeout: time.Minute, Dial: func() (redis.Conn, error) { return redis.Dial("tcp", addr, opts...) }, TestOnBorrow: func(c redis.Conn, t time.Time) error { _, err := c.Do("PING") return err }, }, nil } // Execute scripts. func ExampleConn() { // create the cluster cluster := redisc.Cluster{ StartupNodes: []string{":7000", ":7001", ":7002"}, DialOptions: []redis.DialOption{redis.DialConnectTimeout(5 * time.Second)}, CreatePool: createPool, } defer cluster.Close() // initialize its mapping if err := cluster.Refresh(); err != nil { log.Fatalf("Refresh failed: %v", err) } // create a script that takes 2 keys and 2 values, and returns 1 var script = redis.NewScript(2, ` redis.call("SET", KEYS[1], ARGV[1]) redis.call("SET", KEYS[2], ARGV[2]) return 1 `) // get a connection from the cluster conn := cluster.Get() defer conn.Close() // bind it to the right node for the required keys, ahead of time if err := redisc.BindConn(conn, "scr{a}1", "src{a}2"); err != nil { log.Fatalf("BindConn failed: %v", err) } // script.Do, sends the whole script on first use v, err := script.Do(conn, "scr{a}1", "scr{a}2", "x", "y") if err != nil { log.Fatalf("script.Do failed: %v", err) } fmt.Println("Do returned ", v) // it is also possible to send only the hash, once it has been // loaded on that node if err := script.SendHash(conn, "scr{a}1", "scr{a}2", "x", "y"); err != nil { log.Fatalf("script.SendHash failed: %v", err) } if err := conn.Flush(); err != nil { log.Fatalf("Flush failed: %v", err) } // and receive the script's result v, err = conn.Receive() if err != nil { log.Fatalf("Receive failed: %v", err) } fmt.Println("Receive returned ", v) } // Automatically retry in case of redirection errors. func ExampleRetryConn() { // create the cluster cluster := redisc.Cluster{ StartupNodes: []string{":7000", ":7001", ":7002"}, DialOptions: []redis.DialOption{redis.DialConnectTimeout(5 * time.Second)}, CreatePool: createPool, } defer cluster.Close() // initialize its mapping if err := cluster.Refresh(); err != nil { log.Fatalf("Refresh failed: %v", err) } // get a connection from the cluster conn := cluster.Get() defer conn.Close() // create the retry connection - only Do, Close and Err are // supported on that connection. It will make up to 3 attempts // to get a valid response, and will wait 100ms before a retry // in case of a TRYAGAIN redis error. retryConn, err := redisc.RetryConn(conn, 3, 100*time.Millisecond) if err != nil { log.Fatalf("RetryConn failed: %v", err) } // call commands v, err := retryConn.Do("GET", "key") if err != nil { log.Fatalf("GET failed: %v", err) } fmt.Println("GET returned ", v) } mna-redisc-96a8e49/go.mod000066400000000000000000000001721451450242600152200ustar00rootroot00000000000000module github.com/mna/redisc go 1.16 require ( github.com/gomodule/redigo v1.8.5 github.com/stretchr/testify v1.7.0 ) mna-redisc-96a8e49/go.sum000066400000000000000000000027771451450242600152620ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gomodule/redigo v1.8.4 h1:Z5JUg94HMTR1XpwBaSH4vq3+PNSIykBLxMdglbw10gg= github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/gomodule/redigo v1.8.5 h1:nRAxCa+SVsyjSBrtZmG/cqb6VbTmuRzpg/PoTFlpumc= github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= mna-redisc-96a8e49/hash.go000066400000000000000000000017641451450242600153740ustar00rootroot00000000000000package redisc import ( "sort" "strings" ) // Slot returns the hash slot for the key. func Slot(key string) int { if start := strings.Index(key, "{"); start >= 0 { if end := strings.Index(key[start+1:], "}"); end > 0 { // if end == 0, then it's {}, so we ignore it end += start + 1 key = key[start+1 : end] } } return int(crc16(key) % HashSlots) } // SplitBySlot takes a list of keys and returns a list of list of keys, // grouped by identical cluster slot. For example: // // bySlot := SplitBySlot("k1", "k2", "k3") // for _, keys := range bySlot { // // keys is a list of keys that belong to the same slot // } func SplitBySlot(keys ...string) [][]string { var slots []int m := make(map[int][]string) for _, k := range keys { slot := Slot(k) _, ok := m[slot] m[slot] = append(m[slot], k) if !ok { slots = append(slots, slot) } } sort.Ints(slots) bySlot := make([][]string, 0, len(m)) for _, slot := range slots { bySlot = append(bySlot, m[slot]) } return bySlot } mna-redisc-96a8e49/hash_test.go000066400000000000000000000025161451450242600164270ustar00rootroot00000000000000package redisc import ( "strings" "testing" "github.com/stretchr/testify/assert" ) func TestSlot(t *testing.T) { cases := []struct { in string out int }{ {"", 0}, {"a", 15495}, {"b", 3300}, {"ab", 13567}, {"abc", 7638}, {"a{b}", 3300}, {"{a}b", 15495}, {"{a}{b}", 15495}, {"{}{a}{b}", 11267}, {"a{b}c", 3300}, {"{a}bc", 15495}, {"{a}{b}{c}", 15495}, {"{}{a}{b}{c}", 1044}, {"a{bc}d", 12685}, {"a{bcd}", 1872}, {"{abcd}", 10294}, {"abcd", 10294}, {"{a", 10276}, {"a}", 5921}, {"123456789", 12739}, {"a≠b", 11870}, {"•", 97}, {"a{}{b}c", 14872}, } for _, c := range cases { got := Slot(c.in) assert.Equal(t, c.out, got, c.in) } } func TestSplitBySlot(t *testing.T) { cases := []struct { // join/split by comma, for convenience in string out []string }{ {"", nil}, {"a", []string{"a"}}, {"a,b", []string{"b", "a"}}, {"a,b,cb{a}", []string{"b", "a,cb{a}"}}, {"a,b,cb{a},a{b}", []string{"b,a{b}", "a,cb{a}"}}, {"a,b,cb{a},a{b},abc", []string{"b,a{b}", "abc", "a,cb{a}"}}, } for _, c := range cases { args := strings.Split(c.in, ",") if c.in == "" { args = nil } got := SplitBySlot(args...) exp := make([][]string, len(c.out)) for i, o := range c.out { exp[i] = strings.Split(o, ",") } assert.Equal(t, exp, got, c.in) t.Logf("%#v", got) } } mna-redisc-96a8e49/redistest/000077500000000000000000000000001451450242600161205ustar00rootroot00000000000000mna-redisc-96a8e49/redistest/mock_server.go000066400000000000000000000036401451450242600207710ustar00rootroot00000000000000package redistest import ( "bufio" "net" "sync" "testing" "time" "github.com/mna/redisc/redistest/resp" "github.com/stretchr/testify/require" ) // MockServer is a mock redis server. type MockServer struct { Addr string done chan struct{} wg sync.WaitGroup h func(string, ...string) interface{} t *testing.T l net.Listener } // StartMockServer creates and starts a mock redis server. The handler is // called for each command received by the server. The returned value is // encoded in the redis protocol and sent to the client. The caller should close // the server after use. func StartMockServer(t *testing.T, handler func(cmd string, args ...string) interface{}) *MockServer { l, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err, "net.Listen") _, port, _ := net.SplitHostPort(l.Addr().String()) s := &MockServer{ Addr: ":" + port, done: make(chan struct{}), h: handler, t: t, l: l, } go s.serve() return s } // Close closes the mock redis server. func (s *MockServer) Close() { select { case <-s.done: return default: } require.NoError(s.t, s.l.Close(), "Close listener") <-s.done exit := make(chan struct{}) go func() { s.wg.Wait() close(exit) }() // wait for a few seconds for connections to finish, otherwise fail select { case <-exit: return case <-time.After(5 * time.Second): s.t.Fatal("failed to cleanly stop the mock server") } } func (s *MockServer) serve() { defer close(s.done) for { conn, err := s.l.Accept() if err != nil { return } s.wg.Add(1) go s.serveConn(conn) } } func (s *MockServer) serveConn(c net.Conn) { defer s.wg.Done() go func() { <-s.done c.Close() }() br := bufio.NewReader(c) for { // Get the request ar, err := resp.DecodeRequest(br) if err != nil { return } // Handle the response v := s.h(ar[0], ar[1:]...) if err := resp.Encode(c, v); err != nil { panic(err) } } } mna-redisc-96a8e49/redistest/mock_server_test.go000066400000000000000000000007761451450242600220370ustar00rootroot00000000000000package redistest import ( "testing" "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestMockServer(t *testing.T) { s := StartMockServer(t, func(cmd string, args ...string) interface{} { return cmd }) defer s.Close() c, err := redis.Dial("tcp", s.Addr) require.NoError(t, err, "Dial") v, err := redis.String(c.Do("ECHO", "a")) require.NoError(t, err, "ECHO") assert.Equal(t, "ECHO", v, "Should return the command name") } mna-redisc-96a8e49/redistest/resp/000077500000000000000000000000001451450242600170715ustar00rootroot00000000000000mna-redisc-96a8e49/redistest/resp/decode.go000066400000000000000000000141241451450242600206450ustar00rootroot00000000000000// Package resp implements an efficient decoder for the Redis Serialization Protocol (RESP). // // See http://redis.io/topics/protocol for the reference. package resp import ( "bytes" "errors" "fmt" "io" ) var ( // ErrInvalidPrefix is returned if the data contains an unrecognized prefix. ErrInvalidPrefix = errors.New("resp: invalid prefix") // ErrMissingCRLF is returned if a \r\n is missing in the data slice. ErrMissingCRLF = errors.New("resp: missing CRLF") // ErrInvalidInteger is returned if an invalid character is found while parsing an integer. ErrInvalidInteger = errors.New("resp: invalid integer character") // ErrInvalidBulkString is returned if the bulk string data cannot be decoded. ErrInvalidBulkString = errors.New("resp: invalid bulk string") // ErrInvalidArray is returned if the array data cannot be decoded. ErrInvalidArray = errors.New("resp: invalid array") // ErrNotAnArray is returned if the DecodeRequest function is called and // the decoded value is not an array. ErrNotAnArray = errors.New("resp: expected an array type") // ErrInvalidRequest is returned if the DecodeRequest function is called and // the decoded value is not an array containing only bulk strings, and at least 1 element. ErrInvalidRequest = errors.New("resp: invalid request, must be an array of bulk strings with at least one element") ) // BytesReader defines the methods required for the Decode* family of methods. // Notably, a *bufio.Reader and a *bytes.Buffer both satisfy this interface. type BytesReader interface { io.Reader io.ByteReader ReadBytes(byte) ([]byte, error) } // Array represents an array of values, as defined by the RESP. type Array []interface{} // String is the Stringer implementation for the Array. func (a Array) String() string { var buf bytes.Buffer for i, v := range a { buf.WriteString(fmt.Sprintf("[%2d] %[2]v (%[2]T)\n", i, v)) } return buf.String() } // DecodeRequest decodes the provided byte slice and returns the array // representing the request. If the encoded value is not an array, it // returns ErrNotAnArray, and if it is not a valid request, it returns ErrInvalidRequest. func DecodeRequest(r BytesReader) ([]string, error) { // Decode the value val, err := Decode(r) if err != nil { return nil, err } // Must be an array ar, ok := val.(Array) if !ok { return nil, ErrNotAnArray } // Must have at least one element if len(ar) < 1 { return nil, ErrInvalidRequest } // Must have only strings strs := make([]string, len(ar)) for i, v := range ar { v, ok := v.(string) if !ok { return nil, ErrInvalidRequest } strs[i] = v } return strs, nil } // Decode decodes the provided byte slice and returns the parsed value. func Decode(r BytesReader) (interface{}, error) { return decodeValue(r) } // decodeValue parses the byte slice and decodes the value based on its // prefix, as defined by the RESP protocol. func decodeValue(r BytesReader) (interface{}, error) { ch, err := r.ReadByte() if err != nil { return nil, err } var val interface{} switch ch { case '+': // Simple string val, err = decodeSimpleString(r) case '-': // Error val, err = decodeError(r) case ':': // Integer val, err = decodeInteger(r) case '$': // Bulk string val, err = decodeBulkString(r) case '*': // Array val, err = decodeArray(r) default: err = ErrInvalidPrefix } return val, err } // decodeArray decodes the byte slice as an array. It assumes the // '*' prefix is already consumed. func decodeArray(r BytesReader) (Array, error) { // First comes the number of elements in the array cnt, err := decodeInteger(r) if err != nil { return nil, err } switch { case cnt == -1: // Nil array return nil, nil case cnt == 0: // Empty, but allocated, array return Array{}, nil case cnt < 0: // Invalid length return nil, ErrInvalidArray default: // Allocate the array ar := make(Array, cnt) // Decode each value for i := 0; i < int(cnt); i++ { val, err := decodeValue(r) if err != nil { return nil, err } ar[i] = val } return ar, nil } } // decodeBulkString decodes the byte slice as a binary-safe string. The // '$' prefix is assumed to be already consumed. func decodeBulkString(r BytesReader) (interface{}, error) { // First comes the length of the bulk string, an integer cnt, err := decodeInteger(r) if err != nil { return nil, err } switch { case cnt == -1: // Special case to represent a nil bulk string return nil, nil case cnt < -1: return nil, ErrInvalidBulkString default: // Then the string is cnt long, and bytes read is cnt+n+2 (for ending CRLF) need := cnt + 2 got := 0 buf := make([]byte, need) for { nb, err := r.Read(buf[got:]) if err != nil { return nil, err } got += nb if int64(got) == need { break } } return string(buf[:got-2]), err } } // decodeInteger decodes the byte slice as a singed 64bit integer. The // ':' prefix is assumed to be already consumed. func decodeInteger(r BytesReader) (val int64, err error) { var cr bool var sign int64 = 1 var n int loop: for { ch, err := r.ReadByte() if err != nil { return 0, err } n++ switch ch { case '\r': cr = true break loop case '\n': break loop case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': val = val*10 + int64(ch-'0') case '-': if n == 1 { sign = -1 continue } fallthrough default: return 0, ErrInvalidInteger } } if !cr { return 0, ErrMissingCRLF } // Presume next byte was \n _, err = r.ReadByte() if err != nil { return 0, err } return sign * val, nil } // decodeSimpleString decodes the byte slice as a SimpleString. The // '+' prefix is assumed to be already consumed. func decodeSimpleString(r BytesReader) (interface{}, error) { v, err := r.ReadBytes('\r') if err != nil { return nil, err } // Presume next byte was \n _, err = r.ReadByte() if err != nil { return nil, err } return string(v[:len(v)-1]), nil } // decodeError decodes the byte slice as an Error. The '-' prefix // is assumed to be already consumed. func decodeError(r BytesReader) (interface{}, error) { return decodeSimpleString(r) } mna-redisc-96a8e49/redistest/resp/decode_test.go000066400000000000000000000122571451450242600217110ustar00rootroot00000000000000package resp import ( "bytes" "io" "reflect" "testing" ) var decodeErrCases = []struct { enc []byte val interface{} err error }{ 0: {[]byte("+ceci n'est pas un string"), nil, io.EOF}, 1: {[]byte("+"), nil, io.EOF}, 2: {[]byte("-ceci n'est pas un string"), nil, io.EOF}, 3: {[]byte("-"), nil, io.EOF}, 4: {[]byte(":123\n"), int64(0), ErrMissingCRLF}, 5: {[]byte(":123a\r\n"), int64(0), ErrInvalidInteger}, 6: {[]byte(":123"), int64(0), io.EOF}, 7: {[]byte(":-1-3\r\n"), int64(0), ErrInvalidInteger}, 8: {[]byte(":"), int64(0), io.EOF}, 9: {[]byte("$"), nil, io.EOF}, 10: {[]byte("$6\r\nc\r\n"), nil, io.EOF}, 11: {[]byte("$6\r\nabc\r\n"), nil, io.EOF}, 12: {[]byte("$6\nabc\r\n"), nil, ErrMissingCRLF}, 13: {[]byte("$4\r\nabc\r\n"), nil, io.EOF}, 14: {[]byte("$-3\r\n"), nil, ErrInvalidBulkString}, 15: {[]byte("*1\n:10\r\n"), Array(nil), ErrMissingCRLF}, 16: {[]byte("*-3\r\n"), Array(nil), ErrInvalidArray}, 17: {[]byte(":\r\n"), int64(0), nil}, 18: {[]byte("$\r\n\r\n"), "", nil}, } var decodeValidCases = []struct { enc []byte val interface{} err error }{ 0: {[]byte{'+', '\r', '\n'}, "", nil}, 1: {[]byte{'+', 'a', '\r', '\n'}, "a", nil}, 2: {[]byte{'+', 'O', 'K', '\r', '\n'}, "OK", nil}, 3: {[]byte("+ceci n'est pas un string\r\n"), "ceci n'est pas un string", nil}, 4: {[]byte{'-', '\r', '\n'}, "", nil}, 5: {[]byte{'-', 'a', '\r', '\n'}, "a", nil}, 6: {[]byte{'-', 'K', 'O', '\r', '\n'}, "KO", nil}, 7: {[]byte("-ceci n'est pas un string\r\n"), "ceci n'est pas un string", nil}, 8: {[]byte(":1\r\n"), int64(1), nil}, 9: {[]byte(":123\r\n"), int64(123), nil}, 10: {[]byte(":-123\r\n"), int64(-123), nil}, 11: {[]byte("$0\r\n\r\n"), "", nil}, 12: {[]byte("$24\r\nceci n'est pas un string\r\n"), "ceci n'est pas un string", nil}, 13: {[]byte("$51\r\nceci n'est pas un string\r\navec\rdes\nsauts\r\nde\x00ligne.\r\n"), "ceci n'est pas un string\r\navec\rdes\nsauts\r\nde\x00ligne.", nil}, 14: {[]byte("$-1\r\n"), nil, nil}, 15: {[]byte("*0\r\n"), Array{}, nil}, 16: {[]byte("*1\r\n:10\r\n"), Array{int64(10)}, nil}, 17: {[]byte("*-1\r\n"), Array(nil), nil}, 18: {[]byte("*3\r\n+string\r\n-error\r\n:-2345\r\n"), Array{"string", "error", int64(-2345)}, nil}, 19: {[]byte("*5\r\n+string\r\n-error\r\n:-2345\r\n$4\r\nallo\r\n*2\r\n$0\r\n\r\n$-1\r\n"), Array{"string", "error", int64(-2345), "allo", Array{"", nil}}, nil}, } var decodeRequestCases = []struct { raw []byte exp []string err error }{ 0: {[]byte("*-1\r\n"), nil, ErrInvalidRequest}, 1: {[]byte(":4\r\n"), nil, ErrNotAnArray}, 2: {[]byte("*0\r\n"), nil, ErrInvalidRequest}, 3: {[]byte("*1\r\n:6\r\n"), nil, ErrInvalidRequest}, 4: {[]byte("*1\r\n$2\r\nab\r\n"), []string{"ab"}, nil}, 5: {[]byte("*3\r\n$3\r\nSET\r\n$5\r\nmykey\r\n$24\r\nceci n'est pas un string\r\n"), []string{"SET", "mykey", "ceci n'est pas un string"}, nil}, } func TestDecode(t *testing.T) { for i, c := range append(decodeValidCases, decodeErrCases...) { got, err := Decode(bytes.NewBuffer(c.enc)) if err != c.err { t.Errorf("%d: expected error %v, got %v", i, c.err, err) } if got == nil && c.val == nil { continue } assertValue(t, i, got, c.val) } } func TestDecodeRequest(t *testing.T) { for i, c := range decodeRequestCases { buf := bytes.NewBuffer(c.raw) got, err := DecodeRequest(buf) if err != c.err { t.Errorf("%d: expected error %v, got %v", i, c.err, err) } if got == nil && c.exp == nil { continue } assertValue(t, i, got, c.exp) } } func assertValue(t *testing.T, i int, got, exp interface{}) { tgot, texp := reflect.TypeOf(got), reflect.TypeOf(exp) if tgot != texp { t.Errorf("%d: expected type %s, got %s", i, texp, tgot) } if !reflect.DeepEqual(got, exp) { t.Errorf("%d: expected output %v, got %v", i, exp, got) } } var forbenchmark interface{} func BenchmarkDecodeSimpleString(b *testing.B) { var val interface{} var err error for i := 0; i < b.N; i++ { r := bytes.NewBuffer(decodeValidCases[3].enc) val, err = Decode(r) } if err != nil { b.Fatal(err) } forbenchmark = val } func BenchmarkDecodeError(b *testing.B) { var val interface{} var err error for i := 0; i < b.N; i++ { r := bytes.NewBuffer(decodeValidCases[7].enc) val, err = Decode(r) } if err != nil { b.Fatal(err) } forbenchmark = val } func BenchmarkDecodeInteger(b *testing.B) { var val interface{} var err error for i := 0; i < b.N; i++ { r := bytes.NewBuffer(decodeValidCases[10].enc) val, err = Decode(r) } if err != nil { b.Fatal(err) } forbenchmark = val } func BenchmarkDecodeBulkString(b *testing.B) { var val interface{} var err error for i := 0; i < b.N; i++ { r := bytes.NewBuffer(decodeValidCases[13].enc) val, err = Decode(r) } if err != nil { b.Fatal(err) } forbenchmark = val } func BenchmarkDecodeArray(b *testing.B) { var val interface{} var err error for i := 0; i < b.N; i++ { r := bytes.NewBuffer(decodeValidCases[19].enc) val, err = Decode(r) } if err != nil { b.Fatal(err) } forbenchmark = val } func BenchmarkDecodeRequest(b *testing.B) { var val interface{} var err error for i := 0; i < b.N; i++ { r := bytes.NewBuffer(decodeRequestCases[5].raw) val, err = Decode(r) } if err != nil { b.Fatal(err) } forbenchmark = val } mna-redisc-96a8e49/redistest/resp/encode.go000066400000000000000000000106161451450242600206610ustar00rootroot00000000000000package resp import ( "errors" "io" "strconv" ) var ( // Common encoding values optimized to avoid allocations. pong = []byte("+PONG\r\n") ok = []byte("+OK\r\n") t = []byte(":1\r\n") f = []byte(":0\r\n") one = t zero = f ) // ErrInvalidValue is returned if the value to encode is invalid. var ErrInvalidValue = errors.New("resp: invalid value") // Error represents an error string as defined by the RESP. It cannot // contain \r or \n characters. It must be used as a type conversion // so that Encode serializes the string as an Error. type Error string // Pong is a sentinel type used to indicate that the PONG simple string // value should be encoded. type Pong struct{} // OK is a sentinel type used to indicate that the OK simple string // value should be encoded. type OK struct{} // SimpleString represents a simple string as defined by the RESP. It // cannot contain \r or \n characters. It must be used as a type conversion // so that Encode serializes the string as a SimpleString. type SimpleString string // BulkString represents a binary-safe string as defined by the RESP. // It can be used as a type conversion so that Encode serializes the string // as a BulkString, but this is the default encoding for a normal Go string. type BulkString string // Encode encode the value v and writes the serialized data to w. func Encode(w io.Writer, v interface{}) error { return encodeValue(w, v) } // encodeValue encodes the value v and writes the serialized data to w. func encodeValue(w io.Writer, v interface{}) error { switch v := v.(type) { case OK: _, err := w.Write(ok) return err case Pong: _, err := w.Write(pong) return err case bool: if v { _, err := w.Write(t) return err } _, err := w.Write(f) return err case SimpleString: return encodeSimpleString(w, v) case Error: return encodeError(w, v) case int64: switch v { case 0: _, err := w.Write(zero) return err case 1: _, err := w.Write(one) return err default: return encodeInteger(w, v) } case string: return encodeBulkString(w, BulkString(v)) case BulkString: return encodeBulkString(w, v) case []string: return encodeStringArray(w, v) case []interface{}: return encodeArray(w, Array(v)) case Array: return encodeArray(w, v) case nil: return encodeNil(w) default: return ErrInvalidValue } } // encodeStringArray is a specialized array encoding func to avoid having to // allocate an empty slice interface and copy values to it to use encodeArray. func encodeStringArray(w io.Writer, v []string) error { // Special case for a nil array if v == nil { err := encodePrefixed(w, '*', "-1") return err } // First encode the number of elements n := len(v) err := encodePrefixed(w, '*', strconv.Itoa(n)) if err != nil { return err } // Then encode each value for _, el := range v { err = encodeBulkString(w, BulkString(el)) if err != nil { return err } } return nil } // encodeArray encodes an array value to w. func encodeArray(w io.Writer, v Array) error { // Special case for a nil array if v == nil { err := encodePrefixed(w, '*', "-1") return err } // First encode the number of elements n := len(v) err := encodePrefixed(w, '*', strconv.Itoa(n)) if err != nil { return err } // Then encode each value for _, el := range v { err = encodeValue(w, el) if err != nil { return err } } return nil } // encodeBulkString encodes a bulk string to w. func encodeBulkString(w io.Writer, v BulkString) error { n := len(v) data := strconv.Itoa(n) + "\r\n" + string(v) return encodePrefixed(w, '$', data) } // encodeInteger encodes an integer value to w. func encodeInteger(w io.Writer, v int64) error { return encodePrefixed(w, ':', strconv.FormatInt(v, 10)) } // encodeSimpleString encodes a simple string value to w. func encodeSimpleString(w io.Writer, v SimpleString) error { return encodePrefixed(w, '+', string(v)) } // encodeError encodes an error value to w. func encodeError(w io.Writer, v Error) error { return encodePrefixed(w, '-', string(v)) } // encodeNil encodes a nil value as a nil bulk string. func encodeNil(w io.Writer) error { return encodePrefixed(w, '$', "-1") } // encodePrefixed encodes the data v to w, with the specified prefix. func encodePrefixed(w io.Writer, prefix byte, v string) error { buf := make([]byte, len(v)+3) buf[0] = prefix copy(buf[1:], v) copy(buf[len(buf)-2:], "\r\n") _, err := w.Write(buf) return err } mna-redisc-96a8e49/redistest/resp/encode_test.go000066400000000000000000000055421451450242600217220ustar00rootroot00000000000000package resp import ( "bytes" "testing" ) var encodeValidCases = []struct { enc []byte val interface{} err error }{ 0: {[]byte{'+', '\r', '\n'}, SimpleString(""), nil}, 1: {[]byte{'+', 'a', '\r', '\n'}, SimpleString("a"), nil}, 2: {[]byte{'+', 'O', 'K', '\r', '\n'}, SimpleString("OK"), nil}, 3: {[]byte("+ceci n'est pas un string\r\n"), SimpleString("ceci n'est pas un string"), nil}, 4: {[]byte{'-', '\r', '\n'}, Error(""), nil}, 5: {[]byte{'-', 'a', '\r', '\n'}, Error("a"), nil}, 6: {[]byte{'-', 'K', 'O', '\r', '\n'}, Error("KO"), nil}, 7: {[]byte("-ceci n'est pas un string\r\n"), Error("ceci n'est pas un string"), nil}, 8: {[]byte(":1\r\n"), int64(1), nil}, 9: {[]byte(":123\r\n"), int64(123), nil}, 10: {[]byte(":-123\r\n"), int64(-123), nil}, 11: {[]byte("$0\r\n\r\n"), "", nil}, 12: {[]byte("$24\r\nceci n'est pas un string\r\n"), "ceci n'est pas un string", nil}, 13: {[]byte("$51\r\nceci n'est pas un string\r\navec\rdes\nsauts\r\nde\x00ligne.\r\n"), "ceci n'est pas un string\r\navec\rdes\nsauts\r\nde\x00ligne.", nil}, 14: {[]byte("$-1\r\n"), nil, nil}, 15: {[]byte("*0\r\n"), Array{}, nil}, 16: {[]byte("*1\r\n:10\r\n"), Array{int64(10)}, nil}, 17: {[]byte("*-1\r\n"), Array(nil), nil}, 18: {[]byte("*3\r\n+string\r\n-error\r\n:-2345\r\n"), Array{SimpleString("string"), Error("error"), int64(-2345)}, nil}, 19: {[]byte("*5\r\n+string\r\n-error\r\n:-2345\r\n$4\r\nallo\r\n*2\r\n$0\r\n\r\n$-1\r\n"), Array{SimpleString("string"), Error("error"), int64(-2345), "allo", Array{"", nil}}, nil}, } func TestEncode(t *testing.T) { var buf bytes.Buffer var err error for i, c := range encodeValidCases { buf.Reset() err = Encode(&buf, c.val) if err != nil { t.Errorf("%d: got error %s", i, err) continue } if !bytes.Equal(buf.Bytes(), c.enc) { t.Errorf("%d: expected %x (%q), got %x (%q)", i, c.enc, string(c.enc), buf.Bytes(), buf.String()) } } } func BenchmarkEncodeSimpleString(b *testing.B) { var err error var buf bytes.Buffer for i := 0; i < b.N; i++ { err = Encode(&buf, encodeValidCases[3].val) } if err != nil { b.Fatal(err) } } func BenchmarkEncodeError(b *testing.B) { var err error var buf bytes.Buffer for i := 0; i < b.N; i++ { err = Encode(&buf, encodeValidCases[7].val) } if err != nil { b.Fatal(err) } } func BenchmarkEncodeInteger(b *testing.B) { var err error var buf bytes.Buffer for i := 0; i < b.N; i++ { err = Encode(&buf, encodeValidCases[10].val) } if err != nil { b.Fatal(err) } } func BenchmarkEncodeBulkString(b *testing.B) { var err error var buf bytes.Buffer for i := 0; i < b.N; i++ { err = Encode(&buf, encodeValidCases[13].val) } if err != nil { b.Fatal(err) } } func BenchmarkEncodeArray(b *testing.B) { var err error var buf bytes.Buffer for i := 0; i < b.N; i++ { err = Encode(&buf, encodeValidCases[19].val) } if err != nil { b.Fatal(err) } } mna-redisc-96a8e49/redistest/server.go000066400000000000000000000235161451450242600177640ustar00rootroot00000000000000// Package redistest provides test helpers to manage a redis server. package redistest import ( "bufio" "bytes" "fmt" "io" "net" "os/exec" "strconv" "strings" "testing" "time" "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/require" ) // ClusterConfig is the configuration to use for servers started in // redis-cluster mode. The value must contain a single reference to // a string placeholder (%s), the port number. var ClusterConfig = ` port %s cluster-enabled yes cluster-config-file nodes.%[1]s.conf cluster-node-timeout 5000 appendonly no ` // NumClusterNodes is the number of nodes started in a test cluster. // When a cluster is started with replicas, there is 1 replica per // cluster node, so the total number of nodes is NumClusterNodes * 2. const NumClusterNodes = 3 // StartServer starts a redis-server instance on a free port. // It returns the started *exec.Cmd and the port used. The caller // should make sure to stop the command. If the redis-server // command is not found in the PATH, the test is skipped. // // If w is not nil, both stdout and stderr of the server are // written to it. If a configuration is specified, it is supplied // to the server via stdin. func StartServer(t testing.TB, w io.Writer, conf string) (*exec.Cmd, string) { if _, err := exec.LookPath("redis-server"); err != nil { t.Skip("redis-server not found in $PATH") } port := getFreePort(t) return startServerWithConfig(t, port, w, conf), port } // StartClusterWithReplicas starts a redis cluster of NumClusterNodes with // 1 replica each. It returns the cleanup function to call after use // (typically in a defer) and the list of ports for each node, // masters first, then replicas. func StartClusterWithReplicas(t testing.TB, w io.Writer) (func(), []string) { fn, ports := StartCluster(t, w) mapping := getClusterNodeIDs(t, ports...) replicaPorts := make([]string, 0, len(ports)) replicaCmds := make([]*exec.Cmd, 0, len(ports)) replicaMaster := make(map[string]string) for _, master := range ports { port := getClusterFreePort(t) cmd := startServerWithConfig(t, port, w, fmt.Sprintf(ClusterConfig, port)) joinCluster(t, port, master) replicaPorts = append(replicaPorts, port) replicaCmds = append(replicaCmds, cmd) replicaMaster[port] = master } // wait for the cluster to stabilize require.True(t, waitForCluster(t, 10*time.Second, replicaPorts...), "wait for cluster replicas") for _, port := range replicaPorts { setupReplica(t, port, mapping[replicaMaster[port]]) } // wait for replicas to join require.True(t, waitForReplicas(t, 10*time.Second, append(ports, replicaPorts...)...), "wait for cluster replicas") return func() { for _, c := range replicaCmds { _ = c.Process.Kill() } fn() }, append(ports, replicaPorts...) } // StartCluster starts a redis cluster of NumClusterNodes using the // ClusterConfig variable as configuration. If w is not nil, // stdout and stderr of each node will be written to it. // // It returns a function that should be called after the test // (typically in a defer), and the list of ports for all nodes // in the cluster. func StartCluster(t testing.TB, w io.Writer) (func(), []string) { if _, err := exec.LookPath("redis-server"); err != nil { t.Skip("redis-server not found in $PATH") } const hashSlots = 16384 cmds := make([]*exec.Cmd, NumClusterNodes) ports := make([]string, NumClusterNodes) slotsPerNode := hashSlots / NumClusterNodes for i := 0; i < NumClusterNodes; i++ { port := getClusterFreePort(t) cmd := startServerWithConfig(t, port, w, fmt.Sprintf(ClusterConfig, port)) cmds[i], ports[i] = cmd, port // configure the cluster - add the slots and join var meetPort string if i > 0 { meetPort = ports[i-1] } countSlots := slotsPerNode if i == NumClusterNodes-1 { // add all remaining slots in the last node countSlots = hashSlots - (i * slotsPerNode) } setupClusterNode(t, port, i*slotsPerNode, countSlots) if meetPort != "" { joinCluster(t, port, meetPort) } } // wait for the cluster to catch up require.True(t, waitForCluster(t, 10*time.Second, ports...), "wait for cluster") return func() { for _, c := range cmds { _ = c.Process.Kill() } }, ports } //nolint:deadcode,unused func printClusterNodes(t testing.TB, port string) { conn, err := redis.Dial("tcp", ":"+port) require.NoError(t, err, "Dial to cluster node") defer conn.Close() res, err := conn.Do("CLUSTER", "NODES") require.NoError(t, err, "CLUSTER NODES") fmt.Println(string(res.([]byte))) } //nolint:deadcode,unused func printClusterSlots(t testing.TB, port string) { conn, err := redis.Dial("tcp", ":"+port) require.NoError(t, err, "Dial to cluster node") defer conn.Close() cmd := exec.Command("redis-cli", "-p", port, "CLUSTER", "SLOTS") b, err := cmd.CombinedOutput() require.NoError(t, err, "CLUSTER SLOTS via redis-cli") fmt.Println(string(b)) } func joinCluster(t testing.TB, nodePort, clusterPort string) { conn, err := redis.Dial("tcp", ":"+nodePort) require.NoError(t, err, "Dial to node") defer conn.Close() // join the cluster _, err = conn.Do("CLUSTER", "MEET", "127.0.0.1", clusterPort) require.NoError(t, err, "CLUSTER MEET") } func getClusterNodeIDs(t testing.TB, ports ...string) map[string]string { if len(ports) == 0 { return nil } conn, err := redis.Dial("tcp", ":"+ports[0]) require.NoError(t, err, "Dial to node") defer conn.Close() nodes, err := redis.String(conn.Do("CLUSTER", "NODES")) require.NoError(t, err, "CLUSTER NODES") mapping := make(map[string]string) s := bufio.NewScanner(strings.NewReader(nodes)) for s.Scan() { fields := strings.Fields(s.Text()) addrField := fields[1] if ix := strings.Index(addrField, "@"); ix >= 0 { addrField = addrField[:ix] } for _, port := range ports { if addrField == "127.0.0.1:"+port { mapping[port] = fields[0] break } } } require.Equal(t, len(ports), len(mapping), "Find IDs for all ports") return mapping } func setupReplica(t testing.TB, replicaPort, masterID string) { conn, err := redis.Dial("tcp", ":"+replicaPort) require.NoError(t, err, "Dial to replica node") defer conn.Close() _, err = conn.Do("CLUSTER", "REPLICATE", masterID) require.NoError(t, err, "CLUSTER REPLICATE") } func setupClusterNode(t testing.TB, port string, start, count int) { conn, err := redis.Dial("tcp", ":"+port) require.NoError(t, err, "Dial to cluster node") defer conn.Close() args := redis.Args{"ADDSLOTS"} for i := start; i < start+count; i++ { args = args.Add(i) } _, err = conn.Do("CLUSTER", args...) require.NoError(t, err, "CLUSTER ADDSLOTS") } func waitForReplicas(t testing.TB, timeout time.Duration, ports ...string) bool { deadline := time.Now().Add(timeout) for _, port := range ports { conn, err := redis.Dial("tcp", ":"+port) require.NoError(t, err, "Dial") for time.Now().Before(deadline) { v, err := redis.String(conn.Do("CLUSTER", "NODES")) require.NoError(t, err, "CLUSTER NODES") ms, rs := 0, 0 s := bufio.NewScanner(strings.NewReader(v)) for s.Scan() { fields := strings.Fields(s.Text()) if fields[7] == "connected" { if strings.Contains(fields[2], "master") { ms++ } else { rs++ } } } if ms == NumClusterNodes && rs == NumClusterNodes { break } time.Sleep(100 * time.Millisecond) } conn.Close() if time.Now().After(deadline) { return false } } return true } func waitForCluster(t testing.TB, timeout time.Duration, ports ...string) bool { deadline := time.Now().Add(timeout) for _, port := range ports { conn, err := redis.Dial("tcp", ":"+port) require.NoError(t, err, "Dial") for time.Now().Before(deadline) { vals, err := redis.Bytes(conn.Do("CLUSTER", "INFO")) require.NoError(t, err, "CLUSTER INFO") if bytes.Contains(vals, []byte("cluster_state:ok")) { break } time.Sleep(100 * time.Millisecond) } conn.Close() if time.Now().After(deadline) { return false } } return true } func startServerWithConfig(t testing.TB, port string, w io.Writer, conf string) *exec.Cmd { var args []string if conf == "" { args = []string{"--port", port} } else { args = []string{"-"} } c := exec.Command("redis-server", args...) c.Dir = t.TempDir() if w != nil { c.Stderr = w c.Stdout = w } if conf != "" { c.Stdin = strings.NewReader(conf) } // start the server require.NoError(t, c.Start(), "start redis-server") // wait for the server to start accepting connections require.True(t, waitForPort(port, 10*time.Second), "wait for redis-server") t.Logf("redis-server started on port %s", port) return c } func waitForPort(port string, timeout time.Duration) bool { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { conn, err := net.DialTimeout("tcp", ":"+port, time.Second) if err == nil { conn.Close() return true } time.Sleep(10 * time.Millisecond) } return false } func getClusterFreePort(t testing.TB) string { const maxPort = 55535 // the port number in a redis-cluster must be below 55535 because // the nodes communicate with others on port p+10000. Try to get // lucky and subtract 10000 from the random port received if it // is too high. port := getFreePort(t) if n, _ := strconv.Atoi(port); n >= maxPort { port = strconv.Itoa(n - 10000) } return port } func getFreePort(t testing.TB) string { l, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err, "listen on port 0") defer l.Close() _, p, err := net.SplitHostPort(l.Addr().String()) require.NoError(t, err, "parse host and port") return p } // NewPool creates a redis pool to return connections on the specified // addr. func NewPool(_ testing.TB, addr string) *redis.Pool { return &redis.Pool{ MaxIdle: 2, MaxActive: 10, IdleTimeout: time.Minute, Dial: func() (redis.Conn, error) { return redis.Dial("tcp", addr) }, TestOnBorrow: func(c redis.Conn, t time.Time) error { _, err := c.Do("PING") return err }, } } mna-redisc-96a8e49/retry_conn.go000066400000000000000000000101511451450242600166210ustar00rootroot00000000000000package redisc import ( "errors" "time" "github.com/gomodule/redigo/redis" ) // RetryConn wraps the connection c (which must be a *redisc.Conn) into a // connection that automatically handles cluster redirections (MOVED and ASK // replies) and retries for TRYAGAIN errors. Only Do, Close and Err can be // called on that connection, all other methods return an error. // // The maxAtt parameter indicates the maximum number of attempts to // successfully execute the command. The tryAgainDelay is the duration to wait // before retrying a TRYAGAIN error. // // The only case where it returns a non-nil error is if c is not a // *redisc.Conn. func RetryConn(c redis.Conn, maxAtt int, tryAgainDelay time.Duration) (redis.Conn, error) { cc, ok := c.(*Conn) if !ok { return nil, errors.New("redisc: connection is not a *Conn") } return &retryConn{c: cc, maxAttempts: maxAtt, tryAgainDelay: tryAgainDelay}, nil } type retryConn struct { c *Conn maxAttempts int // immutable tryAgainDelay time.Duration // immutable } func (rc *retryConn) Do(cmd string, args ...interface{}) (interface{}, error) { return rc.do(cmd, args...) } func (rc *retryConn) do(cmd string, args ...interface{}) (interface{}, error) { var att int var asking bool cluster := rc.c.cluster for rc.maxAttempts <= 0 || att < rc.maxAttempts { if asking { if err := rc.c.Send("ASKING"); err != nil { return nil, err } asking = false } v, err := rc.c.Do(cmd, args...) re := ParseRedir(err) if re == nil { if IsTryAgain(err) { // handle retry time.Sleep(rc.tryAgainDelay) att++ continue } // not a retry error nor a redirection, return result return v, err } // handle redirection rc.c.mu.Lock() readOnly := rc.c.readOnly connAddr := rc.c.boundAddr rc.c.mu.Unlock() if readOnly { // check if the connection was already made to that slot, meaning that // the redirection is because the command can't be served by the replica // and a non-readonly connection must be made to the slot's master. If // that's not the case, then keep the readonly flag to true, meaning that // it will attempt a connection // to a replica for the new slot. cluster.mu.Lock() slotMappings := cluster.mapping[re.NewSlot] cluster.mu.Unlock() if isIn(slotMappings, connAddr) { readOnly = false } } var conn redis.Conn addr := re.Addr asking = re.Type == "ASK" if asking { // if redirecting due to ASK, use the address that was provided in the // ASK error reply. conn, err = cluster.getConnForAddr(addr, rc.c.forceDial) if err != nil { return nil, err } // TODO(mna): does redis cluster send ASK replies that redirect to // replicas if the source node was a replica? Assume no for now. readOnly = false } else { // if redirecting due to a MOVED, the slot mapping is already updated to // reflect the new server for that slot (done in rc.c.Do), so // getConnForSlot will return a connection to the correct address. conn, addr, err = cluster.getConnForSlot(re.NewSlot, rc.c.forceDial, readOnly) if err != nil { // could not get connection to that node, return that error return nil, err } } var cerr error rc.c.mu.Lock() // close and replace the old connection (close must come before assignments) cerr = rc.c.closeLocked() rc.c.rc = conn rc.c.boundAddr = addr rc.c.readOnly = readOnly rc.c.mu.Unlock() if cerr != nil && cluster.BgError != nil { go cluster.BgError(RetryCloseConn, cerr) } att++ } return nil, errors.New("redisc: too many attempts") } func (rc *retryConn) Err() error { return rc.c.Err() } func (rc *retryConn) Close() error { return rc.c.Close() } func (rc *retryConn) Send(_ string, _ ...interface{}) error { return errors.New("redisc: unsupported call to Send") } func (rc *retryConn) Receive() (interface{}, error) { return nil, errors.New("redisc: unsupported call to Receive") } func (rc *retryConn) Flush() error { return errors.New("redisc: unsupported call to Flush") } func isIn(list []string, v string) bool { for _, vv := range list { if v == vv { return true } } return false } mna-redisc-96a8e49/retry_conn_test.go000066400000000000000000000231241451450242600176640ustar00rootroot00000000000000package redisc import ( "net" "strconv" "strings" "sync/atomic" "testing" "time" "github.com/gomodule/redigo/redis" "github.com/mna/redisc/redistest" "github.com/mna/redisc/redistest/resp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestRetryConnAsk(t *testing.T) { var s *redistest.MockServer var asking int32 // test a RetryConn that receves an ASK error, but that redirects // to the same address (only to validate that ASKING is properly // sent first). s = redistest.StartMockServer(t, func(cmd string, args ...string) interface{} { switch cmd { case "CLUSTER": addr, port, _ := net.SplitHostPort(s.Addr) nPort, _ := strconv.Atoi(port) // reply that all slots are served by this server return resp.Array{ 0: resp.Array{0: int64(0), 1: int64(HashSlots - 1), 2: resp.Array{0: addr, 1: int64(nPort)}}, } case "GET": // if asking wasn't sent first, reply with ASK for slot 1234 to // the same current address. if atomic.LoadInt32(&asking) == 0 { return resp.Error("ASK 1234 " + s.Addr) } // if asking was sent, reply with OK return "ok" case "ASKING": // record that ASKING was sent atomic.AddInt32(&asking, 1) return nil } return resp.Error("unexpected command " + cmd) }) defer s.Close() c := &Cluster{ StartupNodes: []string{s.Addr}, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") conn := c.Get() defer conn.Close() _, err := conn.Do("GET", "x") if assert.Error(t, err, "GET without retry") { re := ParseRedir(err) if assert.NotNil(t, re, "ParseRedir") { assert.Equal(t, "ASK", re.Type, "ASK") } } rc, err := RetryConn(conn, 3, time.Second) require.NoError(t, err, "RetryConn") v, err := rc.Do("GET", "x") if assert.NoError(t, err, "GET with retry") { assert.Equal(t, []byte("ok"), v, "expected result") } } func TestRetryConnAskDistinctServers(t *testing.T) { var s1, s2 *redistest.MockServer var asking int32 // all slots are served by server s1, but simulate a migration on a slot // and reply with ASK to s2, and make sure that s2 received the ASKING // and subsequent command. s1 = redistest.StartMockServer(t, func(cmd string, args ...string) interface{} { switch cmd { case "CLUSTER": addr, port, _ := net.SplitHostPort(s1.Addr) nPort, _ := strconv.Atoi(port) // reply that all slots are served by this server return resp.Array{ 0: resp.Array{0: int64(0), 1: int64(HashSlots - 1), 2: resp.Array{0: addr, 1: int64(nPort)}}, } case "GET": // reply with ASK redirection return resp.Error("ASK 1234 " + s2.Addr) } return resp.Error("unexpected command " + cmd) }) defer s1.Close() s2 = redistest.StartMockServer(t, func(cmd string, args ...string) interface{} { switch cmd { case "GET": return "ok" case "ASKING": // record that ASKING was sent atomic.AddInt32(&asking, 1) return nil } return resp.Error("unexpected command " + cmd) }) defer s2.Close() c := &Cluster{ StartupNodes: []string{s1.Addr}, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") conn := c.Get() defer conn.Close() rc, err := RetryConn(conn, 3, time.Second) require.NoError(t, err, "RetryConn") v, err := rc.Do("GET", "x") if assert.NoError(t, err, "GET with retry") { assert.Equal(t, []byte("ok"), v, "expected result") assert.Equal(t, int32(1), atomic.LoadInt32(&asking)) } } func TestRetryConnTryAgain(t *testing.T) { var s *redistest.MockServer var tryagain int32 s = redistest.StartMockServer(t, func(cmd string, args ...string) interface{} { switch cmd { case "CLUSTER": addr, port, _ := net.SplitHostPort(s.Addr) nPort, _ := strconv.Atoi(port) return resp.Array{ 0: resp.Array{0: int64(0), 1: int64(16383), 2: resp.Array{0: addr, 1: int64(nPort)}}, } case "GET": if atomic.LoadInt32(&tryagain) < 2 { atomic.AddInt32(&tryagain, 1) return resp.Error("TRYAGAIN") } return "ok" } return resp.Error("unexpected command " + cmd) }) defer s.Close() c := &Cluster{ StartupNodes: []string{s.Addr}, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") conn := c.Get() defer conn.Close() _, err := conn.Do("GET", "x") if assert.Error(t, err, "GET without retry") { assert.True(t, IsTryAgain(err), "IsTryAgain") } rc, err := RetryConn(conn, 3, 1*time.Millisecond) require.NoError(t, err, "RetryConn") v, err := rc.Do("GET", "x") if assert.NoError(t, err, "GET with retry") { assert.Equal(t, []byte("ok"), v, "expected result") } } func TestRetryConnErrs(t *testing.T) { c := &Cluster{ StartupNodes: []string{":6379"}, } defer c.Close() conn := c.Get() require.NoError(t, conn.Close(), "Close") rc, err := RetryConn(conn, 3, time.Second) require.NoError(t, err, "RetryConn") _, err = rc.Do("A") assert.Error(t, err, "Do after Close") assert.Error(t, rc.Err(), "Err after Close") assert.Error(t, rc.Flush(), "Flush") _, err = rc.Receive() assert.Error(t, err, "Receive") assert.Error(t, rc.Send("A"), "Send") assert.Error(t, rc.Close(), "Close after Close") _, err = RetryConn(rc, 3, time.Second) // RetryConn, but conn is not a *Conn assert.Error(t, err, "RetryConn with a non-*Conn") } func testRetryConnTooManyAttempts(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: ports, DialOptions: []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)}, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") // create a connection and bind to key "a" conn := c.Get() defer conn.Close() require.NoError(t, conn.(*Conn).Bind("a"), "Bind") // wrap it in a RetryConn with a single attempt allowed rc, err := RetryConn(conn, 1, 100*time.Millisecond) require.NoError(t, err, "RetryConn") _, err = rc.Do("SET", "b", "x") if assert.Error(t, err, "SET b") { assert.Contains(t, err.Error(), "too many attempts") } } func testRetryConnMoved(t *testing.T, ports []string) { c := &Cluster{ StartupNodes: ports, DialOptions: []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)}, } defer c.Close() require.NoError(t, c.Refresh(), "Refresh") // create a connection and bind to key "a" conn := c.Get() defer conn.Close() require.NoError(t, conn.(*Conn).Bind("a"), "Bind") // cluster's mapping for "a" should be 15495, "b" is 3300, check that // the MOVED did update the mapping of "b", and did not touch "a" c.mu.Lock() addrA := c.mapping[15495] addrB := c.mapping[3300] c.mapping[3300] = []string{"x"} c.mu.Unlock() // set key "b", which is on a different node (generates a MOVED) - this is NOT a RetryConn _, err := conn.Do("SET", "b", "x") if assert.Error(t, err, "SET b") { re := ParseRedir(err) if assert.NotNil(t, re, "ParseRedir") { assert.Equal(t, "MOVED", re.Type, "Redir type") } } // cluster updated its mapping even though it did not follow the redirection c.mu.Lock() assert.Equal(t, addrA, c.mapping[15495], "Addr A") assert.Equal(t, addrB, c.mapping[3300], "Sentinel value B") c.mapping[3300] = []string{"x"} c.mu.Unlock() // now wrap it in a RetryConn rc, err := RetryConn(conn, 3, 100*time.Millisecond) require.NoError(t, err, "RetryConn") _, err = rc.Do("SET", "b", "x") assert.NoError(t, err, "SET b") // the cluster should've updated its mapping c.mu.Lock() assert.Equal(t, addrA, c.mapping[15495], "Addr A") assert.Equal(t, addrB, c.mapping[3300], "Addr B") c.mu.Unlock() v, err := redis.String(rc.Do("GET", "b")) if assert.NoError(t, err, "GET b") { assert.Equal(t, "x", v, "GET value") } } func testRetryConnTriggerRefreshes(t *testing.T, ports []string) { var count int64 done := make(chan bool, 1) c := &Cluster{ StartupNodes: []string{ports[0]}, DialOptions: []redis.DialOption{redis.DialConnectTimeout(2 * time.Second)}, CreatePool: createPool, LayoutRefresh: func(old, new [HashSlots][]string) { atomic.AddInt64(&count, 1) select { case done <- true: default: } }, } defer c.Close() conn := c.Get() conn, _ = RetryConn(conn, 3, 100*time.Millisecond) defer conn.Close() // set keys from different slots served by different servers // (a=15495, b=3300, abc=7638). _, err := conn.Do("SET", "a", 1) assert.NoError(t, err, "SET a") _, err = conn.Do("SET", "b", 2) assert.NoError(t, err, "SET b") _, err = conn.Do("SET", "abc", 3) assert.NoError(t, err, "SET abc") _, err = conn.Do("INCR", "a") assert.NoError(t, err, "INCR a") _, err = conn.Do("INCR", "b") assert.NoError(t, err, "INCR b") _, err = conn.Do("INCR", "abc") assert.NoError(t, err, "INCR abc") v, err := redis.Int(conn.Do("GET", "a")) if assert.NoError(t, err, "GET a") { assert.Equal(t, 2, v) } v, err = redis.Int(conn.Do("GET", "b")) if assert.NoError(t, err, "GET b") { assert.Equal(t, 3, v) } v, err = redis.Int(conn.Do("GET", "abc")) if assert.NoError(t, err, "GET abc") { assert.Equal(t, 4, v) } // return the conn to the pool assert.NoError(t, conn.Close(), "Close conn") waitForClusterRefresh(c, nil) <-done // only the first command triggered a refresh, the rest were all known assert.Equal(t, 1, int(atomic.LoadInt64(&count))) stats := c.Stats() assert.Len(t, stats, 3) var inuse, idle int for _, st := range stats { inuse += st.ActiveCount - st.IdleCount idle += st.IdleCount } assert.Equal(t, 0, inuse, "connections in use") assert.Equal(t, len(ports), idle, "idle connections in pools") // verify the connections count from the servers var clients int err = c.EachNode(false, func(addr string, conn redis.Conn) error { s, err := redis.String(conn.Do("CLIENT", "LIST", "TYPE", "normal")) if err != nil { return err } clients += strings.Count(s, "\n") return nil }) require.NoError(t, err) assert.Equal(t, idle, clients, "server-reported clients count") }