pax_global_header00006660000000000000000000000064151721404440014514gustar00rootroot0000000000000052 comment=aaf9e2e852c936705d731b94571f50cc26319308 redis-rb-redis-cluster-client-aaf9e2e/000077500000000000000000000000001517214044400200405ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/.github/000077500000000000000000000000001517214044400214005ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/.github/workflows/000077500000000000000000000000001517214044400234355ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/.github/workflows/redis-cluster-proxy.yaml000066400000000000000000000024251517214044400302700ustar00rootroot00000000000000--- # @see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions name: Redis Cluster Proxy on: push: branches: - "master" paths: - test/proxy/redis-cluster-proxy/** - .github/workflows/redis-cluster-proxy.yaml defaults: run: shell: bash jobs: container-image: name: Container Image if: github.repository == 'redis-rb/redis-cluster-client' timeout-minutes: 15 runs-on: ubuntu-latest concurrency: redis-cluster-proxy permissions: packages: write defaults: run: working-directory: test/proxy/redis-cluster-proxy env: IMAGE_NAME: redis-cluster-proxy steps: - name: Check out code uses: actions/checkout@v3 - name: Build image run: | docker build . --tag $IMAGE_NAME - name: Log into GitHub Container Registry run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin - name: Push image to GitHub Container Registry run: | IMAGE_ID=$(echo "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME" | tr '[A-Z]' '[a-z]') VERSION=latest docker tag $IMAGE_NAME $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION redis-rb-redis-cluster-client-aaf9e2e/.github/workflows/release.yaml000066400000000000000000000012011517214044400257330ustar00rootroot00000000000000--- # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions # https://github.com/actions/virtual-environments name: Release on: push: tags: - "v*" concurrency: ${{ github.workflow }} jobs: gem: name: Gem if: github.repository == 'redis-rb/redis-cluster-client' timeout-minutes: 10 runs-on: ubuntu-latest permissions: id-token: write contents: read steps: - uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: ruby bundler-cache: true - uses: rubygems/release-gem@v1 redis-rb-redis-cluster-client-aaf9e2e/.github/workflows/test.yaml000066400000000000000000000246641517214044400253140ustar00rootroot00000000000000--- # @see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions # # Services feature of GitHub Actions isn't fit for our purposes for testing. # We cannot overwrite arguments of ENTRYPOINT. # @see https://docs.docker.com/engine/reference/commandline/create/#options # # @see https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 name: Test permissions: contents: read on: push: branches: - master pull_request: branches: - master schedule: - cron: '0 9,21 * * *' defaults: run: shell: bash jobs: main: name: Main if: github.event_name != 'schedule' || github.repository == 'redis-rb/redis-cluster-client' timeout-minutes: 15 runs-on: ubuntu-latest strategy: fail-fast: false max-parallel: 8 matrix: include: - {redis: '8', ruby: '4.0'} - {redis: '8', ruby: '4.0', compose: compose.ssl.yaml} - {redis: '8', ruby: '4.0', driver: 'hiredis'} - {redis: '8', ruby: '4.0', driver: 'hiredis', compose: compose.ssl.yaml} - {redis: '8', ruby: '4.0', compose: compose.auth.yaml} - {redis: '8', ruby: '4.0', compose: compose.replica.yaml, replica: '2'} - {redis: '9', ruby: '4.0', compose: compose.valkey.yaml, replica: '2'} - {redis: '8', ruby: '4.0', compose: compose.valkey.yaml, replica: '2'} - {redis: '7', ruby: '3.4'} - {redis: '6', ruby: '3.3'} - {redis: '5', ruby: '2.7'} - {ruby: 'jruby'} - {ruby: 'truffleruby'} - {task: test_cluster_down} - {task: test_cluster_broken, restart: 'no', startup: '6'} - {task: test_cluster_state, pattern: 'PrimaryOnly', compose: compose.valkey.yaml, redis: '9', replica: '2', startup: '9'} - {task: test_cluster_state, pattern: 'Pooled', compose: compose.valkey.yaml, redis: '9', replica: '2', startup: '9'} - {task: test_cluster_state, pattern: 'ScaleReadRandom', compose: compose.valkey.yaml, redis: '9', replica: '2', startup: '9'} - {task: test_cluster_state, pattern: 'ScaleReadRandomWithPrimary', compose: compose.valkey.yaml, redis: '9', replica: '2', startup: '9'} - {task: test_cluster_state, pattern: 'ScaleReadLatency', compose: compose.valkey.yaml, redis: '9', replica: '2', startup: '9'} - {task: test_cluster_scale, pattern: 'Single', compose: compose.scale.yaml, startup: '8'} - {task: test_cluster_scale, pattern: 'Pipeline', compose: compose.scale.yaml, startup: '8'} - {task: test_cluster_scale, pattern: 'Transaction', compose: compose.scale.yaml, startup: '8'} - {task: test_cluster_scale, pattern: 'PubSub', compose: compose.scale.yaml, startup: '8'} env: REDIS_VERSION: ${{ matrix.redis || '8' }} DOCKER_COMPOSE_FILE: ${{ matrix.compose || 'compose.yaml' }} REDIS_CONNECTION_DRIVER: ${{ matrix.driver || 'ruby' }} REDIS_REPLICA_SIZE: ${{ matrix.replica || '1' }} RESTART_POLICY: ${{ matrix.restart || 'always' }} REDIS_CLIENT_MAX_STARTUP_SAMPLE: ${{ matrix.startup || '3' }} TEST_CLASS_PATTERN: ${{ matrix.pattern || '' }} steps: - name: Check out code uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby || '4.0' }} bundler-cache: true - name: Pull Docker images run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull - name: Run containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d - name: Wait for Redis cluster to be ready run: bundle exec rake wait - name: Print containers run: docker compose -f $DOCKER_COMPOSE_FILE ps - name: Run minitest run: bundle exec rake ${{ matrix.task || 'test' }} - name: Stop containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true lint: name: Lint if: github.event_name != 'schedule' || github.repository == 'redis-rb/redis-cluster-client' timeout-minutes: 5 runs-on: ubuntu-latest steps: - name: Check out code uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: '4.0' bundler-cache: true - name: Run rubocop run: bundle exec rubocop nat-ted-env: if: github.event_name == 'schedule' && github.repository == 'redis-rb/redis-cluster-client' name: NAT-ted Environments timeout-minutes: 5 runs-on: ubuntu-latest env: REDIS_VERSION: '8' DOCKER_COMPOSE_FILE: 'compose.nat.yaml' steps: - name: Check out code uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: '4.0' bundler-cache: true - name: Get IP address of host run: | host_ip_addr=$(ip a | grep eth0 | grep inet | awk '{print $2}' | cut -d'/' -f1) echo "HOST_IP_ADDR=$host_ip_addr" >> $GITHUB_ENV - name: Pull Docker images run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull - name: Run containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up --wait --wait-timeout 30 env: HOST_ADDR: ${{ env.HOST_IP_ADDR }} - name: Print containers run: docker compose -f $DOCKER_COMPOSE_FILE ps - name: Build cluster run: bundle exec rake "build_cluster[$HOST_ADDR]" env: HOST_ADDR: ${{ env.HOST_IP_ADDR }} DEBUG: "1" - name: Run minitest run: bundle exec rake test - name: Stop containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true ips: if: github.event_name == 'schedule' && github.repository == 'redis-rb/redis-cluster-client' name: IPS timeout-minutes: 10 runs-on: ubuntu-latest env: REDIS_VERSION: '8' DOCKER_COMPOSE_FILE: 'compose.latency.yaml' REDIS_REPLICA_SIZE: '2' REDIS_CLIENT_MAX_THREADS: '10' DELAY_TIME: '0ms' steps: - name: Check out code uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: '4.0' bundler-cache: true - name: Pull Docker images run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull - name: Run containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d - name: Wait for Redis cluster to be ready run: bundle exec rake wait - name: Print containers run: docker compose -f $DOCKER_COMPOSE_FILE ps - name: Print cpu info run: grep 'model name' /proc/cpuinfo - name: Run iteration per second run: bundle exec rake ips - name: Stop containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true profiling: if: github.event_name == 'schedule' && github.repository == 'redis-rb/redis-cluster-client' name: Profiling timeout-minutes: 5 runs-on: ubuntu-latest strategy: fail-fast: false matrix: mode: - single - excessive_pipelining - pipelining_in_moderation - original_mget - emulated_mget env: REDIS_VERSION: '8' DOCKER_COMPOSE_FILE: 'compose.yaml' steps: - name: Check out code uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: '4.0' bundler-cache: true - name: Pull Docker images run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull - name: Run containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d - name: Wait for Redis cluster to be ready run: bundle exec rake wait - name: Print containers run: docker compose -f $DOCKER_COMPOSE_FILE ps - name: Run profiler run: bundle exec rake prof env: PROFILE_MODE: ${{ matrix.mode }} - name: Stop containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true massive: if: github.event_name == 'schedule' && github.repository == 'redis-rb/redis-cluster-client' name: Massive Cluster timeout-minutes: 10 runs-on: ubuntu-latest env: REDIS_VERSION: '8' DOCKER_COMPOSE_FILE: 'compose.massive.yaml' REDIS_SHARD_SIZE: '10' REDIS_REPLICA_SIZE: '2' REDIS_CLIENT_MAX_STARTUP_SAMPLE: '5' steps: - name: Check out code uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: '4.0' bundler-cache: true - name: Print user limits run: ulimit -a - name: Print kernel params run: | sysctl fs.file-max sysctl vm.swappiness sysctl vm.overcommit_memory sysctl net.ipv4.tcp_sack sysctl net.ipv4.tcp_timestamps sysctl net.ipv4.tcp_window_scaling sysctl net.ipv4.tcp_congestion_control sysctl net.ipv4.tcp_syncookies sysctl net.ipv4.tcp_tw_reuse sysctl net.ipv4.tcp_max_syn_backlog sysctl net.core.somaxconn sysctl net.core.rmem_max sysctl net.core.wmem_max - name: Tune kernel params for redis run: | # https://developer.redis.com/operate/redis-at-scale/talking-to-redis/initial-tuning/ sudo sysctl -w vm.overcommit_memory=1 sudo sysctl -w net.ipv4.tcp_tw_reuse=1 # reuse sockets quickly sudo sysctl -w net.ipv4.tcp_max_syn_backlog=1024 # backlog setting sudo sysctl -w net.core.somaxconn=1024 # up the number of connections per port - name: Pull Docker images run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE pull - name: Run containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE up -d - name: Print memory info run: free -w - name: Wait for Redis cluster to be ready run: bundle exec rake wait env: DEBUG: '1' - name: Print containers run: docker compose -f $DOCKER_COMPOSE_FILE ps - name: Run profiler run: bundle exec rake prof env: PROFILE_MODE: pipelining_in_moderation - name: Stop containers run: docker compose --progress quiet -f $DOCKER_COMPOSE_FILE down || true redis-rb-redis-cluster-client-aaf9e2e/.gitignore000066400000000000000000000000421517214044400220240ustar00rootroot00000000000000.bundle *.gem *.json Gemfile.lock redis-rb-redis-cluster-client-aaf9e2e/.rubocop.yml000066400000000000000000000016551517214044400223210ustar00rootroot00000000000000--- plugins: - rubocop-performance - rubocop-rake - rubocop-minitest # https://rubocop.readthedocs.io/en/latest/ AllCops: TargetRubyVersion: 2.7 DisplayCopNames: true NewCops: disable Metrics/AbcSize: Exclude: - 'test/**/*' Metrics/CyclomaticComplexity: Exclude: - 'test/**/*' Metrics/PerceivedComplexity: Exclude: - 'test/**/*' Metrics/ClassLength: Max: 500 Metrics/ModuleLength: Max: 500 Exclude: - 'test/**/*' Metrics/MethodLength: Max: 50 Exclude: - 'test/**/*' Metrics/BlockLength: Max: 40 Exclude: - 'test/**/*' Metrics/ParameterLists: Max: 10 Layout/LineLength: Max: 200 Layout/MultilineMethodCallIndentation: Enabled: false Style/NumericPredicate: Enabled: false Style/FormatStringToken: Enabled: false Style/Documentation: Enabled: false Gemspec/RequiredRubyVersion: Enabled: false Naming/FileName: Exclude: - 'lib/redis-cluster-client.rb' redis-rb-redis-cluster-client-aaf9e2e/Gemfile000066400000000000000000000010261517214044400213320ustar00rootroot00000000000000# frozen_string_literal: true source 'https://rubygems.org' gemspec name: 'redis-cluster-client' gem 'async-redis', platform: :mri gem 'benchmark' gem 'benchmark-ips' gem 'hiredis-client', '~> 0.6' gem 'irb' gem 'logger' gem 'memory_profiler' gem 'minitest' gem 'rake' gem 'rubocop' gem 'rubocop-minitest', require: false gem 'rubocop-performance', require: false gem 'rubocop-rake', require: false gem 'stackprof', platform: :mri gem 'vernier', platform: :mri if RUBY_ENGINE == 'ruby' && Integer(RUBY_VERSION.split('.').first) > 2 redis-rb-redis-cluster-client-aaf9e2e/LICENSE000066400000000000000000000020511517214044400210430ustar00rootroot00000000000000MIT License Copyright (c) 2022 redis-rb Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. redis-rb-redis-cluster-client-aaf9e2e/README.md000066400000000000000000000355211517214044400213250ustar00rootroot00000000000000[![Gem Version](https://badge.fury.io/rb/redis-cluster-client.svg)](https://badge.fury.io/rb/redis-cluster-client) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/redis-rb/redis-cluster-client) ![Test status](https://github.com/redis-rb/redis-cluster-client/actions/workflows/test.yaml/badge.svg?branch=master) ![Release status](https://github.com/redis-rb/redis-cluster-client/actions/workflows/release.yaml/badge.svg) Redis Cluster Client =============================================================================== This library is a client for [Redis cluster](https://redis.io/docs/reference/cluster-spec/). It depends on [redis-client](https://github.com/redis-rb/redis-client). So it would be better to read `redis-client` documents first. ## Background This gem underlies the official gem [redis-clustering](https://rubygems.org/gems/redis-clustering). The redis-clustering gem was decoupled from [the redis gem](https://rubygems.org/gems/redis) as of `v5`. Both are maintained by [the repository](https://github.com/redis/redis-rb) in the official organization. The redis gem supported cluster mode since [the pull request](https://github.com/redis/redis-rb/pull/716) was merged until `v4`. You can see more details and reasons in [the issue](https://github.com/redis/redis-rb/issues/1070) if you are interested. ## Installation ```ruby gem 'redis-cluster-client' ``` ## Initialization | key | type | default | description | | --- | --- | --- | --- | | `:nodes` | String or Hash or Array | `['redis://127.0.0.1:6379']` | node addresses for startup connection | | `:replica` | Boolean | `false` | `true` if client should use scale read feature | | `:replica_affinity` | Symbol or String | `:random` | scale reading strategy, `:random`, `random_with_primary` or `:latency` are valid | | `:fixed_hostname` | String | `nil` | required if client should connect to single endpoint with SSL | | `:slow_command_timeout` | Integer | `-1` | timeout used for slow commands that fetch metadata, e.g. CLUSTER SHARDS, COMMAND | | `:concurrency` | Hash | `{ model: :none }` | concurrency settings, `:on_demand`, `:pooled` and `:none` are valid models, size is a max number of workers, `:none` model is no concurrency, Please choose the one suited to your environment if needed. | | `:connect_with_original_config` | Boolean | `false` | `true` if client should retry the connection using the original endpoint that was passed in | | `:max_startup_sample` | Integer | `3` | maximum number of nodes to fetch `CLUSTER SHARDS` information for startup | Also, [the other generic options](https://github.com/redis-rb/redis-client#configuration) can be passed. But `:url`, `:host`, `:port` and `:path` are ignored because they conflict with the `:nodes` option. ```ruby require 'redis_cluster_client' # The following examples are Docker containers on localhost. # The client first attempts to connect to redis://127.0.0.1:6379 internally. # To connect to primary nodes only RedisClient.cluster.new_client #=> # # To connect to all nodes to use scale reading feature RedisClient.cluster(replica: true).new_client #=> # # To connect to all nodes to use scale reading feature + make reads equally likely from replicas and primary RedisClient.cluster(replica: true, replica_affinity: :random_with_primary).new_client #=> # # To connect to all nodes to use scale reading feature prioritizing low-latency replicas RedisClient.cluster(replica: true, replica_affinity: :latency).new_client #=> # # With generic options for redis-client RedisClient.cluster(timeout: 3.0).new_client ``` ```ruby # To connect with a subset of nodes for startup RedisClient.cluster(nodes: %w[redis://node1:6379 redis://node2:6379]).new_client ``` ```ruby # To connect with a subset of auth-needed nodes for startup ## with URL: ### User name and password should be URI encoded and the same in every node. username = 'myuser' password = URI.encode_www_form_component('!&<123-abc>') RedisClient.cluster(nodes: %W[redis://#{username}:#{password}@node1:6379 redis://#{username}:#{password}@node2:6379]).new_client ## with options: RedisClient.cluster(nodes: %w[redis://node1:6379 redis://node2:6379], username: 'myuser', password: '!&<123-abc>').new_client ``` ```ruby # To connect to single endpoint RedisClient.cluster(nodes: 'redis://endpoint.example.com:6379').new_client ``` ```ruby # To connect to single endpoint with SSL/TLS (such as Amazon ElastiCache for Redis) RedisClient.cluster(nodes: 'rediss://endpoint.example.com:6379').new_client ``` ```ruby # To connect to NAT-ted endpoint with SSL/TLS (such as Microsoft Azure Cache for Redis) RedisClient.cluster(nodes: 'rediss://endpoint.example.com:6379', fixed_hostname: 'endpoint.example.com').new_client ``` ```ruby # To specify a timeout for "slow" commands (CLUSTER SHARDS, COMMAND) RedisClient.cluster(slow_command_timeout: 4).new_client ``` ```ruby # To specify concurrency settings RedisClient.cluster(concurrency: { model: :on_demand, size: 6 }).new_client RedisClient.cluster(concurrency: { model: :pooled, size: 3 }).new_client RedisClient.cluster(concurrency: { model: :none }).new_client # The above settings are used by sending commands to multiple nodes like pipelining. # Please choose the one suited your workloads. ``` ```ruby # To reconnect using the original configuration options on error. This can be useful when using a DNS endpoint and the underlying host IPs are all updated RedisClient.cluster(connect_with_original_config: true).new_client ``` ## Interfaces The following methods are able to be used like `redis-client`. * `#call` * `#call_v` * `#call_once` * `#call_once_v` * `#blocking_call` * `#blocking_call_v` * `#scan` * `#sscan` * `#hscan` * `#zscan` * `#pipelined` * `#multi` * `#pubsub` * `#close` The `#scan` method iterates all keys around every node seamlessly. The `#pipelined` method splits and sends commands to each node and aggregates replies. The `#multi` method supports the transaction feature but you should use a hashtag for your keys. The `#pubsub` method supports sharded subscriptions. Every interface handles redirections and resharding states internally. ## Multiple keys and CROSSSLOT error A subset of commands can be passed multiple keys. In cluster mode, these commands have a constraint that passed keys should belong to the same slot and not just the same node. Therefore, the following error occurs: ``` $ redis-cli -c mget key1 key2 key3 (error) CROSSSLOT Keys in request don't hash to the same slot $ redis-cli -c cluster keyslot key1 (integer) 9189 $ redis-cli -c cluster keyslot key2 (integer) 4998 $ redis-cli -c cluster keyslot key3 (integer) 935 ``` For the constraint, Redis cluster provides a feature to be able to bias keys to the same slot with a hash tag. ``` $ redis-cli -c mget {key}1 {key}2 {key}3 1) (nil) 2) (nil) 3) (nil) $ redis-cli -c cluster keyslot {key}1 (integer) 12539 $ redis-cli -c cluster keyslot {key}2 (integer) 12539 $ redis-cli -c cluster keyslot {key}3 (integer) 12539 ``` In addition, this gem handles multiple keys without a hash tag in MGET, MSET and DEL commands using pipelining internally automatically. If the first key includes a hash tag, this gem sends the command to the node as is. If the first key doesn't have a hash tag, this gem converts the command into single-key commands and sends them to nodes with pipelining, then gathering replies and returning them. ```ruby r = RedisClient.cluster.new_client #=> # r.call('mget', 'key1', 'key2', 'key3') #=> [nil, nil, nil] r.call('mget', '{key}1', '{key}2', '{key}3') #=> [nil, nil, nil] ``` This behavior is for higher-level libraries to maintain compatibility with a standalone client. You can exploit this behavior for migrating from a standalone server to a cluster. Although repeated single-key queries are slower than pipelining, pipelined queries are still slower than a single-slot query with multiple keys. Hence, we recommend using a hash tag in this use case for better performance. ## Transactions This gem supports [Redis transactions](https://redis.io/topics/transactions), including atomicity with `MULTI`/`EXEC`, and conditional execution with `WATCH`. Redis does not support cross-node transactions, so all keys used within a transaction must live in the same key slot. To use transactions, you can use `#multi` method same as the [redis-client](https://github.com/redis-rb/redis-client#usage): ```ruby cli.multi do |tx| tx.call('INCR', 'my_key') tx.call('INCR', 'my_key') end ``` More commonly, however, you will want to perform transactions across multiple keys. To do this, you need to ensure that all keys used in the transaction hash to the same slot; Redis provides a mechanism called [hashtags](https://redis.io/docs/reference/cluster-spec/#hash-tags) to achieve this. If a key contains a hashtag (e.g. in the key `{foo}bar`, the hashtag is `foo`), then it is guaranteed to hash to the same slot (and thus always live on the same node) as other keys which contain the same hashtag. So, whilst it's not possible in Redis cluster to perform a transaction on the keys `foo` and `bar`, it _is_ possible to perform a transaction on the keys `{tag}foo` and `{tag}bar`. To perform such transactions on this gem, use the hashtag: ```ruby cli.multi do |tx| tx.call('INCR', '{user123}coins_spent') tx.call('DECR', '{user123}coins_available') end ``` ```ruby # Conditional execution with WATCH can be used to e.g. atomically swap two keys cli.call('MSET', '{myslot}1', 'v1', '{myslot}2', 'v2') cli.multi(watch: %w[{myslot}1 {myslot}2]) do |tx| old_key1 = cli.call('GET', '{myslot}1') old_key2 = cli.call('GET', '{myslot}2') tx.call('SET', '{myslot}1', old_key2) tx.call('SET', '{myslot}2', old_key1) end # This transaction will swap the values of {myslot}1 and {myslot}2 only if no concurrent connection modified # either of the values ``` You can early return out of your block with a `next` statement if you want to cancel your transaction. In this context, don't use `break` and `return` statements. ```ruby # The transaction isn't executed. cli.multi do |tx| next if some_conditions? tx.call('SET', '{key}1', '1') tx.call('SET', '{key}2', '2') end ``` ```ruby # The watching state is automatically cleared with an execution of an empty transaction. cli.multi(watch: %w[{key}1 {key}2]) do |tx| next if some_conditions? tx.call('SET', '{key}1', '1') tx.call('SET', '{key}2', '2') end ``` `RedisClient::Cluster#multi` is aware of redirections and node failures like ordinary calls to `RedisClient::Cluster`, but because you may have written non-idempotent code inside your block, the block is called once if e.g. the slot it is operating on moves to a different node. ## ACL The cluster client internally calls [COMMAND](https://redis.io/commands/command) and [CLUSTER SHARDS](https://redis.io/commands/cluster-shards) commands to operate correctly. Please grant the following permissions. ```ruby # The default user is administrator. cli1 = RedisClient.cluster.new_client # To create a user with permissions # Typically, user settings are configured in the config file for the server beforehand. cli1.call('ACL', 'SETUSER', 'foo', 'ON', '+COMMAND', '+CLUSTER|SHARDS', '+PING', '>mysecret') # To initialize client with the user cli2 = RedisClient.cluster(username: 'foo', password: 'mysecret').new_client # The user can only call the PING command. cli2.call('PING') #=> "PONG" cli2.call('GET', 'key1') #=> NOPERM this user has no permissions to run the 'get' command (RedisClient::PermissionError) ``` Otherwise: ```ruby RedisClient.cluster(username: 'foo', password: 'mysecret').new_client #=> Redis client could not fetch cluster information: NOPERM this user has no permissions to run the 'cluster|nodes' command (RedisClient::Cluster::InitialSetupError) ``` ## Connection pooling You can use the internal connection pooling feature implemented by [redis-client](https://github.com/redis-rb/redis-client#usage) if needed. ```ruby # example of docker on localhost RedisClient.cluster.new_pool(timeout: 1.0, size: 2) #=> # ``` ## Connection drivers Please see [redis-client](https://github.com/redis-rb/redis-client#drivers). ## Development Please make sure the following tools are installed on your machine. | Tool | Version | URL | | --- | --- | --- | | Docker | latest stable | https://docs.docker.com/engine/install/ | | Ruby | latest stable | https://www.ruby-lang.org/en/ | Please fork this repository and check out the code. ``` $ git clone git@github.com:your-account-name/redis-cluster-client.git $ cd redis-cluster-client/ $ git remote add upstream https://github.com/redis-rb/redis-cluster-client.git $ git fetch -p upstream ``` Please do the following steps. * Build a Redis cluster with Docker * Install gems * Run basic test cases ``` ## If you use Docker server and your OS is Linux: $ bundle config set path '.bundle' $ bundle install --jobs=$(grep process /proc/cpuinfo | wc -l) $ docker compose up $ bundle exec rake test ## else: $ docker compose --profile ruby up $ docker compose --profile ruby exec ruby bundle install $ docker compose --profile ruby exec ruby bundle exec rake test ``` You can see more information in the YAML file for GitHub Actions. ## Migration This library might help you if you want to migrate your Redis from a standalone server to a cluster. Here is an example code. ```ruby # frozen_string_literal: true require 'bundler/inline' gemfile do source 'https://rubygems.org' gem 'redis-cluster-client' end src = RedisClient.config(url: ENV.fetch('REDIS_URL')).new_client dest = RedisClient.cluster(nodes: ENV.fetch('REDIS_CLUSTER_URL')).new_client node = dest.instance_variable_get(:@router).instance_variable_get(:@node) src.scan do |key| slot = ::RedisClient::Cluster::KeySlotConverter.convert(key) node_key = node.find_node_key_of_primary(slot) host, port = ::RedisClient::Cluster::NodeKey.split(node_key) src.blocking_call(10, 'MIGRATE', host, port, key, 0, 7, 'COPY', 'REPLACE') end ``` Further optimization is needed to perform well in production environments with large numbers of keys. Also, it should handle errors. ## See also * https://redis.io/docs/reference/cluster-spec/ * https://github.com/redis/redis-rb/issues/1070 * https://github.com/redis/redis/issues/8948 * https://github.com/valkey-io/valkey/issues/384 * https://github.com/antirez/redis-rb-cluster * https://twitter.com/antirez * https://bsky.app/profile/antirez.bsky.social * http://antirez.com/latest/0 * https://www.youtube.com/@antirez * https://www.twitch.tv/thetrueantirez/ redis-rb-redis-cluster-client-aaf9e2e/Rakefile000066400000000000000000000035041517214044400215070ustar00rootroot00000000000000# frozen_string_literal: true require 'rake/testtask' require 'rubocop/rake_task' require 'bundler/gem_helper' RuboCop::RakeTask.new Bundler::GemHelper.install_tasks SLUGGISH_TEST_TYPES = %w[down broken scale state].freeze task default: :test Rake::TestTask.new(:test) do |t| t.libs << :lib t.libs << :test t.options = '-v' t.test_files = if ARGV.size > 1 ARGV[1..] else Dir['test/**/test_*.rb'].grep_v(/test_against_cluster_(#{SLUGGISH_TEST_TYPES.join('|')})/) end end SLUGGISH_TEST_TYPES.each do |type| Rake::TestTask.new("test_cluster_#{type}".to_sym) do |t| t.libs << :lib t.libs << :test t.options = '-v' t.test_files = ["test/test_against_cluster_#{type}.rb"] end end %i[ips prof].each do |k| Rake::TestTask.new(k) do |t| t.libs << :lib t.libs << :test t.options = '-v' t.warning = false t.test_files = ARGV.size > 1 ? ARGV[1..] : Dir["test/**/#{k}_*.rb"] end end desc 'Wait for cluster to be ready' task :wait do $LOAD_PATH.unshift(File.expand_path('test', __dir__)) require 'testing_constants' require 'cluster_controller' ::ClusterController.new( TEST_NODE_URIS, shard_size: TEST_SHARD_SIZE, replica_size: TEST_REPLICA_SIZE, **TEST_GENERIC_OPTIONS ).wait_for_cluster_to_be_ready end desc 'Build cluster' task :build_cluster, %i[addr1 addr2] do |_, args| $LOAD_PATH.unshift(File.expand_path('test', __dir__)) require 'cluster_controller' hosts = args.values_at(:addr1, :addr2).compact ports = (6379..6384).to_a nodes = hosts.product(ports).map { |host, port| "redis://#{host}:#{port}" } shard_size = 3 replica_size = (nodes.size / shard_size) - 1 ::ClusterController.new( nodes, shard_size: shard_size, replica_size: replica_size, timeout: 30.0 ).rebuild end redis-rb-redis-cluster-client-aaf9e2e/bin/000077500000000000000000000000001517214044400206105ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/bin/console000077500000000000000000000002441517214044400222000ustar00rootroot00000000000000#!/usr/bin/env ruby # frozen_string_literal: true require 'irb' require 'bundler/setup' require 'redis_cluster_client' IRB.start(File.expand_path('..', __dir__)) redis-rb-redis-cluster-client-aaf9e2e/bin/gen_cert.sh000077500000000000000000000026221517214044400227370ustar00rootroot00000000000000#!/bin/bash set -o errexit set -o nounset set -o pipefail # https://www.openssl.org/docs/man1.1.1/man1/req.html readonly NAME='redis-rb' readonly EXPIRATION_DAYS='109500' readonly SBJ='/C=US/ST=Georgia/L=Atlanta/O=redis-rb/OU=redis-cluster-client/CN=127.0.0.1' readonly CERT_DIR="$(dirname $(realpath $0))/../test/ssl_certs" readonly WORK_DIR=$(mktemp -d) readonly SAN=(localhost node1 node2 node3 node4 node5 node6 node7 node8 node9) readonly SAN_TXT=$(echo ${SAN[@]} | sed -e 's# #,DNS:#g' | xargs -I{} echo "DNS:{}") cd ${WORK_DIR} mkdir -p ${CERT_DIR} ./demoCA ./demoCA/certs ./demoCA/crl ./demoCA/newcerts ./demoCA/private touch ./demoCA/index.txt rm -f ${CERT_DIR}/* openssl genpkey\ -algorithm RSA\ -pkeyopt rsa_keygen_bits:2048\ -out ./${NAME}-ca.key openssl req\ -new\ -x509\ -days ${EXPIRATION_DAYS}\ -key ./${NAME}-ca.key\ -sha256\ -out ${CERT_DIR}/${NAME}-ca.crt\ -subj "${SBJ}"\ -addext "subjectAltName = ${SAN_TXT}" openssl x509\ -in ${CERT_DIR}/${NAME}-ca.crt\ -noout\ -next_serial\ -out ./demoCA/serial openssl req\ -newkey rsa:2048\ -keyout ${CERT_DIR}/${NAME}-cert.key\ -nodes\ -out ./${NAME}-cert.req\ -subj "${SBJ}"\ -addext "subjectAltName = ${SAN_TXT}" openssl ca\ -days ${EXPIRATION_DAYS}\ -cert ${CERT_DIR}/${NAME}-ca.crt\ -keyfile ./${NAME}-ca.key\ -out ${CERT_DIR}/${NAME}-cert.crt\ -infiles ./${NAME}-cert.req chmod 644 -R ${CERT_DIR}/* redis-rb-redis-cluster-client-aaf9e2e/bin/pubsub000077500000000000000000000046611517214044400220450ustar00rootroot00000000000000#!/usr/bin/env ruby # frozen_string_literal: true require 'bundler/setup' require 'redis_cluster_client' module PubSubDebug WAIT_SEC = 2.0 module_function def spawn_publisher(client, channel) Thread.new(client, channel) do |cli, chan| role = 'Publisher' i = 0 loop do handle_errors(role) do cli.call('spublish', chan, i) log(role, :spublish, chan, i) i += 1 end ensure sleep WAIT_SEC end rescue StandardError => e log(role, :dead, e.class, e.message) raise end end def spawn_subscriber(client, channel) # rubocop:disable Metrics/AbcSize Thread.new(client, channel) do |cli, chan| role = 'Subscriber' ps = nil loop do ps = cli.pubsub ps.call('ssubscribe', chan) break rescue StandardError => e log(role, :init, e.class, e.message) ps&.close ensure sleep WAIT_SEC end loop do handle_errors('Subscriber') do event = ps.next_event(WAIT_SEC) log(role, *event) unless event.nil? case event&.first when 'sunsubscribe' then ps.call('ssubscribe', chan) end end end rescue StandardError, SignalException => e log(role, :dead, e.class, e.message) ps&.close raise end end def handle_errors(role) yield rescue RedisClient::ConnectionError, RedisClient::Cluster::InitialSetupError, RedisClient::Cluster::NodeMightBeDown => e log(role, e.class) rescue RedisClient::CommandError => e log(role, e.class, e.message) raise unless e.message.start_with?('CLUSTERDOWN') rescue StandardError => e log(role, e.class, e.message) raise end def log(*texts) return if texts.nil? || texts.empty? message = texts.map { |text| "#{' ' * [15 - text.to_s.size, 0].max}#{text}" }.join(': ') print "#{message}\n" end end nodes = (6379..6384).map { |port| "redis://127.0.0.1:#{port}" }.freeze clients = Array.new(6) { RedisClient.cluster(nodes: nodes, connect_with_original_config: true).new_client }.freeze threads = [] Signal.trap(:INT) do threads.each(&:exit) clients.each(&:close) PubSubDebug.log("\nBye bye") exit 0 end %w[chan1 chan2 chan3].each_with_index do |channel, i| threads << PubSubDebug.spawn_subscriber(clients[i], channel) threads << PubSubDebug.spawn_publisher(clients[i + 3], channel) end threads.each(&:join) redis-rb-redis-cluster-client-aaf9e2e/bin/reload_bomb000077500000000000000000000017241517214044400230070ustar00rootroot00000000000000#!/usr/bin/env ruby # frozen_string_literal: true require 'logger' require 'bundler/setup' require 'redis_cluster_client' logger = Logger.new($stdout) pids = Array.new(Integer(ENV.fetch('N', '10'))) do Process.fork do Signal.trap(:INT, 'IGNORE') nodes = (6379..6384).map { |port| "redis://127.0.0.1:#{port}" }.freeze client = RedisClient.cluster(nodes: nodes, connect_with_original_config: true).new_client Signal.trap(:TERM) do client.close Process.exit end logger.info("#{Process.pid}: child process started") loop do client.send(:router).renew_cluster_state sleep 0.03 rescue StandardError => e logger.error("#{Process.pid}: #{e.message}") sleep 1.0 end end end Signal.trap(:INT) do pids.each { |pid| Process.kill(:TERM, pid) } end pids.each do |pid| Process.waitpid2(pid) logger.info("#{pid}: child process closed") end logger.info("#{Process.pid}: parent process closed") Process.exit redis-rb-redis-cluster-client-aaf9e2e/bin/render_compose000077500000000000000000000006551517214044400235500ustar00rootroot00000000000000#!/usr/bin/env ruby # frozen_string_literal: true require 'erb' path = File.join(File.expand_path('..', __dir__), 'compose.yaml.erb') content = File.read(path) template = ERB.new(content, trim_mode: '<>') # rubocop:disable Lint/UselessAssignment shards = ENV.fetch('REDIS_SHARD_SIZE', '10').to_i n = shards + shards * ENV.fetch('REDIS_REPLICA_SIZE', '2').to_i port = 6379 # rubocop:enable Lint/UselessAssignment template.run redis-rb-redis-cluster-client-aaf9e2e/bin/singlepiptx000077500000000000000000000060111517214044400231020ustar00rootroot00000000000000#!/usr/bin/env ruby # frozen_string_literal: true require 'bundler/setup' require 'redis_cluster_client' module SinglePipTxDebug WAIT_SEC = 2.0 module_function def spawn_single(client, key) Thread.new(client, key) do |cli, k| role = 'Single' loop do handle_errors(role) do reply = cli.call('incr', k) log(role, k, reply) end ensure sleep WAIT_SEC end rescue StandardError => e log(role, :dead, e.class, e.message) raise end end def spawn_pipeline(client, key) Thread.new(client, key) do |cli, k| role = 'Pipeline' loop do handle_errors(role) do reply = cli.pipelined do |pi| pi.call('incr', k) pi.call('incr', k) end log(role, k, reply.last) end ensure sleep WAIT_SEC end rescue StandardError => e log(role, :dead, e.class, e.message) raise end end def spawn_transaction(client, key) Thread.new(client, key) do |cli, k| role = 'Transaction' i = 0 loop do handle_errors(role) do reply = cli.multi(watch: i.odd? ? [k] : nil) do |tx| tx.call('incr', k) tx.call('incr', k) end log(role, k, reply.last) i += 1 end ensure sleep WAIT_SEC end rescue StandardError => e log(role, :dead, e.class, e.message) raise end end def handle_errors(role) # rubocop:disable Metrics/AbcSize yield rescue RedisClient::ConnectionError, RedisClient::Cluster::InitialSetupError, RedisClient::Cluster::NodeMightBeDown => e log(role, e.class) rescue RedisClient::CommandError => e log(role, e.class, e.message) raise unless e.message.start_with?('CLUSTERDOWN') rescue RedisClient::Cluster::ErrorCollection => e log(role, e.class, e.message) raise unless e.errors.values.all? do |err| err.message.start_with?('CLUSTERDOWN') || err.is_a?(::RedisClient::ConnectionError) end rescue StandardError => e log(role, e.class, e.message) raise end def log(*texts) return if texts.nil? || texts.empty? message = texts.map { |text| "#{' ' * [15 - text.to_s.size, 0].max}#{text}" }.join(': ') print "#{message}\n" end end nodes = (6379..6384).map { |port| "redis://127.0.0.1:#{port}" }.freeze clients = Array.new(9) { RedisClient.cluster(nodes: nodes, connect_with_original_config: true).new_client }.freeze threads = [] Signal.trap(:INT) do threads.each(&:exit) clients.each(&:close) SinglePipTxDebug.log("\nBye bye") exit 0 end %w[single1 single3 single4].each_with_index do |key, i| threads << SinglePipTxDebug.spawn_single(clients[i], key) end %w[pipeline1 pipeline2 pipeline4].each_with_index do |key, i| threads << SinglePipTxDebug.spawn_pipeline(clients[i + 3], key) end %w[transaction1 transaction3 transaction4].each_with_index do |key, i| threads << SinglePipTxDebug.spawn_transaction(clients[i + 6], key) end threads.each(&:join) redis-rb-redis-cluster-client-aaf9e2e/compose.auth.yaml000066400000000000000000000032331517214044400233320ustar00rootroot00000000000000--- services: node1: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --requirepass '!&<123-abc>' --masterauth '!&<123-abc>' --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" node2: <<: *node ports: - "6380:6379" node3: <<: *node ports: - "6381:6379" node4: <<: *node ports: - "6382:6379" node5: <<: *node ports: - "6383:6379" node6: <<: *node ports: - "6384:6379" clustering: image: "redis:${REDIS_VERSION:-8}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | redis-cli -a '!&<123-abc>' --cluster create $$(dig node1 +short):6379 $$(dig node2 +short):6379 $$(dig node3 +short):6379 $$(dig node4 +short):6379 $$(dig node5 +short):6379 $$(dig node6 +short):6379 --cluster-replicas 1" depends_on: node1: condition: service_healthy node2: condition: service_healthy node3: condition: service_healthy node4: condition: service_healthy node5: condition: service_healthy node6: condition: service_healthy redis-rb-redis-cluster-client-aaf9e2e/compose.latency.yaml000066400000000000000000000063571517214044400240420ustar00rootroot00000000000000--- # 3 shards plus with each 2 replicas simutlated network delay services: node1: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" node2: <<: *node ports: - "6380:6379" node3: <<: *node ports: - "6381:6379" node4: <<: *node ports: - "6382:6379" node5: &far_node <<: *node command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y iproute2 iputils-ping > /dev/null && rm -rf /var/lib/apt/lists/* && (tc qdisc add dev eth0 root netem delay ${DELAY_TIME:-1ms} || echo skipped) && redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000" cap_add: - NET_ADMIN ports: - "6383:6379" node6: <<: *node ports: - "6384:6379" node7: <<: *far_node ports: - "6385:6379" node8: <<: *node ports: - "6386:6379" node9: <<: *far_node ports: - "6387:6379" clustering: image: "redis:${REDIS_VERSION:-8}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | redis-cli --cluster create $$(dig node1 +short):6379 $$(dig node2 +short):6379 $$(dig node3 +short):6379 $$(dig node4 +short):6379 $$(dig node5 +short):6379 $$(dig node6 +short):6379 $$(dig node7 +short):6379 $$(dig node8 +short):6379 $$(dig node9 +short):6379 --cluster-replicas 2" depends_on: node1: condition: service_healthy node2: condition: service_healthy node3: condition: service_healthy node4: condition: service_healthy node5: condition: service_healthy node6: condition: service_healthy node7: condition: service_healthy node8: condition: service_healthy node9: condition: service_healthy envoy: image: envoyproxy/envoy:v1.35.3 restart: "${RESTART_POLICY:-always}" command: - envoy - -c - /etc/envoy/envoy.yaml - -l - warn ports: - "3000:10000" - "7000:10001" volumes: - ./test/proxy/envoy.yaml:/etc/envoy/envoy.yaml depends_on: clustering: condition: service_completed_successfully redis-cluster-proxy: image: ghcr.io/redis-rb/redis-cluster-proxy:latest restart: "${RESTART_POLICY:-always}" command: - "--bind" - "0.0.0.0" - "node1:6379" ports: - "7001:7777" depends_on: clustering: condition: service_completed_successfully redis-rb-redis-cluster-client-aaf9e2e/compose.massive.yaml000066400000000000000000000144661517214044400240520ustar00rootroot00000000000000--- # bin/render_compose > compose.massive.yaml # # https://developer.redis.com/operate/redis-at-scale/talking-to-redis/initial-tuning/ # https://github.com/redis/redis/blob/unstable/redis.conf services: node001: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 32mb --maxmemory-policy allkeys-lru --maxmemory-clients 32% --maxclients 256 --tcp-backlog 1024 --timeout 0 --tcp-keepalive 300 --save "" --rdb-del-sync-files no --replica-serve-stale-data yes --replica-read-only yes --repl-backlog-size 2mb --repl-backlog-ttl 0 --repl-disable-tcp-nodelay yes --repl-diskless-sync yes --repl-diskless-sync-delay 0 --repl-diskless-sync-max-replicas 0 --repl-diskless-load on-empty-db --repl-ping-replica-period 60 --repl-timeout 300 --min-replicas-to-write 0 --min-replicas-max-lag 0 --shutdown-timeout 0 --shutdown-on-sigint nosave force now --shutdown-on-sigterm nosave force now --client-output-buffer-limit replica 0 0 0 --client-query-buffer-limit 4mb --appendonly no --appendfsync no --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 300000 --cluster-replica-validity-factor 5 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" node002: <<: *node ports: - "6380:6379" node003: <<: *node ports: - "6381:6379" node004: <<: *node ports: - "6382:6379" node005: <<: *node ports: - "6383:6379" node006: <<: *node ports: - "6384:6379" node007: <<: *node ports: - "6385:6379" node008: <<: *node ports: - "6386:6379" node009: <<: *node ports: - "6387:6379" node010: <<: *node ports: - "6388:6379" node011: <<: *node ports: - "6389:6379" node012: <<: *node ports: - "6390:6379" node013: <<: *node ports: - "6391:6379" node014: <<: *node ports: - "6392:6379" node015: <<: *node ports: - "6393:6379" node016: <<: *node ports: - "6394:6379" node017: <<: *node ports: - "6395:6379" node018: <<: *node ports: - "6396:6379" node019: <<: *node ports: - "6397:6379" node020: <<: *node ports: - "6398:6379" node021: <<: *node ports: - "6399:6379" node022: <<: *node ports: - "6400:6379" node023: <<: *node ports: - "6401:6379" node024: <<: *node ports: - "6402:6379" node025: <<: *node ports: - "6403:6379" node026: <<: *node ports: - "6404:6379" node027: <<: *node ports: - "6405:6379" node028: <<: *node ports: - "6406:6379" node029: <<: *node ports: - "6407:6379" node030: <<: *node ports: - "6408:6379" clustering: image: "redis:${REDIS_VERSION:-8}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | redis-cli --cluster create $$(dig node001 +short):6379 $$(dig node002 +short):6379 $$(dig node003 +short):6379 $$(dig node004 +short):6379 $$(dig node005 +short):6379 $$(dig node006 +short):6379 $$(dig node007 +short):6379 $$(dig node008 +short):6379 $$(dig node009 +short):6379 $$(dig node010 +short):6379 $$(dig node011 +short):6379 $$(dig node012 +short):6379 $$(dig node013 +short):6379 $$(dig node014 +short):6379 $$(dig node015 +short):6379 $$(dig node016 +short):6379 $$(dig node017 +short):6379 $$(dig node018 +short):6379 $$(dig node019 +short):6379 $$(dig node020 +short):6379 $$(dig node021 +short):6379 $$(dig node022 +short):6379 $$(dig node023 +short):6379 $$(dig node024 +short):6379 $$(dig node025 +short):6379 $$(dig node026 +short):6379 $$(dig node027 +short):6379 $$(dig node028 +short):6379 $$(dig node029 +short):6379 $$(dig node030 +short):6379 --cluster-replicas 2" depends_on: node001: condition: service_healthy node002: condition: service_healthy node003: condition: service_healthy node004: condition: service_healthy node005: condition: service_healthy node006: condition: service_healthy node007: condition: service_healthy node008: condition: service_healthy node009: condition: service_healthy node010: condition: service_healthy node011: condition: service_healthy node012: condition: service_healthy node013: condition: service_healthy node014: condition: service_healthy node015: condition: service_healthy node016: condition: service_healthy node017: condition: service_healthy node018: condition: service_healthy node019: condition: service_healthy node020: condition: service_healthy node021: condition: service_healthy node022: condition: service_healthy node023: condition: service_healthy node024: condition: service_healthy node025: condition: service_healthy node026: condition: service_healthy node027: condition: service_healthy node028: condition: service_healthy node029: condition: service_healthy node030: condition: service_healthy redis-rb-redis-cluster-client-aaf9e2e/compose.nat.yaml000066400000000000000000000077311517214044400231620ustar00rootroot00000000000000--- # First things first, make sure the default-address-pools settings in /etc/docker/daemon.json # HOST_ADDR=192.168.11.9 docker compose -f compose.nat.yaml up # HOST_ADDR=192.168.11.7 docker compose -f compose.nat.yaml up # DEBUG=1 bundle exec rake 'build_cluster[192.168.11.9,192.168.11.7]' services: node1: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --replica-announce-ip "${HOST_ADDR:-127.0.0.1}" --replica-announce-port 6379 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 10000 --cluster-announce-ip "${HOST_ADDR:-127.0.0.1}" --cluster-announce-port 6379 --cluster-announce-bus-port 16379 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" - "16379:16379" node2: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --replica-announce-ip "${HOST_ADDR:-127.0.0.1}" --replica-announce-port 6380 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 10000 --cluster-announce-ip "${HOST_ADDR:-127.0.0.1}" --cluster-announce-port 6380 --cluster-announce-bus-port 16380 ports: - "6380:6379" - "16380:16379" node3: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --replica-announce-ip "${HOST_ADDR:-127.0.0.1}" --replica-announce-port 6381 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 10000 --cluster-announce-ip "${HOST_ADDR:-127.0.0.1}" --cluster-announce-port 6381 --cluster-announce-bus-port 16381 ports: - "6381:6379" - "16381:16379" node4: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --replica-announce-ip "${HOST_ADDR:-127.0.0.1}" --replica-announce-port 6382 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 10000 --cluster-announce-ip "${HOST_ADDR:-127.0.0.1}" --cluster-announce-port 6382 --cluster-announce-bus-port 16382 ports: - "6382:6379" - "16382:16379" node5: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --replica-announce-ip "${HOST_ADDR:-127.0.0.1}" --replica-announce-port 6383 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 10000 --cluster-announce-ip "${HOST_ADDR:-127.0.0.1}" --cluster-announce-port 6383 --cluster-announce-bus-port 16383 ports: - "6383:6379" - "16383:16379" node6: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --replica-announce-ip "${HOST_ADDR:-127.0.0.1}" --replica-announce-port 6384 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 10000 --cluster-announce-ip "${HOST_ADDR:-127.0.0.1}" --cluster-announce-port 6384 --cluster-announce-bus-port 16384 ports: - "6384:6379" - "16384:16379" redis-rb-redis-cluster-client-aaf9e2e/compose.replica.yaml000066400000000000000000000037461517214044400240210ustar00rootroot00000000000000--- # 3 shards with double replicas services: node1: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" node2: <<: *node ports: - "6380:6379" node3: <<: *node ports: - "6381:6379" node4: <<: *node ports: - "6382:6379" node5: <<: *node ports: - "6383:6379" node6: <<: *node ports: - "6384:6379" node7: <<: *node ports: - "6385:6379" node8: <<: *node ports: - "6386:6379" node9: <<: *node ports: - "6387:6379" clustering: image: "redis:${REDIS_VERSION:-8}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | redis-cli --cluster create $$(dig node1 +short):6379 $$(dig node2 +short):6379 $$(dig node3 +short):6379 $$(dig node4 +short):6379 $$(dig node5 +short):6379 $$(dig node6 +short):6379 $$(dig node7 +short):6379 $$(dig node8 +short):6379 $$(dig node9 +short):6379 --cluster-replicas 2" depends_on: node1: condition: service_healthy node2: condition: service_healthy node3: condition: service_healthy node4: condition: service_healthy node5: condition: service_healthy node6: condition: service_healthy node7: condition: service_healthy node8: condition: service_healthy node9: condition: service_healthy redis-rb-redis-cluster-client-aaf9e2e/compose.scale.yaml000066400000000000000000000034001517214044400234540ustar00rootroot00000000000000--- services: node1: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" node2: <<: *node ports: - "6380:6379" node3: <<: *node ports: - "6381:6379" node4: <<: *node ports: - "6382:6379" node5: <<: *node ports: - "6383:6379" node6: <<: *node ports: - "6384:6379" node7: <<: *node ports: - "6385:6379" node8: <<: *node ports: - "6386:6379" clustering: image: "redis:${REDIS_VERSION:-8}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | redis-cli --cluster create $$(dig node1 +short):6379 $$(dig node2 +short):6379 $$(dig node3 +short):6379 $$(dig node4 +short):6379 $$(dig node5 +short):6379 $$(dig node6 +short):6379 --cluster-replicas 1" depends_on: node1: condition: service_healthy node2: condition: service_healthy node3: condition: service_healthy node4: condition: service_healthy node5: condition: service_healthy node6: condition: service_healthy node7: condition: service_healthy node8: condition: service_healthy redis-rb-redis-cluster-client-aaf9e2e/compose.ssl.yaml000066400000000000000000000176731517214044400232070ustar00rootroot00000000000000--- # @see https://redis.io/docs/manual/security/encryption/ services: node1: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 0 --tls-port 6379 --tls-cert-file /etc/ssl/private/redis-rb-cert.crt --tls-key-file /etc/ssl/private/redis-rb-cert.key --tls-ca-cert-file /etc/ssl/private/redis-rb-ca.crt --tls-cluster yes --tls-replication yes --replica-announce-ip node1 restart: "${RESTART_POLICY:-always}" healthcheck: test: - "CMD" - "redis-cli" - "-h" - "127.0.0.1" - "-p" - "6379" - "--no-raw" - "--tls" - "--sni" - "localhost" - "--cert" - "/etc/ssl/private/redis-rb-cert.crt" - "--key" - "/etc/ssl/private/redis-rb-cert.key" - "--cacert" - "/etc/ssl/private/redis-rb-ca.crt" - "ping" interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" volumes: - "./test/ssl_certs:/etc/ssl/private:ro" node2: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 0 --tls-port 6380 --tls-cert-file /etc/ssl/private/redis-rb-cert.crt --tls-key-file /etc/ssl/private/redis-rb-cert.key --tls-ca-cert-file /etc/ssl/private/redis-rb-ca.crt --tls-cluster yes --tls-replication yes --replica-announce-ip node2 healthcheck: test: - "CMD" - "redis-cli" - "-h" - "127.0.0.1" - "-p" - "6380" - "--no-raw" - "--tls" - "--sni" - "localhost" - "--cert" - "/etc/ssl/private/redis-rb-cert.crt" - "--key" - "/etc/ssl/private/redis-rb-cert.key" - "--cacert" - "/etc/ssl/private/redis-rb-ca.crt" - "ping" interval: "7s" timeout: "5s" retries: 10 ports: - "6380:6380" node3: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 0 --tls-port 6381 --tls-cert-file /etc/ssl/private/redis-rb-cert.crt --tls-key-file /etc/ssl/private/redis-rb-cert.key --tls-ca-cert-file /etc/ssl/private/redis-rb-ca.crt --tls-cluster yes --tls-replication yes --replica-announce-ip node3 healthcheck: test: - "CMD" - "redis-cli" - "-h" - "127.0.0.1" - "-p" - "6381" - "--no-raw" - "--tls" - "--sni" - "localhost" - "--cert" - "/etc/ssl/private/redis-rb-cert.crt" - "--key" - "/etc/ssl/private/redis-rb-cert.key" - "--cacert" - "/etc/ssl/private/redis-rb-ca.crt" - "ping" interval: "7s" timeout: "5s" retries: 10 ports: - "6381:6381" node4: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 0 --tls-port 6382 --tls-cert-file /etc/ssl/private/redis-rb-cert.crt --tls-key-file /etc/ssl/private/redis-rb-cert.key --tls-ca-cert-file /etc/ssl/private/redis-rb-ca.crt --tls-cluster yes --tls-replication yes --replica-announce-ip node4 healthcheck: test: - "CMD" - "redis-cli" - "-h" - "127.0.0.1" - "-p" - "6382" - "--no-raw" - "--tls" - "--sni" - "localhost" - "--cert" - "/etc/ssl/private/redis-rb-cert.crt" - "--key" - "/etc/ssl/private/redis-rb-cert.key" - "--cacert" - "/etc/ssl/private/redis-rb-ca.crt" - "ping" interval: "7s" timeout: "5s" retries: 10 ports: - "6382:6382" node5: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 0 --tls-port 6383 --tls-cert-file /etc/ssl/private/redis-rb-cert.crt --tls-key-file /etc/ssl/private/redis-rb-cert.key --tls-ca-cert-file /etc/ssl/private/redis-rb-ca.crt --tls-cluster yes --tls-replication yes --replica-announce-ip node5 healthcheck: test: - "CMD" - "redis-cli" - "-h" - "127.0.0.1" - "-p" - "6383" - "--no-raw" - "--tls" - "--sni" - "localhost" - "--cert" - "/etc/ssl/private/redis-rb-cert.crt" - "--key" - "/etc/ssl/private/redis-rb-cert.key" - "--cacert" - "/etc/ssl/private/redis-rb-ca.crt" - "ping" interval: "7s" timeout: "5s" retries: 10 ports: - "6383:6383" node6: <<: *node command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 0 --tls-port 6384 --tls-cert-file /etc/ssl/private/redis-rb-cert.crt --tls-key-file /etc/ssl/private/redis-rb-cert.key --tls-ca-cert-file /etc/ssl/private/redis-rb-ca.crt --tls-cluster yes --tls-replication yes --replica-announce-ip node6 healthcheck: test: - "CMD" - "redis-cli" - "-h" - "127.0.0.1" - "-p" - "6384" - "--no-raw" - "--tls" - "--sni" - "localhost" - "--cert" - "/etc/ssl/private/redis-rb-cert.crt" - "--key" - "/etc/ssl/private/redis-rb-cert.key" - "--cacert" - "/etc/ssl/private/redis-rb-ca.crt" - "ping" interval: "7s" timeout: "5s" retries: 10 ports: - "6384:6384" clustering: image: "redis:${REDIS_VERSION:-8}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | redis-cli --tls --cert /etc/ssl/private/redis-rb-cert.crt --key /etc/ssl/private/redis-rb-cert.key --cacert /etc/ssl/private/redis-rb-ca.crt --cluster create $$(dig node1 +short):6379 $$(dig node2 +short):6380 $$(dig node3 +short):6381 $$(dig node4 +short):6382 $$(dig node5 +short):6383 $$(dig node6 +short):6384 --cluster-replicas 1" volumes: - "./test/ssl_certs:/etc/ssl/private:ro" depends_on: node1: condition: service_healthy node2: condition: service_healthy node3: condition: service_healthy node4: condition: service_healthy node5: condition: service_healthy node6: condition: service_healthy redis-rb-redis-cluster-client-aaf9e2e/compose.valkey.yaml000066400000000000000000000037311517214044400236670ustar00rootroot00000000000000--- services: node1: &node image: "valkey/valkey:${REDIS_VERSION:-9}" command: > valkey-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "valkey-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" node2: <<: *node ports: - "6380:6379" node3: <<: *node ports: - "6381:6379" node4: <<: *node ports: - "6382:6379" node5: <<: *node ports: - "6383:6379" node6: <<: *node ports: - "6384:6379" node7: <<: *node ports: - "6385:6379" node8: <<: *node ports: - "6386:6379" node9: <<: *node ports: - "6387:6379" clustering: image: "valkey/valkey:${REDIS_VERSION:-9}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | valkey-cli --cluster create $$(dig node1 +short):6379 $$(dig node2 +short):6379 $$(dig node3 +short):6379 $$(dig node4 +short):6379 $$(dig node5 +short):6379 $$(dig node6 +short):6379 $$(dig node7 +short):6379 $$(dig node8 +short):6379 $$(dig node9 +short):6379 --cluster-replicas 2" depends_on: node1: condition: service_healthy node2: condition: service_healthy node3: condition: service_healthy node4: condition: service_healthy node5: condition: service_healthy node6: condition: service_healthy node7: condition: service_healthy node8: condition: service_healthy node9: condition: service_healthy redis-rb-redis-cluster-client-aaf9e2e/compose.yaml000066400000000000000000000037751517214044400224050ustar00rootroot00000000000000--- services: node1: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 64mb --maxmemory-policy allkeys-lru --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "6379:6379" node2: <<: *node ports: - "6380:6379" node3: <<: *node ports: - "6381:6379" node4: <<: *node ports: - "6382:6379" node5: <<: *node ports: - "6383:6379" node6: <<: *node ports: - "6384:6379" clustering: image: "redis:${REDIS_VERSION:-8}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | redis-cli --cluster create $$(dig node1 +short):6379 $$(dig node2 +short):6379 $$(dig node3 +short):6379 $$(dig node4 +short):6379 $$(dig node5 +short):6379 $$(dig node6 +short):6379 --cluster-replicas 1" depends_on: node1: condition: service_healthy node2: condition: service_healthy node3: condition: service_healthy node4: condition: service_healthy node5: condition: service_healthy node6: condition: service_healthy ruby: image: "ruby:${RUBY_VERSION:-4}" restart: always working_dir: /client volumes: - .:/client command: - ruby - "-e" - 'Signal.trap(:INT, "EXIT"); Signal.trap(:TERM, "EXIT"); loop { sleep 1 }' environment: REDIS_HOST: node1 cap_drop: - ALL healthcheck: test: ["CMD", "ruby", "-e", "'puts 1'"] interval: "5s" timeout: "3s" retries: 3 profiles: - ruby redis-rb-redis-cluster-client-aaf9e2e/compose.yaml.erb000066400000000000000000000052031517214044400231400ustar00rootroot00000000000000--- # bin/render_compose > compose.massive.yaml # # https://developer.redis.com/operate/redis-at-scale/talking-to-redis/initial-tuning/ # https://github.com/redis/redis/blob/unstable/redis.conf services: node001: &node image: "redis:${REDIS_VERSION:-8}" command: > redis-server --maxmemory 32mb --maxmemory-policy allkeys-lru --maxmemory-clients 32% --maxclients 256 --tcp-backlog 1024 --timeout 0 --tcp-keepalive 300 --save "" --rdb-del-sync-files no --replica-serve-stale-data yes --replica-read-only yes --repl-backlog-size 2mb --repl-backlog-ttl 0 --repl-disable-tcp-nodelay yes --repl-diskless-sync yes --repl-diskless-sync-delay 0 --repl-diskless-sync-max-replicas 0 --repl-diskless-load on-empty-db --repl-ping-replica-period 60 --repl-timeout 300 --min-replicas-to-write 0 --min-replicas-max-lag 0 --shutdown-timeout 0 --shutdown-on-sigint nosave force now --shutdown-on-sigterm nosave force now --client-output-buffer-limit replica 0 0 0 --client-query-buffer-limit 4mb --appendonly no --appendfsync no --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 300000 --cluster-replica-validity-factor 5 restart: "${RESTART_POLICY:-always}" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: "7s" timeout: "5s" retries: 10 ports: - "<%= port %>:<%= port %>" <% (2..n).each do |i| %> node<%= sprintf('%03d', i) %>: <<: *node ports: - "<%= port + i - 1 %>:<%= port %>" <% end %> clustering: image: "redis:${REDIS_VERSION:-8}" command: > bash -c "apt-get update > /dev/null && apt-get install --no-install-recommends --no-install-suggests -y dnsutils > /dev/null && rm -rf /var/lib/apt/lists/* && yes yes | redis-cli --cluster create <% n.times do |i| %> $$(dig node<%= sprintf('%03d', i + 1) %> +short):<%= port %> <% end %> --cluster-replicas <%= (n - shards) / shards %>" depends_on: <% n.times do |i| %> node<%= sprintf('%03d', i + 1) %>: condition: service_healthy <% end %> redis-rb-redis-cluster-client-aaf9e2e/docs/000077500000000000000000000000001517214044400207705ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/docs/class_diagrams_redis_client.md000066400000000000000000000133531517214044400270170ustar00rootroot00000000000000# [RedisClient](https://github.com/redis-rb/redis-client) ```mermaid classDiagram class RedisClient { +self.register_driver() +self.driver() +self.default_driver() +self.default_driver=() +self.config() +self.sentinel() +self.ring() +self.new() +self.register(middleware) +self.now() +self.now_ms() +initialize() +inspect() +size() +server_url() +db() +host() +port() +path() +username() +password() +idle_timeout() +with() +timeout=() +read_timeout=() +write_timeout=() +pubsub() +call() +call_v() +call_once() +call_once_v() +blocking_call() +blocking_call_v() +scan() +sscan() +hscan() +zscan() +connected?() +close() +disable_reconnection() +measure_round_trip_delay() +pipelined() +multi() } class module_RedisClient_Common { +config() +id() +nodes() +connect_timeout() +connect_timeout=() +read_timeout() +read_timeout=() +write_timeout() +write_timeout=() +timeout=() +node_for() +nodes_for() } class RedisClient_PubSub { +initialize() +call() +call_v() +close() +next_event() } class RedisClient_Multi { +initialize() +call() +call_v() +call_once() +call_once_v() } class RedisClient_Pipeline { +initialize() +blocking_call() +blocking_call_v() } class RedisClient_RubyConnection_BufferedIO { +read_timeout() +read_timeout=() +write_timeout() +write_timeout=() +initialize() +close() +closed?() +eof?() +with_timeout() +skip() +write() +getbyte() +gets_chomp() +read_chomp() } class module_RedisClient_RESP3 { +self.dump() +self.load() +self.new_buffer() +self.dump_any() +self.dump_array() +self.dump_set() +self.dump_hash() +self.dump_numeric() +self.dump_string() +self.parse() +self.parse_string() +self.parse_error() +self.parse_boolean() +self.parse_array() +self.parse_set() +self.parse_map() +self.parse_push() +self.parse_sequence() +self.parse_integer() +self.parse_double() +self.parse_null() +self.parse_blob() +self.parse_verbatim_string() } class module_RedisClient_CommandBuilder { +self.generate!() } class RedisClient_Config { +host() +port() +path() +initialize() } class module_RedisClient_Config_Common { +db() +username() +password() +id() +ssl() +ssl?() +ssl_params() +command_builder() +connect_timeout() +read_timeout() +write_timeout() +idle_timeout() +driver() +protocol() +circuit_breaker() +custom() +inherit_socket() +driver_info() +middlewares_stack() +connection_prelude() +initialize() +sentinel?() +resolved?() +new_pool() +new_client() +retry_connecting?() +ssl_context() +server_url() +build_lib_name() } class RedisClient_SentinelConfig { +initialize() +sentinels() +reset() +host() +port() +path() +retry_connecting?() +sentinel?() +resolved?() +check_role!() } class module_RedisClient_ConnectionMixin { +call() +call_pipelined() } class RedisClient_RubyConnection { +self.ssl_context() +initialize() +connected?() +close() +read_timeout=() +write_timeout=() +write() +write_multi() +read() +measure_round_trip_delay() } class module_RedisClient_Decorator { +self.create() } class module_RedisClient_Decorator_CommandsMixin { +initialize() +call() +call_v() +call_once() +call_once_v() +blocking_call() +blocking_call_v() } class RedisClient_Decorator_Pipeline { } class RedisClient_Decorator_Client { +initialize() +with() +pipelined() +multi() +close() +scan() +hscan() +sscan() +zscan() +id() +config() +size() +connect_timeout() +read_timeout() +write_timeout() +timeout=() +connect_timeout=() +read_timeout=() +write_timeout=() } class module_RedisClient_Middlewares { +self.call() +self.call_pipelined() } class RedisClient_Pooled { +initialize() +with() +close() +size() +pipelined() +multi() +pubsub() +call() +call_once() +blocking_call() +scan() +sscan() +hscan() +zscan() } RedisClient ..|> module_RedisClient_Common : include RedisClient ..> RedisClient_PubSub : new RedisClient ..> RedisClient_Multi : new RedisClient ..> RedisClient_Pipeline : new RedisClient ..> module_RedisClient_Middlewares : call RedisClient ..> RedisClient_RubyConnection : new RedisClient ..> RedisClient_Config : new RedisClient ..> RedisClient_SentinelConfig : new RedisClient ..> module_RedisClient_CommandBuilder : call RedisClient_Multi <|.. RedisClient_Pipeline : extend RedisClient_Config ..|> module_RedisClient_Config_Common : include RedisClient_SentinelConfig ..|> module_RedisClient_Config_Common : include module_RedisClient_Config_Common ..> RedisClient_Pooled : new module_RedisClient_Decorator ..> RedisClient_Decorator_Pipeline : new module_RedisClient_Decorator ..> RedisClient_Decorator_Client : new RedisClient_Decorator_Pipeline ..|> module_RedisClient_Decorator_CommandsMixin : include RedisClient_Decorator_Client ..|> module_RedisClient_Decorator_CommandsMixin : include RedisClient_Pooled ..|> module_RedisClient_Common : include RedisClient_RubyConnection ..|> module_RedisClient_ConnectionMixin : include RedisClient_RubyConnection ..> RedisClient_RubyConnection_BufferedIO : new RedisClient_RubyConnection ..> module_RedisClient_RESP3 : call ``` redis-rb-redis-cluster-client-aaf9e2e/docs/class_diagrams_redis_cluster_client.md000066400000000000000000000116411517214044400305560ustar00rootroot00000000000000# RedisClient::Cluster ```mermaid classDiagram class RedisClient_Cluster { +inspect() +call() +call_v() +call_once() +call_once_v() +blocking_call() +blocking_call_v() +scan() +sscan() +hscan() +zscan() +pipelined() +multi() +pubsub() +with() +close() } class RedisClient_ClusterConfig { +inspect() +new_pool() +new_client() +use_replica?() +connect_timeout() +read_timeout() +write_timeout() +client_config_for_node() +resolved?() +sentinel?() +server_url() } class RedisClient_Cluster_Command { +self.load() +extract_first_key() +should_send_to_primary?() +should_send_to_replica?() +exists?() } class module_RedisClient_Cluster_KeySlotConverter { +convert() } class RedisClient_Cluster_Node { +inspect() +each() +sample() +node_keys() +find_by() +call_all() +call_primaries() +call_replicas() +send_ping() +clients() +primary_clients() +replica_clients() +clients_for_scanning() +find_node_key_of_primary() +find_node_key_of_replica() +any_primary_node_key() +any_replica_node_key() +update_slot() +try_reload!() } class RedisClient_Cluster_Node_BaseTopology { +clients() +primary_clients() +replica_clients() +clients_for_scanning() +any_primary_node_key() +process_topology_update!() } class RedisClient_Cluster_Node_PrimaryOnly { +clients_for_scanning() +find_node_key_of_replica() +any_primary_node_key() +any_replica_node_key() +process_topology_update!() } class RedisClient_Cluster_Node_RandomReplica { +replica_clients() +clients_for_scanning() +find_node_key_of_replica() +any_replica_node_key() } class RedisClient_Cluster_Node_RandomReplicaOrPrimary { +replica_clients() +clients_for_scanning() +find_node_key_of_replica() +any_replica_node_key() } class RedisClient_Cluster_Node_LatencyReplica { +clients_for_scanning() +find_node_key_of_replica() +any_replica_node_key() +process_topology_update!() } class module_RedisClient_Cluster_NodeKey { +hashify() +split() +build_from_uri() +build_from_host_port() +build_from_client() } class RedisClient_Cluster_Pipeline { +call() +call_v() +call_once() +call_once_v() +blocking_call() +blocking_call_v() +empty?() +execute() } class RedisClient_Cluster_Transaction { +call() +call_v() +call_once() +call_once_v() +execute() } class RedisClient_Cluster_OptimisticLocking { +watch() } class RedisClient_Cluster_PubSub { +call() +call_v() +close() +next_event() } class RedisClient_Cluster_Router { +send_command() +assign_node_and_send_command() +send_command_to_node() +handle_redirection() +scan() +scan_single_key() +assign_node() +find_node_key_by_key() +find_primary_node_by_slot() +find_node_key() +find_primary_node_key() +find_slot() +find_slot_by_key() +find_node() +command_exists?() +assign_redirection_node() +assign_asking_node() +node_keys() +renew_cluster_state() +close() } RedisClient_ClusterConfig ..> RedisClient_Cluster : new RedisClient_Cluster ..> RedisClient_Cluster_Pipeline : new RedisClient_Cluster ..> RedisClient_Cluster_Transaction : new RedisClient_Cluster ..> RedisClient_Cluster_OptimisticLocking : new RedisClient_Cluster ..> RedisClient_Cluster_PubSub : new RedisClient_Cluster ..> RedisClient_Cluster_Router : new RedisClient_Cluster_Pipeline ..> RedisClient_Cluster_Router : use RedisClient_Cluster_Transaction ..> RedisClient_Cluster_Router : use RedisClient_Cluster_OptimisticLocking ..> RedisClient_Cluster_Router : use RedisClient_Cluster_PubSub ..> RedisClient_Cluster_Router : use RedisClient_Cluster_Router ..> RedisClient_Cluster_Node : new RedisClient_Cluster_Router ..> RedisClient_Cluster_Command : new RedisClient_Cluster_Router ..> RedisClient_Cluster_OptimisticLocking : new RedisClient_Cluster_Router ..> module_RedisClient_Cluster_KeySlotConverter : call RedisClient_Cluster_Router ..> module_RedisClient_Cluster_NodeKey : call RedisClient_Cluster_Node_PrimaryOnly --|> RedisClient_Cluster_Node_BaseTopology : inherit RedisClient_Cluster_Node_RandomReplica --|> RedisClient_Cluster_Node_BaseTopology : inherit RedisClient_Cluster_Node_RandomReplicaOrPrimary --|> RedisClient_Cluster_Node_BaseTopology : inherit RedisClient_Cluster_Node_LatencyReplica --|> RedisClient_Cluster_Node_BaseTopology : inherit RedisClient_Cluster_Node ..> RedisClient_Cluster_Node_PrimaryOnly : new RedisClient_Cluster_Node ..> RedisClient_Cluster_Node_RandomReplica : new RedisClient_Cluster_Node ..> RedisClient_Cluster_Node_RandomReplicaOrPrimary : new RedisClient_Cluster_Node ..> RedisClient_Cluster_Node_LatencyReplica : new ``` redis-rb-redis-cluster-client-aaf9e2e/docs/cloud_services.md000066400000000000000000000023531517214044400243260ustar00rootroot00000000000000# Architectures of Redis cluster with SSL/TLS as imagined ## AWS The client can connect to the nodes directly. The endpoint is just a CNAME record of DNS. It is as simple as that. ```mermaid graph TB client(Cluster Client) subgraph Amazon ElastiCache for Redis node0(Node0) node1(Node1) node2(Node2) end node0-.-node1-.-node2-.-node0 client--rediss://node0.example.com:6379-->node0 client--rediss://node1.example.com:6379-->node1 client--rediss://node2.example.com:6379-->node2 ``` ## Microsoft Azure The service provides a single IP address and multiple ports mapped to each node. The endpoint doesn't support redirection. It does only SSL/TLS termination. ```mermaid graph TB client(Cluster Client) subgraph Azure Cache for Redis subgraph Endpoint endpoint(Active) endpoint_sb(Standby) end subgraph Cluster node0(Node0) node1(Node1) node2(Node2) end end endpoint-.-endpoint_sb node0-.-node1-.-node2-.-node0 client--rediss://endpoint.example.com:15000-->endpoint--redis://node0:6379-->node0 client--rediss://endpoint.example.com:15001-->endpoint--redis://node1:6379-->node1 client--rediss://endpoint.example.com:15002-->endpoint--redis://node2:6379-->node2 ``` redis-rb-redis-cluster-client-aaf9e2e/docs/sequence_diagrams.md000066400000000000000000000020021517214044400247630ustar00rootroot00000000000000# How does cluster client work? ```mermaid sequenceDiagram participant Client participant Server Shard 1 participant Server Shard 2 participant Server Shard 3 Note over Client,Server Shard 3: RedisClient.cluster(nodes: %w[redis://node1:6379]).new_client Client->>+Server Shard 1: CLUSTER SHARDS Server Shard 1-->>-Client: nodes and slots data Note over Client,Server Shard 3: Falls back to CLUSTER NODES for older Redis versions Client->>+Server Shard 1: GET key1 Server Shard 1-->>-Client: value1 Client->>+Server Shard 2: GET key2 Server Shard 2-->>-Client: value2 Client->>+Server Shard 3: GET key3 Server Shard 3-->>-Client: value3 Client->>+Server Shard 3: GET key1 Server Shard 3-->>-Client: MOVED Server Shard 1 Note over Client,Server Shard 3: Client needs to redirect to correct node Client->>+Server Shard 2: MGET key2 key3 Server Shard 2-->>-Client: CROSSSLOTS Note over Client,Server Shard 2: Client cannot command across shards ``` redis-rb-redis-cluster-client-aaf9e2e/lib/000077500000000000000000000000001517214044400206065ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/lib/redis-cluster-client.rb000066400000000000000000000000761517214044400251770ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_cluster_client' redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/000077500000000000000000000000001517214044400232525ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster.rb000066400000000000000000000125301517214044400252610ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client/cluster/concurrent_worker' require 'redis_client/cluster/pipeline' require 'redis_client/cluster/pub_sub' require 'redis_client/cluster/router' require 'redis_client/cluster/transaction' require 'redis_client/cluster/optimistic_locking' class RedisClient class Cluster ZERO_CURSOR_FOR_SCAN = '0' private_constant :ZERO_CURSOR_FOR_SCAN attr_reader :config def initialize(config = nil, pool: nil, concurrency: nil, **kwargs) @config = config.nil? ? ClusterConfig.new(**kwargs) : config @concurrent_worker = ::RedisClient::Cluster::ConcurrentWorker.create(**(concurrency || {})) @command_builder = @config.command_builder @pool = pool @kwargs = kwargs @router = nil @mutex = Mutex.new end def inspect node_keys = @router.nil? ? @config.startup_nodes.keys : router.node_keys "#<#{self.class.name} #{node_keys.join(', ')}>" end def call(*args, **kwargs, &block) command = @command_builder.generate(args, kwargs) router.send_command(:call_v, command, &block) end def call_v(command, &block) command = @command_builder.generate(command) router.send_command(:call_v, command, &block) end def call_once(*args, **kwargs, &block) command = @command_builder.generate(args, kwargs) router.send_command(:call_once_v, command, &block) end def call_once_v(command, &block) command = @command_builder.generate(command) router.send_command(:call_once_v, command, &block) end def blocking_call(timeout, *args, **kwargs, &block) command = @command_builder.generate(args, kwargs) router.send_command(:blocking_call_v, command, timeout, &block) end def blocking_call_v(timeout, command, &block) command = @command_builder.generate(command) router.send_command(:blocking_call_v, command, timeout, &block) end def scan(*args, **kwargs, &block) return to_enum(__callee__, *args, **kwargs) unless block_given? command = @command_builder.generate(['scan', ZERO_CURSOR_FOR_SCAN] + args, kwargs) seed = Random.new_seed loop do cursor, keys = router.scan(command, seed: seed) command[1] = cursor keys.each(&block) break if cursor == ZERO_CURSOR_FOR_SCAN end end def sscan(key, *args, **kwargs, &block) return to_enum(__callee__, key, *args, **kwargs) unless block_given? command = @command_builder.generate(['sscan', key, ZERO_CURSOR_FOR_SCAN] + args, kwargs) router.scan_single_key(command, arity: 1, &block) end def hscan(key, *args, **kwargs, &block) return to_enum(__callee__, key, *args, **kwargs) unless block_given? command = @command_builder.generate(['hscan', key, ZERO_CURSOR_FOR_SCAN] + args, kwargs) router.scan_single_key(command, arity: 2, &block) end def zscan(key, *args, **kwargs, &block) return to_enum(__callee__, key, *args, **kwargs) unless block_given? command = @command_builder.generate(['zscan', key, ZERO_CURSOR_FOR_SCAN] + args, kwargs) router.scan_single_key(command, arity: 2, &block) end def pipelined(exception: true) seed = @config.use_replica? && @config.replica_affinity == :random ? nil : Random.new_seed pipeline = ::RedisClient::Cluster::Pipeline.new( router, @command_builder, @concurrent_worker, exception: exception, seed: seed ) yield pipeline return [] if pipeline.empty? pipeline.execute end def multi(watch: nil) if watch.nil? || watch.empty? transaction = ::RedisClient::Cluster::Transaction.new(router, @command_builder) yield transaction return transaction.execute end ::RedisClient::Cluster::OptimisticLocking.new(router).watch(watch) do |c, slot, asking| transaction = ::RedisClient::Cluster::Transaction.new( router, @command_builder, node: c, slot: slot, asking: asking ) yield transaction transaction.execute end end def pubsub ::RedisClient::Cluster::PubSub.new(router, @command_builder) end # Compatibility layer for RedisClient::Pooled def with(_options = nil) yield self end alias then with # Compatibility layer for RedisClient::HashRing def node_for(_key) self end # Compatibility layer for RedisClient::HashRing def nodes_for(*keys) keys.flatten! { self => keys } end # Compatibility layer for RedisClient::HashRing def nodes [self].freeze end def close @router&.close @concurrent_worker.close nil end private def router return @router unless @router.nil? @mutex.synchronize do @router ||= ::RedisClient::Cluster::Router.new(@config, @concurrent_worker, pool: @pool, **@kwargs) end end def method_missing(name, *args, **kwargs, &block) cmd = name.respond_to?(:name) ? name.name : name.to_s if router.command_exists?(cmd) args.unshift(cmd) command = @command_builder.generate(args, kwargs) return router.send_command(:call_v, command, &block) end super end def respond_to_missing?(name, include_private = false) return true if router.command_exists?(name) super end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/000077500000000000000000000000001517214044400247335ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/command.rb000066400000000000000000000076131517214044400267050ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' require 'redis_client/cluster/errors' require 'redis_client/cluster/key_slot_converter' class RedisClient class Cluster class Command EMPTY_STRING = '' EMPTY_HASH = {}.freeze EMPTY_ARRAY = [].freeze private_constant :EMPTY_STRING, :EMPTY_HASH, :EMPTY_ARRAY Detail = Struct.new( 'RedisCommand', :first_key_position, :key_step, :write?, :readonly?, keyword_init: true ) class << self def load(nodes, slow_command_timeout: -1) # rubocop:disable Metrics/AbcSize cmd = errors = nil nodes&.each do |node| regular_timeout = node.read_timeout node.read_timeout = slow_command_timeout > 0.0 ? slow_command_timeout : regular_timeout reply = node.call('command') node.read_timeout = regular_timeout commands = parse_command_reply(reply) cmd = ::RedisClient::Cluster::Command.new(commands) break rescue ::RedisClient::Error => e errors ||= [] errors << e end return cmd unless cmd.nil? raise ::RedisClient::Cluster::InitialSetupError.from_errors(errors) end private def parse_command_reply(rows) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity rows&.each_with_object({}) do |row, acc| next if row.first.nil? # TODO: in redis 7.0 or later, subcommand information included in the command reply pos = case row.first when 'eval', 'evalsha', 'zinterstore', 'zunionstore' then 3 when 'object', 'xgroup' then 2 when 'migrate', 'xread', 'xreadgroup' then 0 else row[3] end writable = case row.first when 'xgroup' then true else row[2].include?('write') end acc[row.first] = ::RedisClient::Cluster::Command::Detail.new( first_key_position: pos, key_step: row[5], write?: writable, readonly?: row[2].include?('readonly') ) end.freeze || EMPTY_HASH end end def initialize(commands) @commands = commands || EMPTY_HASH end def extract_first_key(command) i = determine_first_key_position(command) return EMPTY_STRING if i == 0 command[i] end def should_send_to_primary?(command) find_command_info(command.first)&.write? end def should_send_to_replica?(command) find_command_info(command.first)&.readonly? end def exists?(name) @commands.key?(name) || @commands.key?(name.to_s.downcase(:ascii)) end private def find_command_info(name) @commands[name] || @commands[name.to_s.downcase(:ascii)] end def determine_first_key_position(command) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/PerceivedComplexity i = find_command_info(command.first)&.first_key_position.to_i return i if i > 0 cmd_name = command.first if cmd_name.casecmp('xread').zero? determine_optional_key_position(command, 'streams') elsif cmd_name.casecmp('xreadgroup').zero? determine_optional_key_position(command, 'streams') elsif cmd_name.casecmp('migrate').zero? command[3].empty? ? determine_optional_key_position(command, 'keys') : 3 elsif cmd_name.casecmp('memory').zero? command[1].to_s.casecmp('usage').zero? ? 2 : 0 else i end end def determine_optional_key_position(command, option_name) i = command.index { |v| v.to_s.casecmp(option_name).zero? } i.nil? ? 0 : i + 1 end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/concurrent_worker.rb000066400000000000000000000043541517214044400310410ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client/cluster/concurrent_worker/on_demand' require 'redis_client/cluster/concurrent_worker/pooled' require 'redis_client/cluster/concurrent_worker/none' class RedisClient class Cluster module ConcurrentWorker InvalidNumberOfTasks = Class.new(StandardError) class Group Task = Struct.new( 'RedisClusterClientConcurrentWorkerTask', :id, :queue, :args, :kwargs, :block, :result, keyword_init: true ) do def exec self[:result] = block&.call(*args, **kwargs) rescue StandardError => e self[:result] = e ensure done end def done queue&.push(self) rescue ClosedQueueError # something was wrong end end def initialize(worker:, queue:, size:) @worker = worker @queue = queue @size = size @count = 0 end def push(id, *args, **kwargs, &block) raise InvalidNumberOfTasks, "max size reached: #{@count}" if @count == @size task = Task.new(id: id, queue: @queue, args: args, kwargs: kwargs, block: block) @worker.push(task) @count += 1 nil end def each raise InvalidNumberOfTasks, "expected: #{@size}, actual: #{@count}" if @count != @size @size.times do task = @queue.pop yield(task.id, task.result) end nil end def close @queue.clear @queue.close if @queue.respond_to?(:close) @count = 0 nil end def inspect "#<#{self.class.name} size: #{@count}, max: #{@size}, worker: #{@worker.class.name}>" end end module_function def create(model: :none, size: 5) case model when :none then ::RedisClient::Cluster::ConcurrentWorker::None.new when :on_demand then ::RedisClient::Cluster::ConcurrentWorker::OnDemand.new(size: size) when :pooled then ::RedisClient::Cluster::ConcurrentWorker::Pooled.new(size: size) else raise ArgumentError, "unknown model: #{model}" end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/concurrent_worker/000077500000000000000000000000001517214044400305065ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/concurrent_worker/none.rb000066400000000000000000000007521517214044400317760ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster module ConcurrentWorker class None def new_group(size:) ::RedisClient::Cluster::ConcurrentWorker::Group.new( worker: self, queue: [], size: size ) end def push(task) task.exec end def close; end def inspect "#<#{self.class.name} main thread only>" end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/concurrent_worker/on_demand.rb000066400000000000000000000016511517214044400327620ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster module ConcurrentWorker class OnDemand def initialize(size:) raise ArgumentError, "size must be positive: #{size}" unless size.positive? @q = SizedQueue.new(size) end def new_group(size:) ::RedisClient::Cluster::ConcurrentWorker::Group.new( worker: self, queue: SizedQueue.new(size), size: size ) end def push(task) @q << spawn_worker(task, @q) end def close @q.clear @q.close nil end def inspect "#<#{self.class.name} active: #{@q.size}, max: #{@q.max}>" end private def spawn_worker(task, queue) Thread.new(task, queue) do |t, q| t.exec q.pop end end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/concurrent_worker/pooled.rb000066400000000000000000000043021517214044400323140ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client/pid_cache' class RedisClient class Cluster module ConcurrentWorker # This class is just an experimental implementation. # Ruby VM allocates 1 MB memory as a stack for a thread. # It is a fixed size but we can modify the size with some environment variables. # So it consumes memory 1 MB multiplied a number of workers. class Pooled IO_ERROR_NEVER = { IOError => :never }.freeze IO_ERROR_IMMEDIATE = { IOError => :immediate }.freeze private_constant :IO_ERROR_NEVER, :IO_ERROR_IMMEDIATE def initialize(size:) raise ArgumentError, "size must be positive: #{size}" unless size.positive? @size = size setup end def new_group(size:) reset if @pid != ::RedisClient::PIDCache.pid ensure_workers if @workers.first.nil? ::RedisClient::Cluster::ConcurrentWorker::Group.new( worker: self, queue: SizedQueue.new(size), size: size ) end def push(task) @q << task end def close @q.clear workers = @workers.compact workers.each(&:exit) workers.each(&:join) @workers.clear @q.close @pid = nil nil end def inspect "#<#{self.class.name} tasks: #{@q.size}, workers: #{@size}>" end private def setup @q = Queue.new @workers = Array.new(@size) @pid = ::RedisClient::PIDCache.pid end def reset close setup end def ensure_workers @size.times do |i| @workers[i] = spawn_worker unless @workers[i]&.alive? end end def spawn_worker Thread.new(@q) do |q| Thread.handle_interrupt(IO_ERROR_NEVER) do loop do Thread.handle_interrupt(IO_ERROR_IMMEDIATE) do q.pop.exec end end end rescue IOError # stream closed in another thread end end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/error_identification.rb000066400000000000000000000021701517214044400314620ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster module ErrorIdentification def self.client_owns_error?(err, client) return true unless identifiable?(err) err.from?(client) end def self.identifiable?(err) err.is_a?(TaggedError) end module TaggedError attr_accessor :config_instance def from?(client) client.config.equal?(config_instance) end end module Middleware def connect(config) super rescue RedisClient::Error => e identify_error(e, config) raise end def call(_command, config) super rescue RedisClient::Error => e identify_error(e, config) raise end def call_pipelined(_command, config) super rescue RedisClient::Error => e identify_error(e, config) raise end private def identify_error(err, config) err.singleton_class.include(TaggedError) err.config_instance = config end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/errors.rb000066400000000000000000000045371517214044400266050ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' class RedisClient class Cluster class Error < ::RedisClient::Error def with_config(config) @config = config self end end ERR_ARG_NORMALIZATION = ->(arg) { Array[arg].flatten.reject { |e| e.nil? || (e.respond_to?(:empty?) && e.empty?) } } Ractor.make_shareable(ERR_ARG_NORMALIZATION) if Object.const_defined?(:Ractor, false) && Ractor.respond_to?(:make_shareable) private_constant :ERR_ARG_NORMALIZATION class InitialSetupError < Error def self.from_errors(errors) msg = ERR_ARG_NORMALIZATION.call(errors).map(&:message).uniq.join(',') new("Redis client could not fetch cluster information: #{msg}") end end class OrchestrationCommandNotSupported < Error def self.from_command(command) str = ERR_ARG_NORMALIZATION.call(command).map(&:to_s).join(' ').upcase msg = "#{str} command should be used with care " \ 'only by applications orchestrating Redis Cluster, like redis-cli, ' \ 'and the command if used out of the right context can leave the cluster ' \ 'in a wrong state or cause data loss.' new(msg) end end class ErrorCollection < Error EMPTY_HASH = {}.freeze private_constant :EMPTY_HASH attr_reader :errors def self.with_errors(errors) if !errors.is_a?(Hash) || errors.empty? new(errors.to_s).with_errors(EMPTY_HASH) else messages = errors.map { |node_key, error| "#{node_key}: (#{error.class}) #{error.message}" }.freeze new(messages.join(', ')).with_errors(errors) end end def initialize(error_message = nil) @errors = nil super end def with_errors(errors) @errors = errors if @errors.nil? self end end class AmbiguousNodeError < Error def self.from_command(command) new("Cluster client doesn't know which node the #{command} command should be sent to.") end end class NodeMightBeDown < Error def initialize(_error_message = nil) super( 'The client is trying to fetch the latest cluster state ' \ 'because a subset of nodes might be down. ' \ 'It might continue to raise errors for a while.' ) end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/key_slot_converter.rb000066400000000000000000000070471517214044400312100ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster module KeySlotConverter HASH_SLOTS = 16_384 EMPTY_STRING = '' LEFT_BRACKET = '{' RIGHT_BRACKET = '}' XMODEM_CRC16_LOOKUP = [ 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0 ].freeze private_constant :HASH_SLOTS, :EMPTY_STRING, :LEFT_BRACKET, :RIGHT_BRACKET, :XMODEM_CRC16_LOOKUP module_function def convert(key) return nil if key.nil? hash_tag = extract_hash_tag(key) key = hash_tag unless hash_tag.empty? crc = 0 key.each_byte do |b| crc = ((crc << 8) & 0xffff) ^ XMODEM_CRC16_LOOKUP[((crc >> 8) ^ b) & 0xff] end crc % HASH_SLOTS end # @see https://redis.io/topics/cluster-spec#keys-hash-tags Keys hash tags def extract_hash_tag(key) key = key.to_s s = key.index(LEFT_BRACKET) return EMPTY_STRING if s.nil? e = key.index(RIGHT_BRACKET, s + 1) return EMPTY_STRING if e.nil? s + 1 < e ? key[s + 1, e - s - 1] : EMPTY_STRING end def hash_tag_included?(key) key = key.to_s s = key.index(LEFT_BRACKET) return false if s.nil? e = key.index(RIGHT_BRACKET, s + 1) return false if e.nil? s + 1 < e end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/node.rb000066400000000000000000000425541517214044400262170ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' require 'redis_client/config' require 'redis_client/cluster/errors' require 'redis_client/cluster/node/primary_only' require 'redis_client/cluster/node/random_replica' require 'redis_client/cluster/node/random_replica_or_primary' require 'redis_client/cluster/node/latency_replica' require 'redis_client/cluster/node_key' class RedisClient class Cluster class Node include Enumerable # less memory consumption, but slow USE_CHAR_ARRAY_SLOT = Integer(ENV.fetch('REDIS_CLIENT_USE_CHAR_ARRAY_SLOT', 1)) == 1 SLOT_SIZE = 16_384 MIN_SLOT = 0 MAX_SLOT = SLOT_SIZE - 1 DEAD_FLAGS = %w[fail? fail handshake noaddr noflags].freeze ROLE_FLAGS = %w[master slave].freeze EMPTY_ARRAY = [].freeze EMPTY_HASH = {}.freeze EMPTY_STRING = '' JITTER_WINDOW = (3_000_000...10_000_000).freeze # micro seconds private_constant :USE_CHAR_ARRAY_SLOT, :SLOT_SIZE, :MIN_SLOT, :MAX_SLOT, :DEAD_FLAGS, :ROLE_FLAGS, :EMPTY_ARRAY, :EMPTY_HASH, :EMPTY_STRING, :JITTER_WINDOW ReloadNeeded = Class.new(::RedisClient::Cluster::Error) Info = Struct.new( 'RedisClusterNode', :id, :node_key, :role, :primary_id, :ping_sent, :pong_recv, :config_epoch, :link_state, :slots, keyword_init: true ) do def primary? role == 'master' end def replica? role == 'slave' end def serialize(str) str << id << node_key << role << primary_id end end class CharArray BASE = '' PADDING = '0' private_constant :BASE, :PADDING def initialize(size, elements) @elements = elements @string = String.new(BASE, encoding: Encoding::BINARY, capacity: size) size.times { @string << PADDING } end def [](index) raise IndexError if index < 0 return if index >= @string.bytesize @elements[@string.getbyte(index)] end def []=(index, element) raise IndexError if index < 0 return if index >= @string.bytesize pos = @elements.find_index(element) # O(N) if pos.nil? raise(RangeError, 'full of elements') if @elements.size >= 256 pos = @elements.size @elements << element end @string.setbyte(index, pos) end end class Config < ::RedisClient::Config def initialize(scale_read: false, **kwargs) @scale_read = scale_read super(**kwargs) end def connection_prelude prelude = super.dup prelude << ['readonly'] if @scale_read prelude.freeze end end def initialize(concurrent_worker, config:, pool: nil, **kwargs) @concurrent_worker = concurrent_worker @slots = build_slot_node_mappings(EMPTY_ARRAY) @replications = build_replication_mappings(EMPTY_ARRAY) klass = make_topology_class(config.use_replica?, config.replica_affinity) @topology = klass.new(pool, @concurrent_worker, **kwargs) @config = config @mutex = Mutex.new @next_reload_time = nil @random = Random.new end def inspect "#<#{self.class.name} #{node_keys.join(', ')}>" end def each(&block) @topology.clients.each_value(&block) end def sample @topology.clients.values.sample end def node_keys @topology.clients.keys.sort end def find_by(node_key) raise ReloadNeeded if node_key.nil? || !@topology.clients.key?(node_key) @topology.clients.fetch(node_key) end def call_all(method, command, args, &block) call_multiple_nodes!(@topology.clients, method, command, args, &block) end def call_primaries(method, command, args, &block) call_multiple_nodes!(@topology.primary_clients, method, command, args, &block) end def call_replicas(method, command, args, &block) call_multiple_nodes!(@topology.replica_clients, method, command, args, &block) end def send_ping(method, command, args, &block) result_values, errors = call_multiple_nodes(@topology.clients, method, command, args, &block) return result_values if errors.nil? || errors.empty? raise ReloadNeeded if errors.values.any?(::RedisClient::ConnectionError) raise ::RedisClient::Cluster::ErrorCollection.with_errors(errors) end def clients_for_scanning(seed: nil) @topology.clients_for_scanning(seed: seed).values.sort_by { |c| "#{c.config.host}-#{c.config.port}" } end def clients @topology.clients.values end def primary_clients @topology.primary_clients.values end def replica_clients @topology.replica_clients.values end def find_node_key_of_primary(slot) return if slot.nil? slot = Integer(slot) return if slot < MIN_SLOT || slot > MAX_SLOT @slots[slot] end def find_node_key_of_replica(slot, seed: nil) primary_node_key = find_node_key_of_primary(slot) @topology.find_node_key_of_replica(primary_node_key, seed: seed) end def any_primary_node_key(seed: nil) @topology.any_primary_node_key(seed: seed) end def any_replica_node_key(seed: nil) @topology.any_replica_node_key(seed: seed) end def update_slot(slot, node_key) return unless @mutex.try_lock @slots[slot] = node_key rescue RangeError @slots = Array.new(SLOT_SIZE) { |i| @slots[i] } @slots[slot] = node_key ensure @mutex.unlock if @mutex.owned? end def try_reload! with_reload_lock do with_reload_jitter do with_startup_clients(@config.max_startup_sample) do |clients| reload!(clients) end end end end private def make_topology_class(with_replica, replica_affinity) if with_replica && replica_affinity == :random ::RedisClient::Cluster::Node::RandomReplica elsif with_replica && replica_affinity == :random_with_primary ::RedisClient::Cluster::Node::RandomReplicaOrPrimary elsif with_replica && replica_affinity == :latency ::RedisClient::Cluster::Node::LatencyReplica else ::RedisClient::Cluster::Node::PrimaryOnly end end def build_slot_node_mappings(node_info_list) slots = make_array_for_slot_node_mappings(node_info_list) node_info_list.each do |info| next if info.slots.nil? || info.slots.empty? info.slots.each { |start, last| (start..last).each { |i| slots[i] = info.node_key } } end slots end def make_array_for_slot_node_mappings(node_info_list) return Array.new(SLOT_SIZE) if !USE_CHAR_ARRAY_SLOT || node_info_list.count(&:primary?) > 256 primary_node_keys = node_info_list.select(&:primary?).map(&:node_key) ::RedisClient::Cluster::Node::CharArray.new(SLOT_SIZE, primary_node_keys) end def build_replication_mappings(node_info_list) # rubocop:disable Metrics/AbcSize dict = node_info_list.to_h { |info| [info.id, info] } node_info_list.each_with_object(Hash.new { |h, k| h[k] = [] }) do |info, acc| primary_info = dict[info.primary_id] acc[primary_info.node_key] << info.node_key unless primary_info.nil? acc[info.node_key] if info.primary? # for the primary which have no replicas end end def call_multiple_nodes(clients, method, command, args, &block) results, errors = try_map(clients) do |_, client| client.public_send(method, *args, command, &block) end [results&.values, errors] end def call_multiple_nodes!(clients, method, command, args, &block) result_values, errors = call_multiple_nodes(clients, method, command, args, &block) return result_values if errors.nil? || errors.empty? raise ::RedisClient::Cluster::ErrorCollection.with_errors(errors) end def try_map(clients, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity return [{}, {}] if clients.empty? work_group = @concurrent_worker.new_group(size: clients.size) clients.each do |node_key, client| work_group.push(node_key, node_key, client, block) do |nk, cli, blk| blk.call(nk, cli) rescue StandardError => e e end end results = errors = nil work_group.each do |node_key, v| case v when StandardError errors ||= {} errors[node_key] = v else results ||= {} results[node_key] = v end end work_group.close [results, errors] end def refetch_node_info_list(startup_clients) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity startup_size = startup_clients.size work_group = @concurrent_worker.new_group(size: startup_size) startup_clients.each_with_index do |raw_client, i| work_group.push(i, raw_client) do |client| regular_timeout = client.read_timeout client.read_timeout = @config.slow_command_timeout > 0.0 ? @config.slow_command_timeout : regular_timeout fetch_cluster_state(client) rescue StandardError => e e ensure client.read_timeout = regular_timeout client&.close end end node_info_list = errors = nil work_group.each do |i, v| case v when StandardError errors ||= Array.new(startup_size) errors[i] = v else node_info_list ||= Array.new(startup_size) node_info_list[i] = v end end work_group.close raise ::RedisClient::Cluster::InitialSetupError.from_errors(errors) if node_info_list.nil? grouped = node_info_list.compact.group_by do |info_list| info_list.sort_by!(&:id) info_list.each_with_object(String.new(capacity: 128 * info_list.size)) { |e, a| e.serialize(a) } end grouped.max_by { |_, v| v.size }[1].first end def fetch_cluster_state(client) reply = client.call_once('cluster', 'shards') parse_cluster_shards_reply(reply) rescue ::RedisClient::CommandError => e raise unless e.message.start_with?('ERR Unknown subcommand') reply = client.call_once('cluster', 'nodes') parse_cluster_node_reply(reply) end def parse_cluster_node_reply(reply) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity reply.each_line("\n", chomp: true).filter_map do |line| fields = line.split flags = fields[2].split(',') next unless fields[7] == 'connected' && (flags & DEAD_FLAGS).empty? slots = if fields[8].nil? EMPTY_ARRAY else fields[8..].reject { |str| str.start_with?('[') } .map { |str| str.split('-').map { |s| Integer(s) } } .map { |a| a.size == 1 ? a << a.first : a } .map(&:sort) end ::RedisClient::Cluster::Node::Info.new( id: fields[0], node_key: parse_node_key(fields[1]), role: (flags & ROLE_FLAGS).first, primary_id: fields[3], ping_sent: fields[4], pong_recv: fields[5], config_epoch: fields[6], link_state: fields[7], slots: slots ) end end def parse_cluster_slots_reply(reply) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity reply.group_by { |e| e[2][2] }.each_with_object([]) do |(primary_id, group), acc| slots = group.map { |e| e[0, 2] }.freeze group.first[2..].each do |arr| ip = arr[0] next if ip.nil? || ip.empty? || ip == '?' id = arr[2] role = id == primary_id ? 'master' : 'slave' acc << ::RedisClient::Cluster::Node::Info.new( id: id, node_key: NodeKey.build_from_host_port(ip, arr[1]), role: role, primary_id: role == 'master' ? EMPTY_STRING : primary_id, slots: role == 'master' ? slots : EMPTY_ARRAY ) end end end def parse_cluster_shards_reply(reply) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity reply.each_with_object([]) do |shard, acc| resp2 = shard.is_a?(Array) shard = shard.each_slice(2).to_h if resp2 nodes = shard.fetch('nodes') nodes = nodes.map { |n| n.each_slice(2).to_h } if resp2 primary_id = nodes.find { |n| n.fetch('role') == 'master' }.fetch('id') nodes.each do |node| ip = node.fetch('ip') next if node.fetch('health') != 'online' || ip.nil? || ip.empty? || ip == '?' role = node.fetch('role') acc << ::RedisClient::Cluster::Node::Info.new( id: node.fetch('id'), node_key: NodeKey.build_from_host_port(ip, node['port'] || node['tls-port']), role: role == 'master' ? role : 'slave', primary_id: role == 'master' ? EMPTY_STRING : primary_id, slots: role == 'master' ? shard.fetch('slots').each_slice(2).to_a.freeze : EMPTY_ARRAY ) end end end # As redirection node_key is dependent on `cluster-preferred-endpoint-type` config, # node_key should use hostname if present in CLUSTER NODES output. # # See https://redis.io/commands/cluster-nodes/ for details on the output format. # node_address matches the format: def parse_node_key(node_address) ip_chunk, hostname, _auxiliaries = node_address.split(',') ip_port_string = ip_chunk.split('@').first return ip_port_string if hostname.nil? || hostname.empty? port = ip_port_string.split(':')[1] "#{hostname}:#{port}" end def reload!(clients) @node_info = refetch_node_info_list(clients) @node_configs = @node_info.to_h do |node_info| [node_info.node_key, @config.client_config_for_node(node_info.node_key)] end @slots = build_slot_node_mappings(@node_info) @replications = build_replication_mappings(@node_info) @topology.process_topology_update!(@replications, @node_configs) end def with_startup_clients(count) # rubocop:disable Metrics/AbcSize if @config.connect_with_original_config # If connect_with_original_config is set, that means we need to build actual client objects # and close them, so that we e.g. re-resolve a DNS entry with the cluster nodes in it. begin # Memoize the startup clients, so we maintain RedisClient's internal circuit breaker configuration # if it's set. @startup_clients ||= @config.startup_nodes.values.sample(count).map do |node_config| ::RedisClient::Cluster::Node::Config.new(**node_config).new_client end yield @startup_clients ensure # Close the startup clients when we're done, so we don't maintain pointless open connections to # the cluster though @startup_clients&.each(&:close) end else # (re-)connect using nodes we already know about. # If this is the first time we're connecting to the cluster, we need to seed the topology with the # startup clients though. @topology.process_topology_update!({}, @config.startup_nodes) if @topology.clients.empty? yield @topology.clients.values.sample(count) end end def with_reload_jitter return unless @next_reload_time.nil? || obtain_current_time >= @next_reload_time yield @next_reload_time = obtain_current_time + @random.rand(JITTER_WINDOW) end def with_reload_lock # What should happen with concurrent calls #try_reload! This is a realistic possibility if the cluster goes into # a CLUSTERDOWN state, and we're using a pooled backend. Every thread will independently discover this, and # call #try_reload!. # For now, if a reload is in progress by a thread, the other threads do not wait for that to complete, and # they throw an error. # Probably in the future we should add a circuit breaker to #try_reload! itself, and stop trying if the cluster is # obviously not working. return unless @mutex.try_lock yield ensure @mutex.unlock if @mutex.owned? end def obtain_current_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) end def bypass_reload! # DO NOT USE THIS METHOD with_reload_lock do with_startup_clients(@config.max_startup_sample) do |clients| reload!(clients) end end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/node/000077500000000000000000000000001517214044400256605ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/node/base_topology.rb000066400000000000000000000045501517214044400310570ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster class Node class BaseTopology IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze EMPTY_HASH = {}.freeze EMPTY_ARRAY = [].freeze private_constant :IGNORE_GENERIC_CONFIG_KEYS, :EMPTY_HASH, :EMPTY_ARRAY attr_reader :clients, :primary_clients, :replica_clients def initialize(pool, concurrent_worker, **kwargs) @pool = pool @clients = {} @client_options = kwargs.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) } @concurrent_worker = concurrent_worker @replications = EMPTY_HASH @primary_node_keys = EMPTY_ARRAY @replica_node_keys = EMPTY_ARRAY @primary_clients = EMPTY_ARRAY @replica_clients = EMPTY_ARRAY end def any_primary_node_key(seed: nil) random = seed.nil? ? Random : Random.new(seed) @primary_node_keys.sample(random: random) end def process_topology_update!(replications, options) # rubocop:disable Metrics/AbcSize @replications = replications.freeze @primary_node_keys = @replications.keys.sort.select { |k| options.key?(k) }.freeze @replica_node_keys = @replications.values.flatten.sort.select { |k| options.key?(k) }.freeze # Disconnect from nodes that we no longer want, and connect to nodes we're not connected to yet disconnect_from_unwanted_nodes(options) connect_to_new_nodes(options) @primary_clients, @replica_clients = @clients.partition { |k, _| @primary_node_keys.include?(k) }.map(&:to_h) @primary_clients.freeze @replica_clients.freeze end private def disconnect_from_unwanted_nodes(options) (@clients.keys - options.keys).each do |node_key| @clients.delete(node_key).close end end def connect_to_new_nodes(options) (options.keys - @clients.keys).each do |node_key| option = options[node_key].merge(@client_options) config = ::RedisClient::Cluster::Node::Config.new(scale_read: @replica_node_keys.include?(node_key), **option) client = @pool.nil? ? config.new_client : config.new_pool(**@pool) @clients[node_key] = client end end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/node/latency_replica.rb000066400000000000000000000056261517214044400313540ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client/cluster/node/base_topology' class RedisClient class Cluster class Node class LatencyReplica < BaseTopology DUMMY_LATENCY_MSEC = 100 * 1000 * 1000 MEASURE_ATTEMPT_COUNT = 10 private_constant :DUMMY_LATENCY_MSEC, :MEASURE_ATTEMPT_COUNT def clients_for_scanning(seed: nil) # rubocop:disable Lint/UnusedMethodArgument @clients_for_scanning end def find_node_key_of_replica(primary_node_key, seed: nil) # rubocop:disable Lint/UnusedMethodArgument @replications.fetch(primary_node_key, EMPTY_ARRAY).first || primary_node_key end def any_replica_node_key(seed: nil) random = seed.nil? ? Random : Random.new(seed) @existed_replicas.sample(random: random)&.first || any_primary_node_key(seed: seed) end def process_topology_update!(replications, options) super all_replica_clients = @clients.select { |k, _| @replica_node_keys.include?(k) } latencies = measure_latencies(all_replica_clients, @concurrent_worker) @replications.each_value { |keys| keys.sort_by! { |k| latencies.fetch(k) } } @replica_clients = select_replica_clients(@replications, @clients) @clients_for_scanning = select_clients_for_scanning(@replications, @clients) @existed_replicas = @replications.values.reject(&:empty?) end private def measure_latencies(clients, concurrent_worker) # rubocop:disable Metrics/AbcSize return {} if clients.empty? work_group = concurrent_worker.new_group(size: clients.size) clients.each do |node_key, client| work_group.push(node_key, client) do |cli| min = DUMMY_LATENCY_MSEC MEASURE_ATTEMPT_COUNT.times do starting = obtain_current_time cli.call_once('ping') duration = obtain_current_time - starting min = duration if duration < min end min rescue StandardError DUMMY_LATENCY_MSEC end end latencies = {} work_group.each { |node_key, v| latencies[node_key] = v } work_group.close latencies end def obtain_current_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) end def select_replica_clients(replications, clients) keys = replications.values.filter_map(&:first) clients.select { |k, _| keys.include?(k) } end def select_clients_for_scanning(replications, clients) keys = replications.map do |primary_node_key, replica_node_keys| replica_node_keys.empty? ? primary_node_key : replica_node_keys.first end clients.select { |k, _| keys.include?(k) } end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/node/primary_only.rb000066400000000000000000000021021517214044400307240ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client/cluster/node/base_topology' class RedisClient class Cluster class Node class PrimaryOnly < BaseTopology alias primary_clients clients alias replica_clients clients def clients_for_scanning(seed: nil) # rubocop:disable Lint/UnusedMethodArgument @clients end def find_node_key_of_replica(primary_node_key, seed: nil) # rubocop:disable Lint/UnusedMethodArgument primary_node_key end def any_primary_node_key(seed: nil) random = seed.nil? ? Random : Random.new(seed) @primary_node_keys.sample(random: random) end alias any_replica_node_key any_primary_node_key def process_topology_update!(replications, options) # Remove non-primary nodes from options (provided that we actually have any primaries at all) options = options.select { |node_key, _| replications.key?(node_key) } if replications.keys.any? super(replications, options) end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/node/random_replica.rb000066400000000000000000000022001517214044400311560ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client/cluster/node/base_topology' class RedisClient class Cluster class Node class RandomReplica < BaseTopology def replica_clients keys = @replications.values.filter_map(&:sample) @clients.select { |k, _| keys.include?(k) } end def clients_for_scanning(seed: nil) random = seed.nil? ? Random : Random.new(seed) keys = @replications.map do |primary_node_key, replica_node_keys| replica_node_keys.empty? ? primary_node_key : replica_node_keys.sample(random: random) end clients.select { |k, _| keys.include?(k) } end def find_node_key_of_replica(primary_node_key, seed: nil) random = seed.nil? ? Random : Random.new(seed) @replications.fetch(primary_node_key, EMPTY_ARRAY).sample(random: random) || primary_node_key end def any_replica_node_key(seed: nil) random = seed.nil? ? Random : Random.new(seed) @replica_node_keys.sample(random: random) || any_primary_node_key(seed: seed) end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/node/random_replica_or_primary.rb000066400000000000000000000035371517214044400334370ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client/cluster/node/base_topology' class RedisClient class Cluster class Node class RandomReplicaOrPrimary < BaseTopology def replica_clients keys = @replications.values.filter_map(&:sample) @clients.select { |k, _| keys.include?(k) } end def clients_for_scanning(seed: nil) random = seed.nil? ? Random : Random.new(seed) keys = @replications.map do |primary_node_key, replica_node_keys| decide_use_primary?(random, replica_node_keys.size) ? primary_node_key : replica_node_keys.sample(random: random) end clients.select { |k, _| keys.include?(k) } end def find_node_key_of_replica(primary_node_key, seed: nil) random = seed.nil? ? Random : Random.new(seed) replica_node_keys = @replications.fetch(primary_node_key, EMPTY_ARRAY) if decide_use_primary?(random, replica_node_keys.size) primary_node_key else replica_node_keys.sample(random: random) || primary_node_key end end def any_replica_node_key(seed: nil) random = seed.nil? ? Random : Random.new(seed) @replica_node_keys.sample(random: random) || any_primary_node_key(seed: seed) end private # Randomly equally likely choose node to read between primary and all replicas # e.g. 1 primary + 1 replica = 50% probability to read from primary # e.g. 1 primary + 2 replica = 33% probability to read from primary # e.g. 1 primary + 0 replica = 100% probability to read from primary def decide_use_primary?(random, replica_nodes) primary_nodes = 1.0 total = primary_nodes + replica_nodes random.rand < primary_nodes / total end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/node_key.rb000066400000000000000000000016411517214044400270570ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster # Node key's format is `:`. # It is different from node id. # Node id is internal identifying code in Redis Cluster. module NodeKey DELIMITER = ':' private_constant :DELIMITER module_function def hashify(node_key) host, port = split(node_key) { host: host, port: port } end def split(node_key) pos = node_key&.rindex(DELIMITER, -1) return [node_key, nil] if pos.nil? [node_key[0, pos], node_key[(pos + 1)..]] end def build_from_uri(uri) return '' if uri.nil? "#{uri.host}#{DELIMITER}#{uri.port}" end def build_from_host_port(host, port) "#{host}#{DELIMITER}#{port}" end def build_from_client(client) "#{client.config.host}#{DELIMITER}#{client.config.port}" end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/noop_command_builder.rb000066400000000000000000000003071517214044400314370ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster module NoopCommandBuilder module_function def generate(args, _kwargs = nil) args end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/optimistic_locking.rb000066400000000000000000000053611517214044400311570ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' require 'redis_client/cluster/transaction' class RedisClient class Cluster class OptimisticLocking def initialize(router) @router = router @asking = false end def watch(keys) # rubocop:disable Metrics/AbcSize slot = find_slot(keys) raise ::RedisClient::Cluster::Transaction::ConsistencyError, "unsafe watch: #{keys.join(' ')}" if slot.nil? handle_redirection(slot, retry_count: 1) do |nd| nd.with do |c| c.ensure_connected_cluster_scoped(retryable: false) do c.call('asking') if @asking c.call('watch', *keys) begin yield(c, slot, @asking) rescue ::RedisClient::ConnectionError # No need to unwatch on a connection error. raise rescue StandardError c.call('unwatch') raise end rescue ::RedisClient::CommandError => e @router.renew_cluster_state if e.message.start_with?('CLUSTERDOWN') raise end rescue ::RedisClient::ConnectionError @router.renew_cluster_state raise end end end private def handle_redirection(slot, retry_count: 1, &blk) # We have not yet selected a node for this transaction, initially, which means we can handle # redirections freely initially (i.e. for the first WATCH call) node = @router.find_primary_node_by_slot(slot) times_block_executed = 0 @router.handle_redirection(node, nil, retry_count: retry_count) do |nd| times_block_executed += 1 handle_asking_once(nd, &blk) end rescue ::RedisClient::ConnectionError # Deduct the number of retries that happened _inside_ router#handle_redirection from our remaining # _external_ retries. Always deduct at least one in case handle_redirection raises without trying the block. retry_count -= [times_block_executed, 1].min raise if retry_count < 0 retry end def handle_asking_once(node) yield node rescue ::RedisClient::CommandError => e raise unless ErrorIdentification.client_owns_error?(e, node) raise unless e.message.start_with?('ASK') node = @router.assign_asking_node(e.message) @asking = true yield node ensure @asking = false end def find_slot(keys) return if keys.empty? return if keys.any? { |k| k.nil? || k.empty? } slots = keys.map { |k| @router.find_slot_by_key(k) } return if slots.uniq.size != 1 slots.first end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/pipeline.rb000066400000000000000000000236741517214044400271010ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' require 'redis_client/cluster/errors' require 'redis_client/cluster/noop_command_builder' require 'redis_client/connection_mixin' require 'redis_client/middlewares' require 'redis_client/pooled' class RedisClient class Cluster class Pipeline class Extended < ::RedisClient::Pipeline attr_reader :outer_indices def initialize(...) super @outer_indices = nil end def add_outer_index(index) @outer_indices ||= [] @outer_indices << index end def get_inner_index(outer_index) @outer_indices&.find_index(outer_index) end def get_callee_method(inner_index) if @timeouts.is_a?(Array) && !@timeouts[inner_index].nil? :blocking_call_v elsif _retryable? :call_once_v else :call_v end end def get_command(inner_index) @commands.is_a?(Array) ? @commands[inner_index] : nil end def get_timeout(inner_index) @timeouts.is_a?(Array) ? @timeouts[inner_index] : nil end def get_block(inner_index) @blocks.is_a?(Array) ? @blocks[inner_index] : nil end end ::RedisClient::ConnectionMixin.module_eval do def call_pipelined_aware_of_redirection(commands, timeouts, exception:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity size = commands.size results = Array.new(commands.size) @pending_reads += size write_multi(commands) redirection_indices = stale_cluster_state = first_exception = nil size.times do |index| timeout = timeouts && timeouts[index] result = read(connection_timeout(timeout)) @pending_reads -= 1 if result.is_a?(::RedisClient::Error) result._set_command(commands[index]) result._set_config(config) if result.is_a?(::RedisClient::CommandError) && result.message.start_with?('MOVED', 'ASK') redirection_indices ||= [] redirection_indices << index elsif exception first_exception ||= result end stale_cluster_state = true if result.message.start_with?('CLUSTERDOWN') end results[index] = result end if redirection_indices err = ::RedisClient::Cluster::Pipeline::RedirectionNeeded.new err.replies = results err.indices = redirection_indices err.first_exception = first_exception raise err end if stale_cluster_state err = ::RedisClient::Cluster::Pipeline::StaleClusterState.new err.replies = results err.first_exception = first_exception raise err end raise first_exception if first_exception results end end ::RedisClient.class_eval do attr_reader :middlewares def ensure_connected_cluster_scoped(retryable: true, &block) ensure_connected(retryable: retryable, &block) end end ReplySizeError = Class.new(::RedisClient::Cluster::Error) class StaleClusterState < ::RedisClient::Cluster::Error attr_accessor :replies, :first_exception end class RedirectionNeeded < ::RedisClient::Cluster::Error attr_accessor :replies, :indices, :first_exception end def initialize(router, command_builder, concurrent_worker, exception:, seed: Random.new_seed) @router = router @command_builder = command_builder @concurrent_worker = concurrent_worker @exception = exception @seed = seed @pipelines = nil @size = 0 end def call(*args, **kwargs, &block) command = @command_builder.generate(args, kwargs) node_key = @router.find_node_key(command, seed: @seed) append_pipeline(node_key).call_v(command, &block) end def call_v(args, &block) command = @command_builder.generate(args) node_key = @router.find_node_key(command, seed: @seed) append_pipeline(node_key).call_v(command, &block) end def call_once(*args, **kwargs, &block) command = @command_builder.generate(args, kwargs) node_key = @router.find_node_key(command, seed: @seed) append_pipeline(node_key).call_once_v(command, &block) end def call_once_v(args, &block) command = @command_builder.generate(args) node_key = @router.find_node_key(command, seed: @seed) append_pipeline(node_key).call_once_v(command, &block) end def blocking_call(timeout, *args, **kwargs, &block) command = @command_builder.generate(args, kwargs) node_key = @router.find_node_key(command, seed: @seed) append_pipeline(node_key).blocking_call_v(timeout, command, &block) end def blocking_call_v(timeout, args, &block) command = @command_builder.generate(args) node_key = @router.find_node_key(command, seed: @seed) append_pipeline(node_key).blocking_call_v(timeout, command, &block) end def empty? @size.zero? end def execute # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity return if @pipelines.nil? || @pipelines.empty? work_group = @concurrent_worker.new_group(size: @pipelines.size) @pipelines.each do |node_key, pipeline| work_group.push(node_key, @router.find_node(node_key), pipeline) do |cli, pl| replies = do_pipelining(cli, pl) raise ReplySizeError, "commands: #{pl._size}, replies: #{replies.size}" if pl._size != replies.size replies end end all_replies = errors = required_redirections = cluster_state_errors = nil work_group.each do |node_key, v| case v when ::RedisClient::Cluster::Pipeline::RedirectionNeeded required_redirections ||= {} required_redirections[node_key] = v when ::RedisClient::Cluster::Pipeline::StaleClusterState cluster_state_errors ||= {} cluster_state_errors[node_key] = v when StandardError cluster_state_errors ||= {} if v.is_a?(::RedisClient::ConnectionError) errors ||= {} errors[node_key] = v else all_replies ||= Array.new(@size) @pipelines[node_key].outer_indices.each_with_index { |outer, inner| all_replies[outer] = v[inner] } end end work_group.close @router.renew_cluster_state if cluster_state_errors raise ::RedisClient::Cluster::ErrorCollection.with_errors(errors).with_config(@router.config) unless errors.nil? required_redirections&.each do |node_key, v| raise v.first_exception if v.first_exception all_replies ||= Array.new(@size) pipeline = @pipelines[node_key] v.indices.each { |i| v.replies[i] = handle_redirection(v.replies[i], pipeline, i) } pipeline.outer_indices.each_with_index { |outer, inner| all_replies[outer] = v.replies[inner] } end cluster_state_errors&.each do |node_key, v| raise v.first_exception if v.first_exception all_replies ||= Array.new(@size) @pipelines[node_key].outer_indices.each_with_index { |outer, inner| all_replies[outer] = v.replies[inner] } end all_replies end private def append_pipeline(node_key) @pipelines ||= {} @pipelines[node_key] ||= ::RedisClient::Cluster::Pipeline::Extended.new(::RedisClient::Cluster::NoopCommandBuilder) @pipelines[node_key].add_outer_index(@size) @size += 1 @pipelines[node_key] end def do_pipelining(client, pipeline) case client when ::RedisClient then send_pipeline(client, pipeline) when ::RedisClient::Pooled then client.with { |cli| send_pipeline(cli, pipeline) } else raise NotImplementedError, "#{client.class.name}#pipelined for cluster client" end end def send_pipeline(client, pipeline) results = client.ensure_connected_cluster_scoped(retryable: pipeline._retryable?) do |connection| commands = pipeline._commands client.middlewares.call_pipelined(commands, client.config) do connection.call_pipelined_aware_of_redirection(commands, pipeline._timeouts, exception: @exception) end end pipeline._coerce!(results) end def handle_redirection(err, pipeline, inner_index) return err unless err.is_a?(::RedisClient::CommandError) if err.message.start_with?('MOVED') node = @router.assign_redirection_node(err.message) try_redirection(node, pipeline, inner_index) elsif err.message.start_with?('ASK') node = @router.assign_asking_node(err.message) try_asking(node) ? try_redirection(node, pipeline, inner_index) : err else err end end def try_redirection(node, pipeline, inner_index) redirect_command(node, pipeline, inner_index) rescue StandardError => e @exception ? raise : e end def redirect_command(node, pipeline, inner_index) method = pipeline.get_callee_method(inner_index) command = pipeline.get_command(inner_index) timeout = pipeline.get_timeout(inner_index) block = pipeline.get_block(inner_index) args = timeout.nil? ? [] : [timeout] if block.nil? @router.send_command_to_node(node, method, command, args) else @router.send_command_to_node(node, method, command, args, &block) end end def try_asking(node) node.call('asking') == 'OK' rescue StandardError false end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/pub_sub.rb000066400000000000000000000130251517214044400267200ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' require 'redis_client/cluster/errors' class RedisClient class Cluster class PubSub class State IO_ERROR_NEVER = { IOError => :never }.freeze IO_ERROR_IMMEDIATE = { IOError => :immediate }.freeze private_constant :IO_ERROR_NEVER, :IO_ERROR_IMMEDIATE def initialize(client, queue) @client = client @worker = nil @queue = queue end def call(command) @client.call_v(command) end def ensure_worker @worker = spawn_worker(@client, @queue) unless @worker&.alive? end def close if @worker&.alive? @worker.exit @worker.join end @client.close rescue ::RedisClient::ConnectionError # ignore end private def spawn_worker(client, queue) # Ruby VM allocates 1 MB memory as a stack for a thread. # It is a fixed size but we can modify the size with some environment variables. # So it consumes memory 1 MB multiplied a number of workers. Thread.new(client, queue, nil) do |pubsub, q, prev_err| Thread.handle_interrupt(IO_ERROR_NEVER) do loop do Thread.handle_interrupt(IO_ERROR_IMMEDIATE) { q << pubsub.next_event } prev_err = nil rescue StandardError => e next sleep 0.005 if e.instance_of?(prev_err.class) && e.message == prev_err&.message Thread.handle_interrupt(IO_ERROR_IMMEDIATE) { q << e } prev_err = e end end rescue IOError # stream closed in another thread end end end BUF_SIZE = Integer(ENV.fetch('REDIS_CLIENT_PUBSUB_BUF_SIZE', 1024)) private_constant :BUF_SIZE def initialize(router, command_builder) @router = router @command_builder = command_builder @queue = SizedQueue.new(BUF_SIZE) @state_dict = {} @commands = [] end def call(*args, **kwargs) command = @command_builder.generate(args, kwargs) _call(command) @commands << command nil end def call_v(command) command = @command_builder.generate(command) _call(command) @commands << command nil end def close @state_dict.each_value(&:close) @state_dict.clear @commands.clear @queue.clear @queue.close nil end def next_event(timeout = nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity @state_dict.each_value(&:ensure_worker) max_duration = calc_max_duration(timeout) starting = obtain_current_time loop do break if max_duration > 0 && obtain_current_time - starting > max_duration case event = @queue.pop(true) when ::RedisClient::CommandError raise event unless event.message.start_with?('MOVED', 'CLUSTERDOWN') break start_over when ::RedisClient::ConnectionError then break start_over when StandardError then raise event when Array then break event end rescue ThreadError sleep 0.005 end end private def _call(command) # rubocop:disable Metrics/AbcSize if command.first.casecmp('subscribe').zero? call_to_single_state(command) elsif command.first.casecmp('psubscribe').zero? call_to_single_state(command) elsif command.first.casecmp('ssubscribe').zero? call_to_single_state(command) elsif command.first.casecmp('unsubscribe').zero? call_to_all_states(command) elsif command.first.casecmp('punsubscribe').zero? call_to_all_states(command) elsif command.first.casecmp('sunsubscribe').zero? call_for_sharded_states(command) else call_to_single_state(command) end end def call_to_single_state(command) node_key = @router.find_node_key(command) handle_connection_error(node_key) do @state_dict[node_key] ||= State.new(@router.find_node(node_key).pubsub, @queue) @state_dict[node_key].call(command) end end def call_to_all_states(command) @state_dict.each do |node_key, state| handle_connection_error(node_key, ignore: true) do state.call(command) end end end def call_for_sharded_states(command) if command.size == 1 call_to_all_states(command) else call_to_single_state(command) end end def obtain_current_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) end def calc_max_duration(timeout) timeout.nil? || timeout < 0 ? 0 : timeout * 1_000_000 end def handle_connection_error(node_key, ignore: false) yield rescue ::RedisClient::ConnectionError @state_dict[node_key]&.close @state_dict.delete(node_key) @router.renew_cluster_state raise unless ignore end def start_over loop do @router.renew_cluster_state @state_dict.each_value(&:close) @state_dict.clear @commands.each { |command| _call(command) } break rescue ::RedisClient::ConnectionError, ::RedisClient::Cluster::NodeMightBeDown sleep 1.0 end end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/router.rb000066400000000000000000000513611517214044400266060ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' require 'redis_client/circuit_breaker' require 'redis_client/cluster/command' require 'redis_client/cluster/errors' require 'redis_client/cluster/key_slot_converter' require 'redis_client/cluster/node' require 'redis_client/cluster/node_key' require 'redis_client/cluster/transaction' require 'redis_client/cluster/optimistic_locking' require 'redis_client/cluster/pipeline' require 'redis_client/cluster/error_identification' class RedisClient class Cluster class Router ZERO_CURSOR_FOR_SCAN = '0' TSF = ->(f, x) { f.nil? ? x : f.call(x) }.curry DEDICATED_ACTIONS = lambda do # rubocop:disable Metrics/BlockLength action = Struct.new('RedisCommandRoutingAction', :method_name, :reply_transformer, keyword_init: true) pick_first = ->(reply) { reply.first } # rubocop:disable Style/SymbolProc flatten_strings = ->(reply) { reply.flatten.sort_by(&:to_s) } sum_num = ->(reply) { reply.select { |e| e.is_a?(Integer) }.sum } sort_numbers = ->(reply) { reply.sort_by(&:to_i) } if Object.const_defined?(:Ractor, false) && Ractor.respond_to?(:make_shareable) Ractor.make_shareable(pick_first) Ractor.make_shareable(flatten_strings) Ractor.make_shareable(sum_num) Ractor.make_shareable(sort_numbers) end multiple_key_action = action.new(method_name: :send_multiple_keys_command) all_node_first_action = action.new(method_name: :send_command_to_all_nodes, reply_transformer: pick_first) primary_first_action = action.new(method_name: :send_command_to_primaries, reply_transformer: pick_first) not_supported_action = action.new(method_name: :fail_not_supported_command) keyless_action = action.new(method_name: :fail_keyless_command) { 'ping' => action.new(method_name: :send_ping_command, reply_transformer: pick_first), 'wait' => action.new(method_name: :send_wait_command), 'keys' => action.new(method_name: :send_command_to_replicas, reply_transformer: flatten_strings), 'dbsize' => action.new(method_name: :send_command_to_replicas, reply_transformer: sum_num), 'scan' => action.new(method_name: :send_scan_command), 'lastsave' => action.new(method_name: :send_command_to_all_nodes, reply_transformer: sort_numbers), 'role' => action.new(method_name: :send_command_to_all_nodes), 'config' => action.new(method_name: :send_config_command), 'client' => action.new(method_name: :send_client_command), 'cluster' => action.new(method_name: :send_cluster_command), 'memory' => action.new(method_name: :send_memory_command), 'script' => action.new(method_name: :send_script_command), 'pubsub' => action.new(method_name: :send_pubsub_command), 'watch' => action.new(method_name: :send_watch_command), 'mget' => multiple_key_action, 'mset' => multiple_key_action, 'del' => multiple_key_action, 'acl' => all_node_first_action, 'auth' => all_node_first_action, 'bgrewriteaof' => all_node_first_action, 'bgsave' => all_node_first_action, 'quit' => all_node_first_action, 'save' => all_node_first_action, 'select' => all_node_first_action, 'flushall' => primary_first_action, 'flushdb' => primary_first_action, 'readonly' => not_supported_action, 'readwrite' => not_supported_action, 'shutdown' => not_supported_action, 'discard' => keyless_action, 'exec' => keyless_action, 'multi' => keyless_action, 'unwatch' => keyless_action }.each_with_object({}) do |(k, v), acc| acc[k] = v.freeze acc[k.upcase] = v.freeze end end.call.freeze private_constant :ZERO_CURSOR_FOR_SCAN, :TSF, :DEDICATED_ACTIONS attr_reader :config def initialize(config, concurrent_worker, pool: nil, **kwargs) @config = config @concurrent_worker = concurrent_worker @pool = pool @client_kwargs = kwargs @node = ::RedisClient::Cluster::Node.new(concurrent_worker, config: config, pool: pool, **kwargs) @node.try_reload! @command = ::RedisClient::Cluster::Command.load(@node.replica_clients.shuffle, slow_command_timeout: config.slow_command_timeout) @command_builder = @config.command_builder rescue ::RedisClient::Cluster::InitialSetupError => e e.with_config(config) raise end def send_command(method, command, *args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity action = DEDICATED_ACTIONS[command.first] return assign_node_and_send_command(method, command, args, &block) if action.nil? return send(action.method_name, method, command, args, &block) if action.reply_transformer.nil? reply = send(action.method_name, method, command, args) action.reply_transformer.call(reply).then(&TSF.call(block)) rescue ::RedisClient::CircuitBreaker::OpenCircuitError raise rescue ::RedisClient::Cluster::Node::ReloadNeeded renew_cluster_state raise ::RedisClient::Cluster::NodeMightBeDown.new.with_config(@config) rescue ::RedisClient::ConnectionError renew_cluster_state raise rescue ::RedisClient::CommandError => e renew_cluster_state if e.message.start_with?('CLUSTERDOWN') raise rescue ::RedisClient::Cluster::ErrorCollection => e e.with_config(@config) raise if e.errors.any?(::RedisClient::CircuitBreaker::OpenCircuitError) renew_cluster_state if e.errors.values.any? do |err| next false if ::RedisClient::Cluster::ErrorIdentification.identifiable?(err) && @node.none? { |c| ::RedisClient::Cluster::ErrorIdentification.client_owns_error?(err, c) } err.message.start_with?('CLUSTERDOWN') || err.is_a?(::RedisClient::ConnectionError) end raise end # @see https://redis.io/docs/reference/cluster-spec/#redirection-and-resharding Redirection and resharding def assign_node_and_send_command(method, command, args, retry_count: 3, &block) node = assign_node(command) send_command_to_node(node, method, command, args, retry_count: retry_count, &block) end def send_command_to_node(node, method, command, args, retry_count: 3, &block) handle_redirection(node, command, retry_count: retry_count) do |on_node| if args.empty? # prevent memory allocation for variable-length args on_node.public_send(method, command, &block) else on_node.public_send(method, *args, command, &block) end end end def handle_redirection(node, command, retry_count:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity yield node rescue ::RedisClient::CircuitBreaker::OpenCircuitError raise rescue ::RedisClient::CommandError => e raise unless ::RedisClient::Cluster::ErrorIdentification.client_owns_error?(e, node) retry_count -= 1 if e.message.start_with?('MOVED') node = assign_redirection_node(e.message) retry if retry_count >= 0 elsif e.message.start_with?('ASK') node = assign_asking_node(e.message) if retry_count >= 0 node.call('asking') retry end elsif e.message.start_with?('CLUSTERDOWN') renew_cluster_state retry if retry_count >= 0 end raise rescue ::RedisClient::ConnectionError => e raise unless ::RedisClient::Cluster::ErrorIdentification.client_owns_error?(e, node) renew_cluster_state retry_count -= 1 if retry_count >= 0 # Find the node to use for this command - if this fails for some reason, though, re-use # the old node. begin node = find_node(find_node_key(command)) if command rescue StandardError # rubocop:disable Lint/SuppressedException end retry end raise end def scan(command, seed: nil) # rubocop:disable Metrics/AbcSize command[1] = ZERO_CURSOR_FOR_SCAN if command.size == 1 input_cursor = Integer(command[1]) client_index = input_cursor % 256 raw_cursor = input_cursor >> 8 clients = @node.clients_for_scanning(seed: seed) client = clients[client_index] return [ZERO_CURSOR_FOR_SCAN, []] unless client command[1] = raw_cursor.to_s result_cursor, result_keys = client.call_v(command) result_cursor = Integer(result_cursor) client_index += 1 if result_cursor == 0 [((result_cursor << 8) + client_index).to_s, result_keys] rescue ::RedisClient::ConnectionError renew_cluster_state raise end def scan_single_key(command, arity:, &block) node = assign_node(command) loop do cursor, values = handle_redirection(node, nil, retry_count: 3) { |n| n.call_v(command) } command[2] = cursor arity < 2 ? values.each(&block) : values.each_slice(arity, &block) break if cursor == ZERO_CURSOR_FOR_SCAN end end def assign_node(command) handle_node_reload_error do node_key = find_node_key(command) @node.find_by(node_key) end end def find_node_key_by_key(key, seed: nil, primary: false) if key && !key.empty? slot = ::RedisClient::Cluster::KeySlotConverter.convert(key) node_key = primary ? @node.find_node_key_of_primary(slot) : @node.find_node_key_of_replica(slot) if node_key.nil? renew_cluster_state raise ::RedisClient::Cluster::NodeMightBeDown.new.with_config(@config) end node_key else primary ? @node.any_primary_node_key(seed: seed) : @node.any_replica_node_key(seed: seed) end end def find_primary_node_by_slot(slot) handle_node_reload_error do node_key = @node.find_node_key_of_primary(slot) @node.find_by(node_key) end end def find_node_key(command, seed: nil) key = @command.extract_first_key(command) find_node_key_by_key(key, seed: seed, primary: @command.should_send_to_primary?(command)) end def find_primary_node_key(command) key = @command.extract_first_key(command) return nil unless key&.size&.> 0 find_node_key_by_key(key, primary: true) end def find_slot(command) find_slot_by_key(@command.extract_first_key(command)) end def find_slot_by_key(key) return if key.empty? ::RedisClient::Cluster::KeySlotConverter.convert(key) end def find_node(node_key) handle_node_reload_error { @node.find_by(node_key) } end def command_exists?(name) @command.exists?(name) end def assign_redirection_node(err_msg) _, slot, node_key = err_msg.split slot = slot.to_i @node.update_slot(slot, node_key) handle_node_reload_error { @node.find_by(node_key) } end def assign_asking_node(err_msg) _, _, node_key = err_msg.split handle_node_reload_error { @node.find_by(node_key) } end def node_keys @node.node_keys end def renew_cluster_state @node.try_reload! rescue ::RedisClient::Cluster::InitialSetupError # ignore end def close @node.each(&:close) end private def send_command_to_all_nodes(method, command, args, &block) @node.call_all(method, command, args, &block) end def send_command_to_primaries(method, command, args, &block) @node.call_primaries(method, command, args, &block) end def send_command_to_replicas(method, command, args, &block) @node.call_replicas(method, command, args, &block) end def send_ping_command(method, command, args, &block) @node.send_ping(method, command, args, &block) end def send_scan_command(_method, command, _args, &_block) scan(command, seed: 1) end def fail_not_supported_command(_method, command, _args, &_block) raise ::RedisClient::Cluster::OrchestrationCommandNotSupported.from_command(command.first).with_config(@config) end def fail_keyless_command(_method, command, _args, &_block) raise ::RedisClient::Cluster::AmbiguousNodeError.from_command(command.first).with_config(@config) end def send_wait_command(method, command, args, retry_count: 1, &block) # rubocop:disable Metrics/AbcSize @node.call_primaries(method, command, args).select { |r| r.is_a?(Integer) }.sum.then(&TSF.call(block)) rescue ::RedisClient::Cluster::ErrorCollection => e raise if e.errors.any?(::RedisClient::CircuitBreaker::OpenCircuitError) raise if retry_count <= 0 raise if e.errors.values.none? { |err| err.message.include?('WAIT cannot be used with replica instances') } renew_cluster_state retry_count -= 1 retry end def send_config_command(method, command, args, &block) # rubocop:disable Metrics/AbcSize if command[1].casecmp('resetstat').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('rewrite').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('set').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) else assign_node(command).public_send(method, *args, command, &block) end end def send_memory_command(method, command, args, &block) if command[1].casecmp('stats').zero? @node.call_all(method, command, args, &block) elsif command[1].casecmp('purge').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) else assign_node(command).public_send(method, *args, command, &block) end end def send_client_command(method, command, args, &block) # rubocop:disable Metrics/AbcSize if command[1].casecmp('list').zero? @node.call_all(method, command, args, &block).flatten elsif command[1].casecmp('pause').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('reply').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('setname').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) else assign_node(command).public_send(method, *args, command, &block) end end def send_cluster_command(method, command, args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity if command[1].casecmp('keyslot').zero? || command[1].casecmp('info').zero? || command[1].casecmp('nodes').zero? || command[1].casecmp('slots').zero? || command[1].casecmp('shards').zero? || command[1].casecmp('count-failure-reports').zero? || command[1].casecmp('slaves').zero? assign_node(command).public_send(method, *args, command, &block) elsif command[1].casecmp('saveconfig').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('countkeysinslot').zero? || command[1].casecmp('getkeysinslot').zero? handle_node_reload_error do node_key = @node.find_node_key_of_replica(command[2]) || @node.any_replica_node_key @node.find_by(node_key).public_send(method, *args, command, &block) end else fail_not_supported_command(method, command, args, &block) end end def send_script_command(method, command, args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity if command[1].casecmp('debug').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('kill').zero? @node.call_all(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('flush').zero? @node.call_primaries(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('load').zero? @node.call_primaries(method, command, args).first.then(&TSF.call(block)) elsif command[1].casecmp('exists').zero? @node.call_all(method, command, args).transpose.map { |arr| arr.any?(&:zero?) ? 0 : 1 }.then(&TSF.call(block)) else assign_node(command).public_send(method, *args, command, &block) end end def send_pubsub_command(method, command, args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity if command[1].casecmp('channels').zero? @node.call_all(method, command, args).flatten.uniq.sort_by(&:to_s).then(&TSF.call(block)) elsif command[1].casecmp('shardchannels').zero? @node.call_replicas(method, command, args).flatten.uniq.sort_by(&:to_s).then(&TSF.call(block)) elsif command[1].casecmp('numpat').zero? @node.call_all(method, command, args).select { |e| e.is_a?(Integer) }.sum.then(&TSF.call(block)) elsif command[1].casecmp('numsub').zero? @node.call_all(method, command, args).reject(&:empty?).map { |e| Hash[*e] } .reduce({}) { |a, e| a.merge(e) { |_, v1, v2| v1 + v2 } }.then(&TSF.call(block)) elsif command[1].casecmp('shardnumsub').zero? @node.call_replicas(method, command, args).reject(&:empty?).map { |e| Hash[*e] } .reduce({}) { |a, e| a.merge(e) { |_, v1, v2| v1 + v2 } }.then(&TSF.call(block)) else assign_node(command).public_send(method, *args, command, &block) end end def send_watch_command(_method, command, _args, &_block) unless block_given? msg = 'A block required. And you need to use the block argument as a client for the transaction.' raise ::RedisClient::Cluster::Transaction::ConsistencyError.new(msg).with_config(@config) end ::RedisClient::Cluster::OptimisticLocking.new(self).watch(command[1..]) do |c, slot, asking| transaction = ::RedisClient::Cluster::Transaction.new( self, @command_builder, node: c, slot: slot, asking: asking ) yield transaction transaction.execute end end def send_multiple_keys_command(method, command, args, &block) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity # This implementation prioritizes performance over readability. cmd = command.first if cmd.casecmp('mget').zero? single_key_cmd = 'get' keys_step = 1 elsif cmd.casecmp('mset').zero? single_key_cmd = 'set' keys_step = 2 elsif cmd.casecmp('del').zero? single_key_cmd = 'del' keys_step = 1 else raise NotImplementedError, cmd end return assign_node_and_send_command(method, command, args, &block) if command.size <= keys_step + 1 || ::RedisClient::Cluster::KeySlotConverter.hash_tag_included?(command[1]) seed = @config.use_replica? && @config.replica_affinity == :random ? nil : Random.new_seed pipeline = ::RedisClient::Cluster::Pipeline.new(self, @command_builder, @concurrent_worker, exception: true, seed: seed) single_command = Array.new(keys_step + 1) single_command[0] = single_key_cmd if keys_step == 1 command[1..].each do |key| single_command[1] = key pipeline.call_v(single_command) end else command[1..].each_slice(keys_step) do |v| keys_step.times { |i| single_command[i + 1] = v[i] } pipeline.call_v(single_command) end end replies = pipeline.execute result = if cmd.casecmp('mset').zero? replies.first elsif cmd.casecmp('del').zero? replies.sum else replies end block_given? ? yield(result) : result end def handle_node_reload_error(retry_count: 1) yield rescue ::RedisClient::Cluster::Node::ReloadNeeded raise ::RedisClient::Cluster::NodeMightBeDown.new.with_config(@config) if retry_count <= 0 renew_cluster_state retry_count -= 1 retry end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster/transaction.rb000066400000000000000000000137451517214044400276170ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' require 'redis_client/cluster/errors' require 'redis_client/cluster/noop_command_builder' require 'redis_client/cluster/pipeline' class RedisClient class Cluster class Transaction ConsistencyError = Class.new(::RedisClient::Cluster::Error) MAX_REDIRECTION = 2 EMPTY_ARRAY = [].freeze private_constant :MAX_REDIRECTION, :EMPTY_ARRAY def initialize(router, command_builder, node: nil, slot: nil, asking: false) @router = router @command_builder = command_builder @retryable = true @pipeline = ::RedisClient::Pipeline.new(::RedisClient::Cluster::NoopCommandBuilder) @pending_commands = [] @node = node prepare_tx unless @node.nil? @watching_slot = slot @asking = asking end def call(*command, **kwargs, &block) command = @command_builder.generate(command, kwargs) if prepare(command) @pipeline.call_v(command, &block) else defer { @pipeline.call_v(command, &block) } end end def call_v(command, &block) command = @command_builder.generate(command) if prepare(command) @pipeline.call_v(command, &block) else defer { @pipeline.call_v(command, &block) } end end def call_once(*command, **kwargs, &block) @retryable = false command = @command_builder.generate(command, kwargs) if prepare(command) @pipeline.call_once_v(command, &block) else defer { @pipeline.call_once_v(command, &block) } end end def call_once_v(command, &block) @retryable = false command = @command_builder.generate(command) if prepare(command) @pipeline.call_once_v(command, &block) else defer { @pipeline.call_once_v(command, &block) } end end def execute @pending_commands.each(&:call) return EMPTY_ARRAY if @pipeline._empty? raise ConsistencyError.new("couldn't determine the node: #{@pipeline._commands}").with_config(@router.config) if @node.nil? commit end private def defer(&block) @pending_commands << block nil end def prepare(command) return true unless @node.nil? node_key = @router.find_primary_node_key(command) return false if node_key.nil? @node = @router.find_node(node_key) prepare_tx true end def prepare_tx @pipeline.call('multi') @pending_commands.each(&:call) @pending_commands.clear end def commit @pipeline.call('exec') settle end def cancel @pipeline.call('discard') settle end def settle # If we needed ASKING on the watch, we need ASKING on the multi as well. @node.call('asking') if @asking # Don't handle redirections at this level if we're in a watch (the watcher handles redirections # at the whole-transaction level.) send_transaction(@node, redirect: !!@watching_slot ? 0 : MAX_REDIRECTION) end def send_transaction(client, redirect:) case client when ::RedisClient then send_pipeline(client, redirect: redirect) when ::RedisClient::Pooled then client.with { |c| send_pipeline(c, redirect: redirect) } else raise NotImplementedError, "#{client.class.name}#multi for cluster client" end end def send_pipeline(client, redirect:) # rubocop:disable Metrics/AbcSize replies = client.ensure_connected_cluster_scoped(retryable: @retryable) do |connection| commands = @pipeline._commands client.middlewares.call_pipelined(commands, client.config) do connection.call_pipelined(commands, nil) rescue ::RedisClient::CommandError => e ensure_the_same_slot!(commands) return handle_command_error!(e, redirect: redirect) unless redirect.zero? raise end end return if replies.last.nil? coerce_results!(replies.last) rescue ::RedisClient::ConnectionError @router.renew_cluster_state if @watching_slot.nil? raise end def coerce_results!(results, offset: 1) results.each_with_index do |result, index| if result.is_a?(::RedisClient::CommandError) result._set_command(@pipeline._commands[index + offset]) raise result end next if @pipeline._blocks.nil? block = @pipeline._blocks[index + offset] next if block.nil? results[index] = block.call(result) end results end def handle_command_error!(err, redirect:) # rubocop:disable Metrics/AbcSize if err.message.start_with?('CROSSSLOT') raise ConsistencyError.new("#{err.message}: #{err.command}").with_config(@router.config) elsif err.message.start_with?('MOVED') node = @router.assign_redirection_node(err.message) send_transaction(node, redirect: redirect - 1) elsif err.message.start_with?('ASK') node = @router.assign_asking_node(err.message) try_asking(node) ? send_transaction(node, redirect: redirect - 1) : err elsif err.message.start_with?('CLUSTERDOWN') @router.renew_cluster_state if @watching_slot.nil? raise err else raise err end end def ensure_the_same_slot!(commands) slots = commands.map { |command| @router.find_slot(command) }.compact.uniq return if slots.size == 1 && @watching_slot.nil? return if slots.size == 1 && @watching_slot == slots.first raise ConsistencyError.new("the transaction should be executed to a slot in a node: #{commands}").with_config(@router.config) end def try_asking(node) node.call('asking') == 'OK' rescue StandardError false end end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_client/cluster_config.rb000066400000000000000000000161151517214044400266110ustar00rootroot00000000000000# frozen_string_literal: true require 'uri' require 'redis_client' require 'redis_client/cluster' require 'redis_client/cluster/errors' require 'redis_client/cluster/node_key' require 'redis_client/cluster/noop_command_builder' require 'redis_client/command_builder' class RedisClient class ClusterConfig DEFAULT_HOST = '127.0.0.1' DEFAULT_PORT = 6379 DEFAULT_SCHEME = 'redis' SECURE_SCHEME = 'rediss' DEFAULT_NODE = "#{DEFAULT_SCHEME}://#{DEFAULT_HOST}:#{DEFAULT_PORT}" DEFAULT_NODES = [DEFAULT_NODE].freeze VALID_SCHEMES = [DEFAULT_SCHEME, SECURE_SCHEME].freeze VALID_NODES_KEYS = %i[ssl username password host port db].freeze MERGE_CONFIG_KEYS = %i[ssl username password db].freeze IGNORE_GENERIC_CONFIG_KEYS = %i[url host port path].freeze MAX_WORKERS = Integer(ENV.fetch('REDIS_CLIENT_MAX_THREADS', -1)) # for backward compatibility # Used for slow commands that fetch metadata, e.g. CLUSTER NODES, COMMAND. SLOW_COMMAND_TIMEOUT = Float(ENV.fetch('REDIS_CLIENT_SLOW_COMMAND_TIMEOUT', -1)) # Controls the balance between startup load and stability during initialization or cluster state changes. MAX_STARTUP_SAMPLE = Integer(ENV.fetch('REDIS_CLIENT_MAX_STARTUP_SAMPLE', 3)) private_constant :DEFAULT_HOST, :DEFAULT_PORT, :DEFAULT_SCHEME, :SECURE_SCHEME, :DEFAULT_NODES, :VALID_SCHEMES, :VALID_NODES_KEYS, :MERGE_CONFIG_KEYS, :IGNORE_GENERIC_CONFIG_KEYS, :MAX_WORKERS, :SLOW_COMMAND_TIMEOUT, :MAX_STARTUP_SAMPLE InvalidClientConfigError = Class.new(::RedisClient::Cluster::Error) attr_reader :command_builder, :client_config, :replica_affinity, :slow_command_timeout, :connect_with_original_config, :startup_nodes, :max_startup_sample, :id def initialize( # rubocop:disable Metrics/ParameterLists nodes: DEFAULT_NODES, replica: false, replica_affinity: :random, fixed_hostname: '', concurrency: nil, connect_with_original_config: false, client_implementation: ::RedisClient::Cluster, # for redis gem slow_command_timeout: SLOW_COMMAND_TIMEOUT, command_builder: ::RedisClient::CommandBuilder, max_startup_sample: MAX_STARTUP_SAMPLE, **client_config ) @replica = true & replica @replica_affinity = replica_affinity.to_s.to_sym @fixed_hostname = fixed_hostname.to_s @command_builder = command_builder node_configs = build_node_configs(nodes.dup) @client_config = merge_generic_config(client_config, node_configs) # Keep tabs on the original startup nodes we were constructed with @startup_nodes = build_startup_nodes(node_configs) @concurrency = merge_concurrency_option(concurrency) @connect_with_original_config = connect_with_original_config @client_implementation = client_implementation @slow_command_timeout = slow_command_timeout @max_startup_sample = max_startup_sample @id = client_config[:id] end def inspect "#<#{self.class.name} #{startup_nodes.values.map { |v| v.reject { |k| k == :command_builder } }}>" end def connect_timeout @client_config[:connect_timeout] || @client_config[:timeout] || ::RedisClient::Config::DEFAULT_TIMEOUT end def read_timeout @client_config[:read_timeout] || @client_config[:timeout] || ::RedisClient::Config::DEFAULT_TIMEOUT end def write_timeout @client_config[:write_timeout] || @client_config[:timeout] || ::RedisClient::Config::DEFAULT_TIMEOUT end def new_pool(size: 5, timeout: 5, **kwargs) @client_implementation.new( self, pool: { size: size, timeout: timeout }, concurrency: @concurrency, **kwargs ) end def new_client(**kwargs) @client_implementation.new(self, concurrency: @concurrency, **kwargs) end def use_replica? @replica end def client_config_for_node(node_key) config = ::RedisClient::Cluster::NodeKey.hashify(node_key) config[:port] = ensure_integer(config[:port]) augment_client_config(config) end def resolved? true end def sentinel? false end def server_url nil end private def merge_concurrency_option(option) opts = {} if MAX_WORKERS.positive? opts[:model] = :on_demand opts[:size] = MAX_WORKERS end opts.merge!(option.transform_keys(&:to_sym)) if option.is_a?(Hash) opts[:model] = :none if opts.empty? opts.freeze end def build_node_configs(addrs) configs = Array[addrs].flatten.filter_map { |addr| parse_node_addr(addr) } raise InvalidClientConfigError, '`nodes` option is empty' if configs.empty? configs end def parse_node_addr(addr) case addr when String parse_node_url(addr) when Hash parse_node_option(addr) else raise InvalidClientConfigError, "`nodes` option includes invalid type values: #{addr}" end end def parse_node_url(addr) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity return if addr.empty? uri = URI(addr) scheme = uri.scheme || DEFAULT_SCHEME raise InvalidClientConfigError, "`nodes` option includes a invalid uri scheme: #{addr}" unless VALID_SCHEMES.include?(scheme) username = uri.user ? URI.decode_www_form_component(uri.user) : nil password = uri.password ? URI.decode_www_form_component(uri.password) : nil host = uri.host || DEFAULT_HOST port = uri.port || DEFAULT_PORT db = uri.path.index('/').nil? ? uri.path : uri.path.split('/')[1] db = db.nil? || db.empty? ? db : ensure_integer(db) { ssl: scheme == SECURE_SCHEME, username: username, password: password, host: host, port: port, db: db } .reject { |_, v| v.nil? || v == '' || v == false } rescue URI::InvalidURIError => e raise InvalidClientConfigError, "#{e.message}: #{addr}" end def parse_node_option(addr) return if addr.empty? addr = addr.transform_keys(&:to_sym) addr[:host] ||= DEFAULT_HOST addr[:port] = ensure_integer(addr[:port] || DEFAULT_PORT) addr.select { |k, _| VALID_NODES_KEYS.include?(k) } end def ensure_integer(value) Integer(value) rescue ArgumentError => e raise InvalidClientConfigError, e.message end def merge_generic_config(client_config, node_configs) cfg = node_configs.first || {} client_config.reject { |k, _| IGNORE_GENERIC_CONFIG_KEYS.include?(k) } .merge(cfg.slice(*MERGE_CONFIG_KEYS)) end def build_startup_nodes(configs) configs.to_h do |config| node_key = ::RedisClient::Cluster::NodeKey.build_from_host_port(config[:host], config[:port]) config = augment_client_config(config) [node_key, config] end end def augment_client_config(config) config = @client_config.merge(config) config = config.merge(host: @fixed_hostname) unless @fixed_hostname.empty? config[:command_builder] = ::RedisClient::Cluster::NoopCommandBuilder # prevent twice call config end end end redis-rb-redis-cluster-client-aaf9e2e/lib/redis_cluster_client.rb000066400000000000000000000002661517214044400253440ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client/cluster_config' class RedisClient class << self def cluster(**kwargs) ClusterConfig.new(**kwargs) end end end redis-rb-redis-cluster-client-aaf9e2e/redis-cluster-client.gemspec000066400000000000000000000014241517214044400254470ustar00rootroot00000000000000# frozen_string_literal: true Gem::Specification.new do |s| s.name = 'redis-cluster-client' s.summary = 'Redis cluster-aware client for Ruby' s.version = '0.16.1' s.license = 'MIT' s.homepage = 'https://github.com/redis-rb/redis-cluster-client' s.authors = ['Taishi Kasuga'] s.email = %w[proxy0721@gmail.com] s.required_ruby_version = '>= 2.7.0' s.metadata['rubygems_mfa_required'] = 'true' s.metadata['allowed_push_host'] = 'https://rubygems.org' s.files = Dir['lib/**/*.rb'] s.add_runtime_dependency 'redis-client', '~> 0.28' end redis-rb-redis-cluster-client-aaf9e2e/test/000077500000000000000000000000001517214044400210175ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/cluster_controller.rb000066400000000000000000000434461517214044400253030ustar00rootroot00000000000000# frozen_string_literal: true require 'redis_client' class ClusterController SLOT_SIZE = 16_384 DEFAULT_SHARD_SIZE = 3 DEFAULT_REPLICA_SIZE = 1 DEFAULT_MAX_ATTEMPTS = 300 DEFAULT_TIMEOUT_SEC = 5.0 SLEEP_SEC = 1.0 private_constant :SLOT_SIZE, :DEFAULT_SHARD_SIZE, :DEFAULT_REPLICA_SIZE, :DEFAULT_MAX_ATTEMPTS, :DEFAULT_TIMEOUT_SEC MaxRetryExceeded = Class.new(StandardError) RedisNodeInfo = Struct.new( 'RedisClusterNodeInfo', :id, :node_key, :flags, :role, :myself?, :primary_id, :ping_sent, :pong_recv, :config_epoch, :link_state, :slots, :client, :client_node_key, keyword_init: true ) do def primary? role == 'master' end def replica? role == 'slave' end def empty_slots? slots.nil? || slots.empty? end def include_slot?(slot) slots&.include?(slot) || false end def slot_size slots&.size.to_i end end def initialize(node_addrs, shard_size: DEFAULT_SHARD_SIZE, replica_size: DEFAULT_REPLICA_SIZE, state_check_attempts: DEFAULT_MAX_ATTEMPTS, **kwargs) @shard_size = shard_size @replica_size = replica_size @number_of_replicas = @replica_size * @shard_size @max_attempts = state_check_attempts @timeout = kwargs.fetch(:timeout, DEFAULT_TIMEOUT_SEC) @kwargs = kwargs.merge(timeout: @timeout) @clients = node_addrs.map { |addr| ::RedisClient.new(url: addr, **@kwargs) } @debug = ENV.fetch('DEBUG', '0') end attr_reader :clients def wait_for_cluster_to_be_ready(skip_clients: []) print_debug('wait for nodes to be recognized...') wait_meeting(@clients, max_attempts: @max_attempts) print_debug('wait for the cluster state to be ok...') wait_cluster_building(@clients, max_attempts: @max_attempts) print_debug('wait for the replication to be established...') wait_replication(@clients, number_of_replicas: @number_of_replicas, max_attempts: @max_attempts) print_debug('wait for commands to be accepted...') wait_cluster_recovering(@clients, max_attempts: @max_attempts, skip_clients: skip_clients) end def rebuild flush_all_data(@clients) reset_cluster(@clients) assign_slots(@clients, shard_size: @shard_size) save_config_epoch(@clients) meet_each_other(@clients) wait_meeting(@clients, max_attempts: @max_attempts) replicate(@clients, shard_size: @shard_size, replica_size: @replica_size) save_config(@clients) wait_cluster_building(@clients, max_attempts: @max_attempts) wait_replication(@clients, number_of_replicas: @number_of_replicas, max_attempts: @max_attempts) wait_cluster_recovering(@clients, max_attempts: @max_attempts) end def down flush_all_data(@clients) reset_cluster(@clients) end def failover rows = associate_with_clients_and_nodes(@clients) primary_info = rows.find(&:primary?) replica_info = rows.find { |row| row.primary_id == primary_info.id } wait_replication_delay(@clients, replica_size: @replica_size, timeout: @timeout) replica_info.client.call_once('CLUSTER', 'FAILOVER', 'TAKEOVER') wait_failover( @clients, primary_node_key: primary_info.node_key, replica_node_key: replica_info.node_key, max_attempts: @max_attempts ) wait_replication_delay(@clients, replica_size: @replica_size, timeout: @timeout) wait_cluster_recovering(@clients, max_attempts: @max_attempts) end def start_resharding(slot:, src_node_key:, dest_node_key:) rows = associate_with_clients_and_nodes(@clients) src_info = rows.find { |r| r.node_key == src_node_key || r.client_node_key == src_node_key } dest_info = rows.find { |r| r.node_key == dest_node_key || r.client_node_key == dest_node_key } src_node_id = src_info.id src_client = src_info.client dest_node_id = dest_info.id dest_client = dest_info.client dest_host, dest_port = dest_info.node_key.split(':') # @see https://redis.io/commands/cluster-setslot/#redis-cluster-live-resharding-explained dest_client.call_once('CLUSTER', 'SETSLOT', slot, 'IMPORTING', src_node_id) src_client.call_once('CLUSTER', 'SETSLOT', slot, 'MIGRATING', dest_node_id) db_idx = '0' timeout_msec = @timeout.to_i * 1000 number_of_keys = src_client.call_once('CLUSTER', 'COUNTKEYSINSLOT', slot) keys = src_client.call_once('CLUSTER', 'GETKEYSINSLOT', slot, number_of_keys) print_debug("#{src_client.config.host}:#{src_client.config.port} => #{dest_client.config.host}:#{dest_client.config.port} ... #{keys}") return if keys.empty? begin src_client.call_once('MIGRATE', dest_host, dest_port, '', db_idx, timeout_msec, 'KEYS', *keys) rescue ::RedisClient::CommandError => e raise unless e.message.start_with?('IOERR') # retry once src_client.call_once('MIGRATE', dest_host, dest_port, '', db_idx, timeout_msec, 'REPLACE', 'KEYS', *keys) end wait_replication_delay(@clients, replica_size: @replica_size, timeout: @timeout) end def finish_resharding(slot:, src_node_key:, dest_node_key:) rows = associate_with_clients_and_nodes(@clients) src_info = rows.find { |r| r.node_key == src_node_key || r.client_node_key == src_node_key } dest_info = rows.find { |r| r.node_key == dest_node_key || r.client_node_key == dest_node_key } src = src_info.client dest = dest_info.client id = dest_info.id rest = rows.reject { |r| r.replica? || r.client.equal?(src) || r.client.equal?(dest) }.map(&:client) ([dest, src] + rest).each do |cli| cli.call_once('CLUSTER', 'SETSLOT', slot, 'NODE', id) print_debug("#{cli.config.host}:#{cli.config.port} ... CLUSTER SETSLOT #{slot} NODE #{id}") rescue ::RedisClient::CommandError => e raise unless e.message.start_with?('ERR Please use SETSLOT only with masters.') # how weird, ignore end wait_replication_delay(@clients, replica_size: @replica_size, timeout: @timeout) end def scale_out(primary_url:, replica_url:) # @see https://redis.io/docs/manual/scaling/ rows = associate_with_clients_and_nodes(@clients) target_host, target_port = rows.find(&:primary?)&.node_key&.split(':') primary = ::RedisClient.new(url: primary_url, **@kwargs) replica = ::RedisClient.new(url: replica_url, **@kwargs) @clients << primary @clients << replica @shard_size += 1 @number_of_replicas = @replica_size * @shard_size primary.call_once('CLUSTER', 'MEET', target_host, target_port) replica.call_once('CLUSTER', 'MEET', target_host, target_port) wait_meeting(@clients, max_attempts: @max_attempts) primary_id = primary.call_once('CLUSTER', 'MYID') replica.call_once('CLUSTER', 'REPLICATE', primary_id) save_config(@clients) wait_for_cluster_to_be_ready(skip_clients: [primary, replica]) rows = associate_with_clients_and_nodes(@clients) SLOT_SIZE.times.to_a.sample(100).sort.each do |slot| src = rows.find { |row| row.include_slot?(slot) }&.node_key dest = rows.find { |row| row.id == primary_id }&.node_key start_resharding(slot: slot, src_node_key: src, dest_node_key: dest) finish_resharding(slot: slot, src_node_key: src, dest_node_key: dest) end end def scale_in rows = associate_with_clients_and_nodes(@clients) primary_info = rows.reject(&:empty_slots?).min_by(&:slot_size) replica_info = rows.find { |r| r.primary_id == primary_info.id } rest_primary_node_keys = rows.reject { |r| r.id == primary_info.id || r.replica? }.map(&:node_key) primary_info.slots.each do |slot| src = primary_info.node_key dest = rest_primary_node_keys.sample print_debug("Resharding slot #{slot}: #{src} => #{dest}") start_resharding(slot: slot, src_node_key: src, dest_node_key: dest) finish_resharding(slot: slot, src_node_key: src, dest_node_key: dest) end replica = replica_info.client primary = primary_info.client threads = @clients.map do |cli| Thread.new(cli) do |c| c.pipelined do |pi| pi.call_once('CLUSTER', 'FORGET', replica_info.id) pi.call_once('CLUSTER', 'FORGET', primary_info.id) end rescue ::RedisClient::Error # ignore end end threads.each(&:join) replica.call_once('CLUSTER', 'RESET', 'SOFT') primary.call_once('CLUSTER', 'RESET', 'SOFT') @clients.reject! { |c| c.equal?(primary) || c.equal?(replica) } @shard_size -= 1 @number_of_replicas = @replica_size * @shard_size wait_for_cluster_to_be_ready wait_for_state(@clients, max_attempts: @max_attempts) do |client| fetch_cluster_nodes(client).size == @shard_size + @number_of_replicas rescue ::RedisClient::ConnectionError true end end def select_resharding_target(slot) rows = associate_with_clients_and_nodes(@clients) src = rows.find { |r| r.primary? && r.include_slot?(slot) } dest = rows.reject { |r| r.replica? || r.id == src.id }.sample [src.node_key, dest.node_key] end def select_sacrifice_of_primary rows = associate_with_clients_and_nodes(@clients) rows.select(&:primary?) .reject { |primary| rows.none? { |r| r.primary_id == primary.id } } .sample.client end def select_sacrifice_of_replica rows = associate_with_clients_and_nodes(@clients) rows.select(&:replica?).sample.client end def close @clients.each do |client| client.close rescue ::RedisClient::ConnectionError # ignore end end private def flush_all_data(clients) clients.each do |c| c.call_once('FLUSHALL') print_debug("#{c.config.host}:#{c.config.port} ... FLUSHALL") rescue ::RedisClient::CommandError, ::RedisClient::ReadOnlyError # READONLY You can't write against a read only replica. rescue ::RedisClient::ConnectionError => e print_debug("#{c.config.host}:#{c.config.port} ... FLUSHALL: #{e.class}: #{e.message}") end end def reset_cluster(clients) clients.each do |c| c.call_once('CLUSTER', 'RESET', 'HARD') print_debug("#{c.config.host}:#{c.config.port} ... CLUSTER RESET HARD") rescue ::RedisClient::ConnectionError => e print_debug("#{c.config.host}:#{c.config.port} ... CLUSTER RESET HARD: #{e.class}: #{e.message}") end end def assign_slots(clients, shard_size:) primaries = take_primaries(clients, shard_size: shard_size) slot_slice = SLOT_SIZE / primaries.size mod = SLOT_SIZE % primaries.size slot_sizes = Array.new(primaries.size, slot_slice) mod.downto(1) { |i| slot_sizes[i] += 1 } slot_idx = 0 primaries.zip(slot_sizes).each do |c, s| slot_range = slot_idx..slot_idx + s - 1 c.call_once('CLUSTER', 'ADDSLOTS', *slot_range.to_a) slot_idx += s print_debug("#{c.config.host}:#{c.config.port} ... CLUSTER ADDSLOTS #{slot_range.to_a}") end end def save_config_epoch(clients) clients.each_with_index do |c, i| c.call_once('CLUSTER', 'SET-CONFIG-EPOCH', i + 1) print_debug("#{c.config.host}:#{c.config.port} ... CLUSTER SET-CONFIG-EPOCH #{i + 1}") rescue ::RedisClient::CommandError # ERR Node config epoch is already non-zero nil end end def meet_each_other(clients) rows = fetch_cluster_nodes(clients.first) rows = parse_cluster_nodes(rows) target_host, target_port = rows.first.node_key.split(':') clients.drop(1).each do |c| c.call_once('CLUSTER', 'MEET', target_host, target_port) print_debug("#{c.config.host}:#{c.config.port} ... CLUSTER MEET #{target_host}:#{target_port}") end end def wait_meeting(clients, max_attempts:) wait_for_state(clients, max_attempts: max_attempts) do |client| info = hashify_cluster_info(client) print_debug("#{client.config.host}:#{client.config.port} ... #{info['cluster_known_nodes']}") info['cluster_known_nodes'].to_s == clients.size.to_s rescue ::RedisClient::ConnectionError true end end def replicate(clients, shard_size:, replica_size:) primaries = take_primaries(clients, shard_size: shard_size) replicas = take_replicas(clients, shard_size: shard_size) replicas.each_slice(replica_size).each_with_index do |subset, i| primary_id = primaries[i].call_once('CLUSTER', 'MYID') loop do begin subset.each do |replica| replica.call_once('CLUSTER', 'REPLICATE', primary_id) print_debug("#{replica.config.host}:#{replica.config.port} ... CLUSTER REPLICATE #{primaries[i].config.host}:#{primaries[i].config.port}") end rescue ::RedisClient::CommandError => e print_debug(e.message) # ERR Unknown node [node-id] sleep SLEEP_SEC primary_id = primaries[i].call_once('CLUSTER', 'MYID') next end break end end end def save_config(clients) clients.each do |c| c.call_once('CLUSTER', 'SAVECONFIG') print_debug("#{c.config.host}:#{c.config.port} ... CLUSTER SAVECONFIG") end end def wait_cluster_building(clients, max_attempts:) wait_for_state(clients, max_attempts: max_attempts) do |client| info = hashify_cluster_info(client) print_debug("#{client.config.host}:#{client.config.port} ... #{info['cluster_state']}") info['cluster_state'] == 'ok' rescue ::RedisClient::ConnectionError true end end def wait_replication(clients, number_of_replicas:, max_attempts:) wait_for_state(clients, max_attempts: max_attempts) do |client| rows = fetch_cluster_nodes(client) rows = parse_cluster_nodes(rows) print_debug("#{client.config.host}:#{client.config.port} ... #{rows.count(&:replica?)}") rows.count(&:replica?) == number_of_replicas rescue ::RedisClient::ConnectionError true end end def wait_failover(clients, primary_node_key:, replica_node_key:, max_attempts:) wait_for_state(clients, max_attempts: max_attempts) do |client| rows = fetch_cluster_nodes(client) rows = parse_cluster_nodes(rows) primary_info = rows.find { |r| r.node_key == primary_node_key || r.client_node_key == primary_node_key } replica_info = rows.find { |r| r.node_key == replica_node_key || r.client_node_key == replica_node_key } primary_info.replica? && replica_info.primary? rescue ::RedisClient::ConnectionError true end end def wait_replication_delay(clients, replica_size:, timeout:) timeout_msec = timeout.to_i * 1000 server_side_timeout = timeout_msec > 100 ? timeout_msec - 100 : 10 wait_for_state(clients, max_attempts: clients.size + 1) do |client| swap_timeout(client, timeout: 0.1) do |cli| cli.blocking_call(timeout, 'WAIT', replica_size, server_side_timeout) if primary_client?(cli) end true rescue ::RedisClient::ConnectionError true end end def wait_cluster_recovering(clients, max_attempts:, skip_clients: []) key = 0 wait_for_state(clients, max_attempts: max_attempts) do |client| print_debug("#{client.config.host}:#{client.config.port} ... GET #{key}") client.call_once('GET', key) if primary_client?(client) && !skip_clients.include?(client) true rescue ::RedisClient::CommandError => e if e.message.start_with?('CLUSTERDOWN') false elsif e.message.start_with?('MOVED') key += 1 false else true end rescue ::RedisClient::ConnectionError true end end def wait_for_state(clients, max_attempts:) attempt_count = 1 clients.each do |client| attempt_count.step(max_attempts) do |i| raise MaxRetryExceeded if i >= max_attempts attempt_count += 1 break if yield(client) sleep SLEEP_SEC end end end def hashify_cluster_info(client) client.call_once('CLUSTER', 'INFO').split("\r\n").to_h { |v| v.split(':') } end def fetch_cluster_nodes(client) client.call_once('CLUSTER', 'NODES').split("\n").map(&:split) end def associate_with_clients_and_nodes(clients) clients.filter_map do |client| rows = fetch_cluster_nodes(client) rows = parse_cluster_nodes(rows) row = rows.find(&:myself?) next if row.nil? row.client = client row.client_node_key = "#{client.config.host}:#{client.config.port}" row rescue ::RedisClient::ConnectionError next end end def parse_cluster_nodes(rows) rows.map do |row| flags = row[2].split(',') slots = if row[8].nil? [] else row[8..].filter_map { |str| str.start_with?('[') ? nil : str.split('-').map { |s| Integer(s) } } .map { |a| a.size == 1 ? a << a.first : a }.map(&:sort) .flat_map { |first, last| (first..last).to_a }.sort end RedisNodeInfo.new( id: row[0], node_key: row[1].split('@').first, flags: flags, role: (flags & %w[master slave]).first, myself?: flags.include?('myself'), primary_id: row[3], ping_sent: row[4], pong_recv: row[5], config_epoch: row[6], link_state: row[7], slots: slots ) end end def take_primaries(clients, shard_size:) clients.select { |cli| primary_client?(cli) }.take(shard_size) end def take_replicas(clients, shard_size:) replicas = clients.select { |cli| replica_client?(cli) } replicas.empty? ? clients[shard_size..] : replicas end def primary_client?(client) client.call_once('ROLE').first == 'master' end def replica_client?(client) client.call_once('ROLE').first == 'slave' end def print_debug(msg) return unless @debug == '1' p msg end def swap_timeout(client, timeout:) updater = lambda do |c, t| c.read_timeout = t c.config.instance_variable_set(:@read_timeout, t) end regular_timeout = client.read_timeout updater.call(client, timeout) result = yield client updater.call(client, regular_timeout) result end end redis-rb-redis-cluster-client-aaf9e2e/test/ips_concurrent_worker.rb000066400000000000000000000027031517214044400257740ustar00rootroot00000000000000# frozen_string_literal: true require 'benchmark/ips' require 'redis_cluster_client' module IpsConcurrentWorker TASK_SIZE = 40 WORKER_SIZE = 5 module_function def run on_demand = make_worker(:on_demand) pooled = make_worker(:pooled) none = make_worker(:none) [0.0, 0.001, 0.003].each do |duration| print_letter('concurrent worker', "sleep: #{duration}") bench(duration, ondemand: on_demand, pooled: pooled, none: none) end end def make_worker(model) ::RedisClient::Cluster::ConcurrentWorker.create(model: model, size: WORKER_SIZE) end def print_letter(title, subtitle) print "################################################################################\n" print "# #{title}: #{subtitle}\n" print "################################################################################\n" print "\n" end def bench(duration, **kwargs) Benchmark.ips do |x| x.time = 5 x.warmup = 1 kwargs.each do |key, worker| x.report("model: #{key}") do group = worker.new_group(size: TASK_SIZE) TASK_SIZE.times do |i| group.push(i, i) do |n| sleep duration 2**n end end sum = 0 group.each do |_, n| # rubocop:disable Style/HashEachMethods sum += n end group.close end end x.compare! end end end IpsConcurrentWorker.run redis-rb-redis-cluster-client-aaf9e2e/test/ips_hashtag_extraction.rb000066400000000000000000000007451517214044400261040ustar00rootroot00000000000000# frozen_string_literal: true require 'benchmark/ips' require 'redis_cluster_client' module HashtagExtraction module_function def run key = 'aaaa{aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}aaaaaaaa' Benchmark.ips do |x| x.time = 5 x.warmup = 1 x.report('Hashtag Extraction') do ::RedisClient::Cluster::KeySlotConverter.extract_hash_tag(key) end x.compare! end end end HashtagExtraction.run redis-rb-redis-cluster-client-aaf9e2e/test/ips_mget.rb000066400000000000000000000027741517214044400231650ustar00rootroot00000000000000# frozen_string_literal: true require 'benchmark/ips' require 'redis_cluster_client' require 'testing_constants' module IpsMget module_function ATTEMPTS = 40 def run cli = make_client prepare(cli) print_letter('mget') bench('mget', cli) end def make_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random, fixed_hostname: TEST_FIXED_HOSTNAME, concurrency: { model: :on_demand }, **TEST_GENERIC_OPTIONS ).new_client end def print_letter(title) print "################################################################################\n" print "# #{title}\n" print "################################################################################\n" print "\n" end def prepare(client) ATTEMPTS.times do |i| client.call('set', "{key}#{i}", "val#{i}") client.call('set', "key#{i}", "val#{i}") end end def bench(cmd, client) original = [cmd] + Array.new(ATTEMPTS) { |i| "{key}#{i}" } emulated = [cmd] + Array.new(ATTEMPTS) { |i| "key#{i}" } single_get = [cmd] Benchmark.ips do |x| x.time = 5 x.warmup = 1 x.report("#{cmd}: original") { client.call_v(original) } x.report("#{cmd}: emulated") { client.call_v(emulated) } x.report("#{cmd}: single_get") do ATTEMPTS.times do |i| single_get[1] = "key#{i}" client.call_v(single_get) end end x.compare! end end end IpsMget.run redis-rb-redis-cluster-client-aaf9e2e/test/ips_pipeline.rb000066400000000000000000000037061517214044400240320ustar00rootroot00000000000000# frozen_string_literal: true require 'benchmark/ips' require 'redis_cluster_client' require 'testing_constants' module IpsPipeline module_function ATTEMPTS = 100 def run on_demand = make_client(:on_demand) pooled = make_client(:pooled) none = make_client(:none) envoy = make_client_for_envoy cluster_proxy = make_client_for_cluster_proxy prepare(on_demand, pooled, none, envoy, cluster_proxy) print_letter('pipelined') bench( ondemand: on_demand, pooled: pooled, none: none, envoy: envoy, cproxy: cluster_proxy ) end def make_client(model) ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random, fixed_hostname: TEST_FIXED_HOSTNAME, concurrency: { model: model }, **TEST_GENERIC_OPTIONS ).new_client end def make_client_for_envoy ::RedisClient.config( **TEST_GENERIC_OPTIONS.merge(BENCH_ENVOY_OPTIONS) ).new_client end def make_client_for_cluster_proxy ::RedisClient.config( **TEST_GENERIC_OPTIONS.merge(BENCH_REDIS_CLUSTER_PROXY_OPTIONS) ).new_client end def print_letter(title) print "################################################################################\n" print "# #{title}\n" print "################################################################################\n" print "\n" end def prepare(*clients) clients.each do |client| client.pipelined do |pi| ATTEMPTS.times do |i| pi.call('set', "key#{i}", "val#{i}") end end end end def bench(**kwargs) Benchmark.ips do |x| x.time = 5 x.warmup = 1 kwargs.each do |key, client| x.report("pipelined: #{key}") do client.pipelined do |pi| ATTEMPTS.times do |i| pi.call('get', "key#{i}") end end end end x.compare! end end end IpsPipeline.run redis-rb-redis-cluster-client-aaf9e2e/test/ips_single.rb000066400000000000000000000043761517214044400235120ustar00rootroot00000000000000# frozen_string_literal: true require 'async/redis/cluster_client' require 'benchmark/ips' require 'redis_cluster_client' require 'testing_constants' module IpsSingle module_function ATTEMPTS = 10 def run cli = make_client envoy = make_client_for_envoy cluster_proxy = make_client_for_cluster_proxy prepare(cli, envoy, cluster_proxy) print_letter('single') bench( cli: cli, envoy: envoy, cproxy: cluster_proxy ) async_bench(make_async_client) end def make_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_client end def make_client_for_envoy ::RedisClient.config( **TEST_GENERIC_OPTIONS.merge(BENCH_ENVOY_OPTIONS) ).new_client end def make_client_for_cluster_proxy ::RedisClient.config( **TEST_GENERIC_OPTIONS.merge(BENCH_REDIS_CLUSTER_PROXY_OPTIONS) ).new_client end def make_async_client endpoints = TEST_NODE_URIS.map { |e| ::Async::Redis::Endpoint.parse(e) } ::Async::Redis::ClusterClient.new(endpoints) end def print_letter(title) print "################################################################################\n" print "# #{title}\n" print "################################################################################\n" print "\n" end def prepare(*clients) clients.each do |client| ATTEMPTS.times do |i| client.call('set', "key#{i}", "val#{i}") end end end def bench(**kwargs) Benchmark.ips do |x| x.time = 5 x.warmup = 1 kwargs.each do |key, client| x.report("single: #{key}") do ATTEMPTS.times do |i| client.call('get', "key#{i}") end end end x.compare! end end def async_bench(cluster) Benchmark.ips do |x| x.time = 5 x.warmup = 1 x.report('single: async') do Async do ATTEMPTS.times do |i| key = "key#{i}" slot = cluster.slot_for(key) client = cluster.client_for(slot) client.get(key) end end end x.compare! end end end IpsSingle.run redis-rb-redis-cluster-client-aaf9e2e/test/ips_slot_node_mapping.rb000066400000000000000000000022031517214044400257150ustar00rootroot00000000000000# frozen_string_literal: true require 'benchmark/ips' require 'redis_cluster_client' module IpsSlotNodeMapping ELEMENTS = %w[foo bar baz].freeze SIZE = 16_384 module_function def run ca = ::RedisClient::Cluster::Node::CharArray.new(SIZE, ELEMENTS) arr = Array.new(SIZE) hs = {} print_letter('Mappings between slots and nodes') fullfill(ca) fullfill(arr) fullfill(hs) bench( { ca.class.name.split('::').last => ca, arr.class.name => arr, hs.class.name => hs } ) end def print_letter(title) print "################################################################################\n" print "# #{title}\n" print "################################################################################\n" print "\n" end def fullfill(arr) SIZE.times { |i| arr[i] = ELEMENTS[i % ELEMENTS.size] } end def bench(subjects) Benchmark.ips do |x| x.time = 5 x.warmup = 1 subjects.each do |subtitle, arr| x.report(subtitle) do arr[0] end end x.compare! end end end IpsSlotNodeMapping.run redis-rb-redis-cluster-client-aaf9e2e/test/middlewares/000077500000000000000000000000001517214044400233175ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/middlewares/command_capture.rb000066400000000000000000000036671517214044400270210ustar00rootroot00000000000000# frozen_string_literal: true module Middlewares module CommandCapture CapturedCommand = Struct.new('CapturedCommand', :server_url, :command, :pipelined, keyword_init: true) do def inspect "#<#{self.class.name} [on #{server_url}] #{command.join(' ')} >" end end # The CommandBuffer is what should be set as the :captured_commands custom option. # It needs to be threadsafe, because redis-cluster-client performs some redis operations on # multiple nodes in parallel, and in e.g. jruby it's not safe to concurrently manipulate the same array. class CommandBuffer def initialize @array = [] @mutex = Mutex.new end def to_a @mutex.synchronize do @array.dup end end def <<(command) @mutex.synchronize do @array << command end end def count(*cmd) @mutex.synchronize do next 0 if @array.empty? @array.count do |e| cmd.size.times.all? { |i| cmd[i].downcase == e.command[i]&.downcase } end end end def clear @mutex.synchronize do @array.clear end end end def call(command, redis_config) redis_config.custom[:captured_commands] << CapturedCommand.new( server_url: ::Middlewares::CommandCapture.normalize_captured_url(redis_config.server_url), command: command, pipelined: false ) super end def call_pipelined(commands, redis_config) commands.map do |command| redis_config.custom[:captured_commands] << CapturedCommand.new( server_url: ::Middlewares::CommandCapture.normalize_captured_url(redis_config.server_url), command: command, pipelined: true ) end super end def self.normalize_captured_url(url) URI.parse(url).tap do |u| u.path = '' end.to_s end end end redis-rb-redis-cluster-client-aaf9e2e/test/middlewares/redirect_count.rb000066400000000000000000000024121517214044400266540ustar00rootroot00000000000000# frozen_string_literal: true module Middlewares module RedirectCount class Counter Result = Struct.new('RedirectCountResult', :moved, :ask, keyword_init: true) def initialize @moved = 0 @ask = 0 @mutex = Mutex.new end def moved @mutex.synchronize { @moved += 1 } end def ask @mutex.synchronize { @ask += 1 } end def get @mutex.synchronize { Result.new(moved: @moved, ask: @ask) } end def zero? @mutex.synchronize { @moved == 0 && @ask == 0 } end def clear @mutex.synchronize do @moved = 0 @ask = 0 end end end def call(cmd, cfg) super rescue ::RedisClient::CommandError => e if e.message.start_with?('MOVED') cfg.custom.fetch(:redirect_count).moved elsif e.message.start_with?('ASK') cfg.custom.fetch(:redirect_count).ask end raise end def call_pipelined(cmd, cfg) super rescue ::RedisClient::CommandError => e if e.message.start_with?('MOVED') cfg.custom.fetch(:redirect_count).moved elsif e.message.start_with?('ASK') cfg.custom.fetch(:redirect_count).ask end raise end end end redis-rb-redis-cluster-client-aaf9e2e/test/middlewares/redirect_fake.rb000066400000000000000000000010421517214044400264300ustar00rootroot00000000000000# frozen_string_literal: true module Middlewares module RedirectFake Setting = Struct.new( 'RedirectFakeSetting', :slot, :to, :command, keyword_init: true ) def call(cmd, cfg) s = cfg.custom.fetch(:redirect_fake) raise RedisClient::CommandError, "MOVED #{s.slot} #{s.to}" if cmd == s.command super end def call_pipelined(cmd, cfg) s = cfg.custom.fetch(:redirect_fake) raise RedisClient::CommandError, "MOVED #{s.slot} #{s.to}" if cmd == s.command super end end end redis-rb-redis-cluster-client-aaf9e2e/test/prof_mem.rb000066400000000000000000000065321517214044400231560ustar00rootroot00000000000000# frozen_string_literal: true require 'memory_profiler' require 'redis_cluster_client' require 'testing_constants' module ProfMem module_function ATTEMPT_COUNT = 1000 MAX_PIPELINE_SIZE = 100 SLICED_NUMBERS = (1..ATTEMPT_COUNT).each_slice(MAX_PIPELINE_SIZE).freeze ORIGINAL_MGET = (%w[mget] + Array.new(40) { |i| "{key}#{i}" }).freeze EMULATED_MGET = (%w[mget] + Array.new(40) { |i| "key#{i}" }).freeze CLI_TYPES = %w[primary_only scale_read_random scale_read_latency pooled].freeze MODES = { single: lambda do |client_builder_method| cli = send(client_builder_method) ATTEMPT_COUNT.times { |i| cli.call('set', i, i) } ATTEMPT_COUNT.times { |i| cli.call('get', i) } end, excessive_pipelining: lambda do |client_builder_method| cli = send(client_builder_method) cli.pipelined do |pi| ATTEMPT_COUNT.times { |i| pi.call('set', i, i) } end cli.pipelined do |pi| ATTEMPT_COUNT.times { |i| pi.call('get', i) } end end, pipelining_in_moderation: lambda do |client_builder_method| cli = send(client_builder_method) SLICED_NUMBERS.each do |numbers| cli.pipelined do |pi| numbers.each { |i| pi.call('set', i, i) } end cli.pipelined do |pi| numbers.each { |i| pi.call('get', i) } end end end, original_mget: lambda do |client_builder_method| cli = send(client_builder_method) ATTEMPT_COUNT.times { cli.call_v(ORIGINAL_MGET) } end, emulated_mget: lambda do |client_builder_method| cli = send(client_builder_method) ATTEMPT_COUNT.times { cli.call_v(EMULATED_MGET) } end }.freeze def run mode = ENV.fetch('PROFILE_MODE', :single).to_sym subject = MODES.fetch(mode) CLI_TYPES.each do |cli_type| prepare print_letter(mode, cli_type) client_builder_method = "new_#{cli_type}_client".to_sym profile { subject.call(client_builder_method) } end end def prepare; end def print_letter(title, sub_titile) print "################################################################################\n" print "# #{title}: #{sub_titile}\n" print "################################################################################\n" print "\n" end def profile(&block) # https://github.com/SamSaffron/memory_profiler report = ::MemoryProfiler.report(top: 20, &block) report.pretty_print(color_output: true, normalize_paths: true) end def new_primary_only_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_client end def new_scale_read_random_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_client end def new_scale_read_latency_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :latency, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_client end def new_pooled_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_pool( timeout: TEST_TIMEOUT_SEC, size: 2 ) end end ProfMem.run redis-rb-redis-cluster-client-aaf9e2e/test/prof_stack.rb000066400000000000000000000034221517214044400235000ustar00rootroot00000000000000# frozen_string_literal: true require 'json' require 'tmpdir' require 'stackprof' require 'redis_cluster_client' require 'testing_constants' module ProfStack SIZE = 40 ATTEMPTS = 1000 ORIGINAL_MGET = (%w[mget] + Array.new(SIZE) { |i| "{key}#{i}" }).freeze EMULATED_MGET = (%w[mget] + Array.new(SIZE) { |i| "key#{i}" }).freeze module_function def run client = make_client mode = ENV.fetch('PROFILE_MODE', :single).to_sym prepare(client) profile = StackProf.run(mode: :cpu, raw: true) { execute(client, mode) } StackProf::Report.new(profile).print_text(false, 40) end def make_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_client end def prepare(client) ATTEMPTS.times do |i| client.pipelined do |pi| SIZE.times do |j| n = SIZE * i + j pi.call('set', "key#{n}", "val#{n}") pi.call('set', "{key}#{n}", "val#{n}") end end end end def execute(client, mode) case mode when :single (ATTEMPTS * SIZE).times { |i| client.call('get', "key#{i}") } when :excessive_pipelining client.pipelined do |pi| (ATTEMPTS * SIZE).times { |i| pi.call('get', "key#{i}") } end when :pipelining_in_moderation ATTEMPTS.times do |i| client.pipelined do |pi| SIZE.times do |j| n = SIZE * i + j pi.call('get', "key#{n}") end end end when :original_mget ATTEMPTS.times { client.call_v(ORIGINAL_MGET) } when :emulated_mget ATTEMPTS.times { client.call_v(EMULATED_MGET) } else raise ArgumentError, mode end end end ProfStack.run redis-rb-redis-cluster-client-aaf9e2e/test/prof_stack2.rb000066400000000000000000000032441517214044400235640ustar00rootroot00000000000000# frozen_string_literal: true require 'vernier' require 'redis_cluster_client' require 'testing_constants' module ProfStack2 SIZE = 40 ATTEMPTS = 1000 module_function def run client = make_client prepare(client) case mode = ENV.fetch('PROFILE_MODE', :single).to_sym when :single execute(client, mode) do |cli| ATTEMPTS.times { |i| cli.call('get', "key#{i}") } end when :transaction execute(client, mode) do |cli| ATTEMPTS.times do |i| cli.multi do |tx| SIZE.times do |j| n = SIZE * i + j tx.call('set', "{group:#{i}}:key:#{n}", n) end end end end when :pipeline execute(client, mode) do |cli| ATTEMPTS.times do |i| cli.pipelined do |pi| SIZE.times do |j| n = SIZE * i + j pi.call('get', "key#{n}") end end end end end end def execute(client, mode) Vernier.profile(out: "vernier_#{mode}.json") do yield(client) end end def make_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random, fixed_hostname: TEST_FIXED_HOSTNAME, # concurrency: { model: :on_demand, size: 6 }, # concurrency: { model: :pooled, size: 6 }, concurrency: { model: :none }, **TEST_GENERIC_OPTIONS ).new_client end def prepare(client) ATTEMPTS.times do |i| client.pipelined do |pi| SIZE.times do |j| n = SIZE * i + j pi.call('set', "key#{n}", "val#{n}") end end end end end ProfStack2.run redis-rb-redis-cluster-client-aaf9e2e/test/proxy/000077500000000000000000000000001517214044400222005ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/proxy/envoy.yaml000066400000000000000000000040461517214044400242300ustar00rootroot00000000000000--- # https://www.envoyproxy.io/docs/envoy/latest/configuration/overview/examples # https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/other_protocols/redis # https://github.com/envoyproxy/envoy/blob/main/examples/redis/envoy.yaml admin: address: socket_address: address: 0.0.0.0 port_value: 10000 static_resources: listeners: - name: redis address: socket_address: address: 0.0.0.0 port_value: 10001 filter_chains: - filters: # https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/network/redis_proxy/v3/redis_proxy.proto - name: envoy.filters.network.redis_proxy typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy stat_prefix: egress_redis settings: op_timeout: 5s enable_hashtagging: true enable_redirection: true read_policy: PREFER_REPLICA dns_cache_config: name: redis prefix_routes: catch_all_route: cluster: redis clusters: # https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/clusters/redis/v3/redis_cluster.proto - name: redis connect_timeout: 5s dns_lookup_family: V4_ONLY cluster_type: name: envoy.clusters.redis typed_config: "@type": type.googleapis.com/google.protobuf.Struct value: cluster_refresh_rate: 5s cluster_refresh_timeout: 4.5s redirect_refresh_interval: 5s redirect_refresh_threshold: 3 load_assignment: cluster_name: redis endpoints: - lb_endpoints: - endpoint: { address: { socket_address: { address: node1, port_value: 6379 } } } overload_manager: resource_monitors: - name: envoy.resource_monitors.global_downstream_max_connections typed_config: "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig max_active_downstream_connections: 100 redis-rb-redis-cluster-client-aaf9e2e/test/proxy/redis-cluster-proxy/000077500000000000000000000000001517214044400261445ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/proxy/redis-cluster-proxy/Dockerfile000066400000000000000000000013521517214044400301370ustar00rootroot00000000000000# https://hub.docker.com/_/debian # https://github.com/RedisLabs/redis-cluster-proxy FROM debian:12 AS builder ARG TAG=1.0-beta2 RUN apt-get update RUN apt-get install -y --no-install-recommends build-essential wget unzip ca-certificates WORKDIR /tmp RUN wget -O redis-cluster-proxy.zip "https://github.com/RedisLabs/redis-cluster-proxy/archive/refs/tags/${TAG}.zip" RUN unzip redis-cluster-proxy.zip WORKDIR /tmp/redis-cluster-proxy-${TAG} RUN make install REDIS_CLUSTER_PROXY_LDFLAGS=-zmuldefs # https://github.com/GoogleContainerTools/distroless FROM gcr.io/distroless/cc-debian12:nonroot COPY --from=builder /usr/local/bin/redis-cluster-proxy /usr/local/bin/redis-cluster-proxy USER nonroot ENTRYPOINT ["/usr/local/bin/redis-cluster-proxy"] redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/000077500000000000000000000000001517214044400234635ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/000077500000000000000000000000001517214044400251445ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/concurrent_worker/000077500000000000000000000000001517214044400307175ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/concurrent_worker/mixin.rb000066400000000000000000000040271517214044400323730ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster module ConcurrentWorker module Mixin def setup @worker = ::RedisClient::Cluster::ConcurrentWorker.create(model: model) end def test_work_group size = 10 group = @worker.new_group(size: size) size.times do |i| group.push(i, i) do |n| sleep 0.001 n * 2 end end want = Array.new(size) { |i| i * 2 } got = [] group.each do |_, v| # rubocop:disable Style/HashEachMethods got << v end assert_equal(want, got.sort) ensure group&.close end def test_work_group_with_error group = @worker.new_group(size: 5) 5.times do |i| group.push(i) { raise StandardError, 'should be handled' } end group.each do |id, v| assert_instance_of(StandardError, v, id) assert_equal('should be handled', v.message) end ensure group&.close end def test_too_many_tasks group = @worker.new_group(size: 5) 5.times { |i| group.push(i, i) { |n| n } } assert_raises(InvalidNumberOfTasks) { group.push(5, 5) { |n| n } } sum = 0 group.each { |_, v| sum += v } # rubocop:disable Style/HashEachMethods assert_equal(10, sum) ensure group&.close end def test_fewer_tasks group = @worker.new_group(size: 5) 4.times { |i| group.push(i, i) { |n| n } } sum = 0 assert_raises(InvalidNumberOfTasks) { group.each { |_, v| sum += v } } # rubocop:disable Style/HashEachMethods group.push(4, 4) { |n| n } group.each { |_, v| sum += v } # rubocop:disable Style/HashEachMethods assert_equal(10, sum) ensure group&.close end def teardown @worker.close end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/concurrent_worker/test_none.rb000066400000000000000000000005061517214044400332430ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' require 'redis_client/cluster/concurrent_worker/mixin' class RedisClient class Cluster module ConcurrentWorker class TestNone < TestingWrapper include Mixin private def model :none end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/concurrent_worker/test_on_demand.rb000066400000000000000000000005171517214044400342320ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' require 'redis_client/cluster/concurrent_worker/mixin' class RedisClient class Cluster module ConcurrentWorker class TestOnDemand < TestingWrapper include Mixin private def model :on_demand end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/concurrent_worker/test_pooled.rb000066400000000000000000000005121517214044400335630ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' require 'redis_client/cluster/concurrent_worker/mixin' class RedisClient class Cluster module ConcurrentWorker class TestPooled < TestingWrapper include Mixin private def model :pooled end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/node/000077500000000000000000000000001517214044400260715ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/node/test_latency_replica.rb000066400000000000000000000045401517214044400326160ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' require 'redis_client/cluster/node/testing_topology_mixin' class RedisClient class Cluster class Node class TestLatencyReplica < TestingWrapper TESTING_TOPOLOGY_OPTIONS = { replica: true, replica_affinity: :latency }.freeze include TestingTopologyMixin def test_clients_with_redis_client got = @test_node.clients got.each { |client| assert_instance_of(::RedisClient, client) } assert_equal(%w[master slave], got.map { |v| v.call('ROLE').first }.uniq.sort) end def test_clients_with_pooled_redis_client test_node = make_node(pool: { timeout: 3, size: 2 }) got = test_node.clients got.each { |client| assert_instance_of(::RedisClient::Pooled, client) } assert_equal(%w[master slave], got.map { |v| v.call('ROLE').first }.uniq.sort) end def test_primary_clients got = @test_node.primary_clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('master', client.call('ROLE').first) end end def test_replica_clients got = @test_node.replica_clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('slave', client.call('ROLE').first) end end def test_clients_for_scanning got = @test_node.clients_for_scanning got.each { |client| assert_instance_of(::RedisClient, client) } assert_equal(TEST_SHARD_SIZE, got.size) end def test_find_node_key_of_replica want = 'dummy_key' got = @test_topology.find_node_key_of_replica('dummy_key') assert_equal(want, got) primary_key = @replications.keys.first replica_keys = @replications.fetch(primary_key) got = @test_topology.find_node_key_of_replica(primary_key) assert_includes(replica_keys, got) end def test_any_primary_node_key got = @test_topology.any_primary_node_key assert_includes(@replications.keys, got) end def test_any_replica_node_key got = @test_topology.any_replica_node_key assert_includes(@replications.values.flatten, got) end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/node/test_primary_only.rb000066400000000000000000000041741517214044400322070ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' require 'redis_client/cluster/node/testing_topology_mixin' class RedisClient class Cluster class Node class TestPrimaryOnly < TestingWrapper TESTING_TOPOLOGY_OPTIONS = { replica: false }.freeze include TestingTopologyMixin def test_clients_with_redis_client got = @test_node.clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('master', client.call('ROLE').first) end end def test_clients_with_pooled_redis_client test_node = make_node(pool: { timeout: 3, size: 2 }) got = test_node.clients got.each do |client| assert_instance_of(::RedisClient::Pooled, client) assert_equal('master', client.call('ROLE').first) end end def test_primary_clients got = @test_node.primary_clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('master', client.call('ROLE').first) end end def test_replica_clients got = @test_node.replica_clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('master', client.call('ROLE').first) end end def test_clients_for_scanning got = @test_node.clients_for_scanning got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('master', client.call('ROLE').first) end end def test_find_node_key_of_replica want = 'dummy_key' got = @test_topology.find_node_key_of_replica('dummy_key') assert_equal(want, got) end def test_any_primary_node_key got = @test_topology.any_primary_node_key assert_includes(@replications.keys, got) end def test_any_replica_node_key got = @test_topology.any_replica_node_key assert_includes(@replications.keys, got) end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/node/test_random_replica.rb000066400000000000000000000045361517214044400324440ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' require 'redis_client/cluster/node/testing_topology_mixin' class RedisClient class Cluster class Node class TestRandomReplica < TestingWrapper TESTING_TOPOLOGY_OPTIONS = { replica: true, replica_affinity: :random }.freeze include TestingTopologyMixin def test_clients_with_redis_client got = @test_node.clients got.each { |client| assert_instance_of(::RedisClient, client) } assert_equal(%w[master slave], got.map { |v| v.call('ROLE').first }.uniq.sort) end def test_clients_with_pooled_redis_client test_node = make_node(pool: { timeout: 3, size: 2 }) got = test_node.clients got.each { |client| assert_instance_of(::RedisClient::Pooled, client) } assert_equal(%w[master slave], got.map { |v| v.call('ROLE').first }.uniq.sort) end def test_primary_clients got = @test_node.primary_clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('master', client.call('ROLE').first) end end def test_replica_clients got = @test_node.replica_clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('slave', client.call('ROLE').first) end end def test_clients_for_scanning got = @test_node.clients_for_scanning got.each { |client| assert_instance_of(::RedisClient, client) } assert_equal(TEST_SHARD_SIZE, got.size) end def test_find_node_key_of_replica want = 'dummy_key' got = @test_topology.find_node_key_of_replica('dummy_key') assert_equal(want, got) primary_key = @replications.keys.first replica_keys = @replications.fetch(primary_key) got = @test_topology.find_node_key_of_replica(primary_key) assert_includes(replica_keys, got) end def test_any_primary_node_key got = @test_topology.any_primary_node_key assert_includes(@replications.keys, got) end def test_any_replica_node_key got = @test_topology.any_replica_node_key assert_includes(@replications.values.flatten, got) end end end end end test_random_replica_or_primary.rb000066400000000000000000000046061517214044400346260ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/node# frozen_string_literal: true require 'testing_helper' require 'redis_client/cluster/node/testing_topology_mixin' class RedisClient class Cluster class Node class TestRandomReplicaWithPrimary < TestingWrapper TESTING_TOPOLOGY_OPTIONS = { replica: true, replica_affinity: :random_with_primary }.freeze include TestingTopologyMixin def test_clients_with_redis_client got = @test_node.clients got.each { |client| assert_instance_of(::RedisClient, client) } assert_equal(%w[master slave], got.map { |v| v.call('ROLE').first }.uniq.sort) end def test_clients_with_pooled_redis_client test_node = make_node(pool: { timeout: 3, size: 2 }) got = test_node.clients got.each { |client| assert_instance_of(::RedisClient::Pooled, client) } assert_equal(%w[master slave], got.map { |v| v.call('ROLE').first }.uniq.sort) end def test_primary_clients got = @test_node.primary_clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('master', client.call('ROLE').first) end end def test_replica_clients got = @test_node.replica_clients got.each do |client| assert_instance_of(::RedisClient, client) assert_equal('slave', client.call('ROLE').first) end end def test_clients_for_scanning got = @test_node.clients_for_scanning got.each { |client| assert_instance_of(::RedisClient, client) } assert_equal(TEST_SHARD_SIZE, got.size) end def test_find_node_key_of_replica want = 'dummy_key' got = @test_topology.find_node_key_of_replica('dummy_key') assert_equal(want, got) primary_key = @replications.keys.first replica_keys = @replications.fetch(primary_key) got = @test_topology.find_node_key_of_replica(primary_key) assert_includes(replica_keys + [primary_key], got) end def test_any_primary_node_key got = @test_topology.any_primary_node_key assert_includes(@replications.keys, got) end def test_any_replica_node_key got = @test_topology.any_replica_node_key assert_includes(@replications.values.flatten, got) end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/node/testing_topology_mixin.rb000066400000000000000000000020141517214044400332300ustar00rootroot00000000000000# frozen_string_literal: true class RedisClient class Cluster class Node module TestingTopologyMixin def make_node(pool: nil, **kwargs) config = ::RedisClient::ClusterConfig.new(**{ nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS, **self.class::TESTING_TOPOLOGY_OPTIONS }.merge(kwargs)) concurrent_worker = ::RedisClient::Cluster::ConcurrentWorker.create ::RedisClient::Cluster::Node.new(concurrent_worker, pool: pool, config: config).tap do |node| node.try_reload! @test_nodes ||= [] @test_nodes << node end end def setup @test_node = make_node @test_topology = @test_node.instance_variable_get(:@topology) @replications = @test_node.instance_variable_get(:@replications) end def teardown @test_nodes&.each { |n| n.each(&:close) } end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/test_command.rb000066400000000000000000000135241517214044400301530ustar00rootroot00000000000000# frozen_string_literal: true require 'set' require 'testing_helper' class RedisClient class Cluster class TestCommand < TestingWrapper def setup @raw_clients = TEST_NODE_URIS.map { |addr| ::RedisClient.config(url: addr, **TEST_GENERIC_OPTIONS).new_client } end def teardown @raw_clients&.each(&:close) end def test_load [ { nodes: @raw_clients, error: nil }, { nodes: [], error: ::RedisClient::Cluster::InitialSetupError }, { nodes: [''], error: NoMethodError }, { nodes: nil, error: ::RedisClient::Cluster::InitialSetupError } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = -> { ::RedisClient::Cluster::Command.load(c[:nodes]) } if c[:error].nil? assert_instance_of(::RedisClient::Cluster::Command, got.call, msg) else assert_raises(c[:error], msg, &got) end end end def test_load_slow_timeout nodes = @raw_clients assert_equal(TEST_TIMEOUT_SEC, nodes.first.read_timeout) nodes.first.singleton_class.prepend(Module.new do def call(...) @slow_timeout = read_timeout super end end) ::RedisClient::Cluster::Command.load(nodes, slow_command_timeout: 9) assert_equal(9, nodes.first.instance_variable_get(:@slow_timeout)) assert_equal(TEST_TIMEOUT_SEC, nodes.first.read_timeout) end def test_parse_command_reply [ { rows: [ ['get', 2, Set['readonly', 'fast'], 1, -1, 1, Set['@read', '@string', '@fast'], Set[], Set[], Set[]], ['set', -3, Set['write', 'denyoom', 'movablekeys'], 1, -1, 2, Set['@write', '@string', '@slow'], Set[], Set[], Set[]] ], want: { 'get' => { first_key_position: 1, key_step: 1, write?: false, readonly?: true }, 'set' => { first_key_position: 1, key_step: 2, write?: true, readonly?: false } } }, { rows: [[]], want: {} }, { rows: [], want: {} }, { rows: nil, want: {} } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = ::RedisClient::Cluster::Command.send(:parse_command_reply, c[:rows]) assert_equal(c[:want].size, got.size, msg) assert_equal(c[:want].keys.sort, got.keys.sort, msg) c[:want].each do |k, v| assert_equal(v, got[k].to_h, "#{msg}: #{k}") end end end def test_extract_first_key cmd = ::RedisClient::Cluster::Command.load(@raw_clients) [ { command: %w[set foo 1], want: 'foo' }, { command: %w[SET foo 1], want: 'foo' }, { command: %w[get foo], want: 'foo' }, { command: %w[get foo{bar}baz], want: 'foo{bar}baz' }, { command: %w[mget foo bar baz], want: 'foo' }, { command: ['eval', 'return ARGV[1]', '0', 'hello'], want: 'hello' }, { command: %w[evalsha sha1 2 foo bar baz zap], want: 'foo' }, { command: %w[migrate host port key 0 5 copy], want: 'key' }, { command: ['migrate', 'host', 'port', '', '0', '5', 'copy', 'keys', 'key'], want: 'key' }, { command: %w[zinterstore out 2 zset1 zset2 weights 2 3], want: 'zset1' }, { command: %w[zunionstore out 2 zset1 zset2 weights 2 3], want: 'zset1' }, { command: %w[object encoding key], want: 'key' }, { command: %w[memory help], want: '' }, { command: %w[memory usage key], want: 'key' }, { command: %w[xread count 2 streams mystream writers 0-0 0-0], want: 'mystream' }, { command: %w[xreadgroup group group consumer streams key id], want: 'key' }, { command: %w[unknown foo bar], want: '' } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = cmd.extract_first_key(c[:command]) assert_equal(c[:want], got, msg) end end def test_should_send_to_primary? cmd = ::RedisClient::Cluster::Command.load(@raw_clients) [ { command: %w[set foo 1], want: true }, { command: %w[SET foo 1], want: true }, { command: %w[get foo], want: false }, { command: %w[GET foo], want: false }, { command: %w[unknown foo bar], want: nil }, { command: [], want: nil } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = cmd.should_send_to_primary?(c[:command]) c[:want].nil? ? assert_nil(got, msg) : assert_equal(c[:want], got, msg) end end def test_should_send_to_replica? cmd = ::RedisClient::Cluster::Command.load(@raw_clients) [ { command: %w[set foo 1], want: false }, { command: %w[SET foo 1], want: false }, { command: %w[get foo], want: true }, { command: %w[GET foo], want: true }, { command: %w[unknown foo bar], want: nil }, { command: [], want: nil } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = cmd.should_send_to_replica?(c[:command]) c[:want].nil? ? assert_nil(got, msg) : assert_equal(c[:want], got, msg) end end def test_exists? cmd = ::RedisClient::Cluster::Command.load(@raw_clients) [ { name: 'ping', want: true }, { name: :ping, want: true }, { name: 'PING', want: true }, { name: 'densaugeo', want: false }, { name: :densaugeo, want: false }, { name: 'DENSAUGEO', want: false }, { name: '', want: false }, { name: 0, want: false }, { name: nil, want: false } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = cmd.exists?(c[:name]) assert_equal(c[:want], got, msg) end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/test_errors.rb000066400000000000000000000062221517214044400300460ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' class RedisClient class Cluster class TestErrors < TestingWrapper DummyError = Class.new(StandardError) def test_initial_setup_error [ { errors: [DummyError.new('foo')], want: 'Redis client could not fetch cluster information: foo' }, { errors: [DummyError.new('foo'), DummyError.new('bar')], want: 'Redis client could not fetch cluster information: foo,bar' }, { errors: [], want: 'Redis client could not fetch cluster information: ' }, { errors: '', want: 'Redis client could not fetch cluster information: ' }, { errors: nil, want: 'Redis client could not fetch cluster information: ' } ].each_with_index do |c, idx| raise ::RedisClient::Cluster::InitialSetupError.from_errors(c[:errors]) rescue StandardError => e assert_equal(c[:want], e.message, "Case: #{idx}") end end def test_orchestration_command_not_supported_error [ { command: %w[CLUSTER FORGET], want: 'CLUSTER FORGET command should be' }, { command: [], want: ' command should be' }, { command: '', want: ' command should be' }, { command: nil, want: ' command should be' } ].each_with_index do |c, idx| raise ::RedisClient::Cluster::OrchestrationCommandNotSupported.from_command(c[:command]) rescue StandardError => e assert(e.message.start_with?(c[:want]), "Case: #{idx}") end end def test_error_collection_error [ { errors: { '127.0.0.1:6379' => DummyError.new('foo') }, want: { msg: '127.0.0.1:6379: (RedisClient::Cluster::TestErrors::DummyError) foo', size: 1 } }, { errors: { '127.0.0.1:6379' => DummyError.new('foo'), '127.0.0.1:6380' => DummyError.new('bar') }, want: { msg: '127.0.0.1:6379: (RedisClient::Cluster::TestErrors::DummyError) foo, 127.0.0.1:6380: (RedisClient::Cluster::TestErrors::DummyError) bar', size: 2 } }, { errors: {}, want: { msg: '{}', size: 0 } }, { errors: [], want: { msg: '[]', size: 0 } }, { errors: '', want: { msg: '', size: 0 } }, { errors: nil, want: { msg: '', size: 0 } } ].each_with_index do |c, idx| raise ::RedisClient::Cluster::ErrorCollection.with_errors(c[:errors]) rescue StandardError => e assert_equal(c[:want][:msg], e.message, "Case: #{idx}") assert_equal(c[:want][:size], e.errors.size, "Case: #{idx}") end end def test_ambiguous_node_error [ { command: 'MULTI', want: "Cluster client doesn't know which node the MULTI command should be sent to." }, { command: nil, want: "Cluster client doesn't know which node the command should be sent to." } ].each_with_index do |c, idx| raise ::RedisClient::Cluster::AmbiguousNodeError.from_command(c[:command]) rescue StandardError => e assert_equal(e.message, c[:want], "Case: #{idx}") end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/test_key_slot_converter.rb000066400000000000000000000047451517214044400324620ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' class RedisClient class Cluster class TestKeySlotConverter < TestingWrapper def setup @raw_clients = TEST_NODE_URIS.map { |addr| ::RedisClient.config(url: addr, **TEST_GENERIC_OPTIONS).new_client } end def teardown @raw_clients&.each(&:close) end def test_convert (1..255).map { |i| "key#{i}" }.each_with_index do |key, idx| want = @raw_clients.first.call('CLUSTER', 'KEYSLOT', key) got = ::RedisClient::Cluster::KeySlotConverter.convert(key) assert_equal(want, got, "Case: #{idx}") end assert_nil(::RedisClient::Cluster::KeySlotConverter.convert(nil), 'Case: nil') multi_byte_key = 'あいうえお' want = @raw_clients.first.call('CLUSTER', 'KEYSLOT', multi_byte_key) got = ::RedisClient::Cluster::KeySlotConverter.convert(multi_byte_key) assert_equal(want, got, "Case: #{multi_byte_key}") end def test_extract_hash_tag [ { key: 'foo', want: '' }, { key: 'foo{bar}baz', want: 'bar' }, { key: 'foo{bar}baz{qux}quuc', want: 'bar' }, { key: 'foo}bar{baz', want: '' }, { key: 'foo{bar', want: '' }, { key: 'foo}bar', want: '' }, { key: 'foo{}bar', want: '' }, { key: '{}foo', want: '' }, { key: 'foo{}', want: '' }, { key: '{}', want: '' }, { key: '', want: '' }, { key: nil, want: '' } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = ::RedisClient::Cluster::KeySlotConverter.extract_hash_tag(c[:key]) assert_equal(c[:want], got, msg) end end def test_hash_tag_included? [ { key: 'foo', want: false }, { key: 'foo{bar}baz', want: true }, { key: 'foo{bar}baz{qux}quuc', want: true }, { key: 'foo}bar{baz', want: false }, { key: 'foo{bar', want: false }, { key: 'foo}bar', want: false }, { key: 'foo{}bar', want: false }, { key: '{}foo', want: false }, { key: 'foo{}', want: false }, { key: '{}', want: false }, { key: '', want: false }, { key: nil, want: false } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = ::RedisClient::Cluster::KeySlotConverter.hash_tag_included?(c[:key]) assert_equal(c[:want], got, msg) end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/test_node.rb000066400000000000000000001373371517214044400274730ustar00rootroot00000000000000# frozen_string_literal: true require 'uri' require 'testing_helper' class RedisClient class Cluster class Node class TestConfig < TestingWrapper def test_connection_prelude [ { params: { scale_read: true }, want: [%w[HELLO 3], %w[readonly]] }, { params: { scale_read: false }, want: [%w[HELLO 3]] }, { params: {}, want: [%w[HELLO 3]] } ].each_with_index do |c, idx| got = ::RedisClient::Cluster::Node::Config.new(**c[:params]).connection_prelude assert_equal(c[:want], got, "Case: #{idx}") end end end end # rubocop:disable Metrics/ClassLength class TestNode < TestingWrapper USE_CHAR_ARRAY_SLOT = Integer(ENV.fetch('REDIS_CLIENT_USE_CHAR_ARRAY_SLOT', 1)) == 1 SLOT_SIZE = 16_384 MAX_STARTUP_SAMPLE = Integer(ENV.fetch('REDIS_CLIENT_MAX_STARTUP_SAMPLE', 3)) def setup @test_node = make_node.tap(&:try_reload!) @test_node_with_scale_read = make_node(replica: true).tap(&:try_reload!) @test_node_info_list = @test_node.instance_variable_get(:@node_info) end def teardown @test_nodes&.each { |n| n&.each(&:close) } end def make_node(capture_buffer: ::Middlewares::CommandCapture::CommandBuffer.new, pool: nil, **kwargs) config = ::RedisClient::ClusterConfig.new(**{ nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, middlewares: [::Middlewares::CommandCapture], custom: { captured_commands: capture_buffer }, **TEST_GENERIC_OPTIONS }.merge(kwargs)) concurrent_worker = ::RedisClient::Cluster::ConcurrentWorker.create ::RedisClient::Cluster::Node.new(concurrent_worker, pool: pool, config: config).tap do |node| @test_nodes ||= [] @test_nodes << node end end def test_parse_cluster_node_reply_continuous_slots info = <<~INFO 07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004 slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 connected 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002 master - 0 1426238316232 2 connected 5461-10922 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003 master - 0 1426238318243 3 connected 10923-16383 6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005 slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 connected 824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006 slave 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 connected e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001 myself,master - 0 0 1 connected 0-5460 INFO want = [ { id: '07c37dfeb235213a872192d90877d0cd55635b91', node_key: '127.0.0.1:30004', role: 'slave', primary_id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', ping_sent: '0', pong_recv: '1426238317239', config_epoch: '4', link_state: 'connected', slots: [] }, { id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', node_key: '127.0.0.1:30002', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '2', link_state: 'connected', slots: [[5461, 10_922]] }, { id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', node_key: '127.0.0.1:30003', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238318243', config_epoch: '3', link_state: 'connected', slots: [[10_923, 16_383]] }, { id: '6ec23923021cf3ffec47632106199cb7f496ce01', node_key: '127.0.0.1:30005', role: 'slave', primary_id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '5', link_state: 'connected', slots: [] }, { id: '824fe116063bc5fcf9f4ffd895bc17aee7731ac3', node_key: '127.0.0.1:30006', role: 'slave', primary_id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', ping_sent: '0', pong_recv: '1426238317741', config_epoch: '6', link_state: 'connected', slots: [] }, { id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', node_key: '127.0.0.1:30001', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '0', config_epoch: '1', link_state: 'connected', slots: [[0, 5460]] } ] got = @test_node.send(:parse_cluster_node_reply, info) assert_equal(want, got.map(&:to_h)) end def test_parse_cluster_node_reply_discrete_slots info = <<~INFO 07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004 slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 connected 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002 master - 0 1426238316232 2 connected 3001 5461-7000 7002-10922 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003 master - 0 1426238318243 3 connected 7001 10923-15000 15002-16383 6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005 slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 connected 824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006 slave 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 connected e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001 myself,master - 0 0 1 connected 0-3000 3002-5460 15001 INFO want = [ { id: '07c37dfeb235213a872192d90877d0cd55635b91', node_key: '127.0.0.1:30004', role: 'slave', primary_id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', ping_sent: '0', pong_recv: '1426238317239', config_epoch: '4', link_state: 'connected', slots: [] }, { id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', node_key: '127.0.0.1:30002', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '2', link_state: 'connected', slots: [[3001, 3001], [5461, 7000], [7002, 10_922]] }, { id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', node_key: '127.0.0.1:30003', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238318243', config_epoch: '3', link_state: 'connected', slots: [[7001, 7001], [10_923, 15_000], [15_002, 16_383]] }, { id: '6ec23923021cf3ffec47632106199cb7f496ce01', node_key: '127.0.0.1:30005', role: 'slave', primary_id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '5', link_state: 'connected', slots: [] }, { id: '824fe116063bc5fcf9f4ffd895bc17aee7731ac3', node_key: '127.0.0.1:30006', role: 'slave', primary_id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', ping_sent: '0', pong_recv: '1426238317741', config_epoch: '6', link_state: 'connected', slots: [] }, { id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', node_key: '127.0.0.1:30001', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '0', config_epoch: '1', link_state: 'connected', slots: [[0, 3000], [3002, 5460], [15_001, 15_001]] } ] got = @test_node.send(:parse_cluster_node_reply, info) assert_equal(want, got.map(&:to_h)) end def test_parse_cluster_node_reply_discrete_slots_and_resharding info = <<~INFO 07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004 slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 connected 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002 master - 0 1426238316232 2 connected 3001 5461-7000 7002-10922 [5462->-292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f] 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003 master - 0 1426238318243 3 connected 7001 10923-15000 15002-16383 [5462-<-67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1] 6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005 slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 connected 824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006 slave 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 connected e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001 myself,master - 0 0 1 connected 0-3000 3002-5460 15001 INFO want = [ { id: '07c37dfeb235213a872192d90877d0cd55635b91', node_key: '127.0.0.1:30004', role: 'slave', primary_id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', ping_sent: '0', pong_recv: '1426238317239', config_epoch: '4', link_state: 'connected', slots: [] }, { id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', node_key: '127.0.0.1:30002', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '2', link_state: 'connected', slots: [[3001, 3001], [5461, 7000], [7002, 10_922]] }, { id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', node_key: '127.0.0.1:30003', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238318243', config_epoch: '3', link_state: 'connected', slots: [[7001, 7001], [10_923, 15_000], [15_002, 16_383]] }, { id: '6ec23923021cf3ffec47632106199cb7f496ce01', node_key: '127.0.0.1:30005', role: 'slave', primary_id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '5', link_state: 'connected', slots: [] }, { id: '824fe116063bc5fcf9f4ffd895bc17aee7731ac3', node_key: '127.0.0.1:30006', role: 'slave', primary_id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', ping_sent: '0', pong_recv: '1426238317741', config_epoch: '6', link_state: 'connected', slots: [] }, { id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', node_key: '127.0.0.1:30001', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '0', config_epoch: '1', link_state: 'connected', slots: [[0, 3000], [3002, 5460], [15_001, 15_001]] } ] got = @test_node.send(:parse_cluster_node_reply, info) assert_equal(want, got.map(&:to_h)) end def test_parse_cluster_node_reply_with_hostname info = <<~INFO 07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004,localhost slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 connected 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002,localhost master - 0 1426238316232 2 connected 5461-10922 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003,localhost master - 0 1426238318243 3 connected 10923-16383 6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005,localhost slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 connected 824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006,localhost slave 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 connected e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001,localhost myself,master - 0 0 1 connected 0-5460 INFO want = [ { id: '07c37dfeb235213a872192d90877d0cd55635b91', node_key: 'localhost:30004', role: 'slave', primary_id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', ping_sent: '0', pong_recv: '1426238317239', config_epoch: '4', link_state: 'connected', slots: [] }, { id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', node_key: 'localhost:30002', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '2', link_state: 'connected', slots: [[5461, 10_922]] }, { id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', node_key: 'localhost:30003', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238318243', config_epoch: '3', link_state: 'connected', slots: [[10_923, 16_383]] }, { id: '6ec23923021cf3ffec47632106199cb7f496ce01', node_key: 'localhost:30005', role: 'slave', primary_id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '5', link_state: 'connected', slots: [] }, { id: '824fe116063bc5fcf9f4ffd895bc17aee7731ac3', node_key: 'localhost:30006', role: 'slave', primary_id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', ping_sent: '0', pong_recv: '1426238317741', config_epoch: '6', link_state: 'connected', slots: [] }, { id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', node_key: 'localhost:30001', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '0', config_epoch: '1', link_state: 'connected', slots: [[0, 5460]] } ] got = @test_node.send(:parse_cluster_node_reply, info) assert_equal(want, got.map(&:to_h)) end def test_parse_cluster_node_reply_with_hostname_and_auxiliaries info = <<~INFO 07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004,localhost,shard-id=69bc080733d1355567173199cff4a6a039a2f024 slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 connected 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002,localhost,shard-id=114f6674a35b84949fe567f5dfd41415ee776261 master - 0 1426238316232 2 connected 5461-10922 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003,localhost,shard-id=fdb36c73e72dd027bc19811b7c219ef6e55c550e master - 0 1426238318243 3 connected 10923-16383 6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005,localhost,shard-id=114f6674a35b84949fe567f5dfd41415ee776261 slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 connected 824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006,localhost,shard-id=fdb36c73e72dd027bc19811b7c219ef6e55c550e slave 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 connected e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001,localhost,shard-id=69bc080733d1355567173199cff4a6a039a2f024 myself,master - 0 0 1 connected 0-5460 INFO want = [ { id: '07c37dfeb235213a872192d90877d0cd55635b91', node_key: 'localhost:30004', role: 'slave', primary_id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', ping_sent: '0', pong_recv: '1426238317239', config_epoch: '4', link_state: 'connected', slots: [] }, { id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', node_key: 'localhost:30002', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '2', link_state: 'connected', slots: [[5461, 10_922]] }, { id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', node_key: 'localhost:30003', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238318243', config_epoch: '3', link_state: 'connected', slots: [[10_923, 16_383]] }, { id: '6ec23923021cf3ffec47632106199cb7f496ce01', node_key: 'localhost:30005', role: 'slave', primary_id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '5', link_state: 'connected', slots: [] }, { id: '824fe116063bc5fcf9f4ffd895bc17aee7731ac3', node_key: 'localhost:30006', role: 'slave', primary_id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', ping_sent: '0', pong_recv: '1426238317741', config_epoch: '6', link_state: 'connected', slots: [] }, { id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', node_key: 'localhost:30001', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '0', config_epoch: '1', link_state: 'connected', slots: [[0, 5460]] } ] got = @test_node.send(:parse_cluster_node_reply, info) assert_equal(want, got.map(&:to_h)) end def test_parse_cluster_node_reply_with_auxiliaries info = <<~INFO 07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004,,shard-id=69bc080733d1355567173199cff4a6a039a2f024 slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 connected 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002,,shard-id=114f6674a35b84949fe567f5dfd41415ee776261 master - 0 1426238316232 2 connected 5461-10922 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003,,shard-id=fdb36c73e72dd027bc19811b7c219ef6e55c550e master - 0 1426238318243 3 connected 10923-16383 6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005,,shard-id=114f6674a35b84949fe567f5dfd41415ee776261 slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 connected 824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006,,shard-id=fdb36c73e72dd027bc19811b7c219ef6e55c550e slave 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 connected e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001,,shard-id=69bc080733d1355567173199cff4a6a039a2f024 myself,master - 0 0 1 connected 0-5460 INFO want = [ { id: '07c37dfeb235213a872192d90877d0cd55635b91', node_key: '127.0.0.1:30004', role: 'slave', primary_id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', ping_sent: '0', pong_recv: '1426238317239', config_epoch: '4', link_state: 'connected', slots: [] }, { id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', node_key: '127.0.0.1:30002', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '2', link_state: 'connected', slots: [[5461, 10_922]] }, { id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', node_key: '127.0.0.1:30003', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '1426238318243', config_epoch: '3', link_state: 'connected', slots: [[10_923, 16_383]] }, { id: '6ec23923021cf3ffec47632106199cb7f496ce01', node_key: '127.0.0.1:30005', role: 'slave', primary_id: '67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1', ping_sent: '0', pong_recv: '1426238316232', config_epoch: '5', link_state: 'connected', slots: [] }, { id: '824fe116063bc5fcf9f4ffd895bc17aee7731ac3', node_key: '127.0.0.1:30006', role: 'slave', primary_id: '292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f', ping_sent: '0', pong_recv: '1426238317741', config_epoch: '6', link_state: 'connected', slots: [] }, { id: 'e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca', node_key: '127.0.0.1:30001', role: 'master', primary_id: '-', ping_sent: '0', pong_recv: '0', config_epoch: '1', link_state: 'connected', slots: [[0, 5460]] } ] got = @test_node.send(:parse_cluster_node_reply, info) assert_equal(want, got.map(&:to_h)) end def test_parse_cluster_node_reply_failure_link_state info = <<~INFO 07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004 slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 disconnected 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002 master - 0 1426238316232 2 disconnected 5461-10922 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003 master - 0 1426238318243 3 disconnected 10923-16383 6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005 slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 disconnected 824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006 slave 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 disconnected e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001 myself,master - 0 0 1 disconnected 0-5460 INFO assert_empty(@test_node.send(:parse_cluster_node_reply, info)) end def test_parse_cluster_node_reply_failure_flags info = <<~INFO 07c37dfeb235213a872192d90877d0cd55635b91 127.0.0.1:30004@31004 fail?,slave e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 0 1426238317239 4 connected 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 127.0.0.1:30002@31002 fail,master - 0 1426238316232 2 connected 5461-10922 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 127.0.0.1:30003@31003 master,handshake - 0 1426238318243 3 connected 10923-16383 6ec23923021cf3ffec47632106199cb7f496ce01 127.0.0.1:30005@31005 noaddr,slave 67ed2db8d677e59ec4a4cefb06858cf2a1a89fa1 0 1426238316232 5 connected 824fe116063bc5fcf9f4ffd895bc17aee7731ac3 127.0.0.1:30006@31006 noflags 292f8b365bb7edb5e285caf0b7e6ddc7265d2f4f 0 1426238317741 6 connected e7d1eecce10fd6bb5eb35b9f99a514335d9ba9ca 127.0.0.1:30001@31001 myself,fail,master - 0 0 1 connected 0-5460 INFO assert_empty(@test_node.send(:parse_cluster_node_reply, info)) end def test_parse_cluster_slots_reply reply = [ [ 0, 5460, ['10.10.1.6', 6379, '00c0d00f2a5eda22b2c8a8929ba27b454c4400fb', {}], ['10.10.1.5', 6379, 'b60c0672f257c01d76f27eacded14b6e6f4f990e', {}] ], [ 5461, 10_922, ['10.10.1.4', 6379, '712b9a6656b38a5e002244903853fccb4d1eef4b', {}], ['10.10.1.7', 6379, '7038691c545e7caa9147030ecfb4acf1eaad0552', {}] ], [ 10_923, 16_383, ['10.10.1.8', 6379, 'ba85d0807043bb40f72bb4e1e8352b029c6e0082', {}], ['10.10.1.3', 6379, 'f2f36b472b187c577ccd93dd296e9045f473ae7a', {}] ] ] want = [ { id: '00c0d00f2a5eda22b2c8a8929ba27b454c4400fb', node_key: '10.10.1.6:6379', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[0, 5460]] }, { id: 'b60c0672f257c01d76f27eacded14b6e6f4f990e', node_key: '10.10.1.5:6379', role: 'slave', primary_id: '00c0d00f2a5eda22b2c8a8929ba27b454c4400fb', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] }, { id: '712b9a6656b38a5e002244903853fccb4d1eef4b', node_key: '10.10.1.4:6379', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[5461, 10_922]] }, { id: '7038691c545e7caa9147030ecfb4acf1eaad0552', node_key: '10.10.1.7:6379', role: 'slave', primary_id: '712b9a6656b38a5e002244903853fccb4d1eef4b', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] }, { id: 'ba85d0807043bb40f72bb4e1e8352b029c6e0082', node_key: '10.10.1.8:6379', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[10_923, 16_383]] }, { id: 'f2f36b472b187c577ccd93dd296e9045f473ae7a', node_key: '10.10.1.3:6379', role: 'slave', primary_id: 'ba85d0807043bb40f72bb4e1e8352b029c6e0082', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] } ] got = @test_node.send(:parse_cluster_slots_reply, reply) assert_equal(want.sort_by { |e| e.fetch(:id) }, got.sort_by(&:id).map(&:to_h)) end def test_parse_cluster_shards_reply_on_resp3 reply = [ { 'slots' => [5461, 10_922], 'nodes' => [ { 'id' => '712b9a6656b38a5e002244903853fccb4d1eef4b', 'port' => 6379, 'ip' => '10.10.1.4', 'endpoint' => '10.10.1.4', 'role' => 'master', 'replication-offset' => 98, 'health' => 'online' }, { 'id' => '7038691c545e7caa9147030ecfb4acf1eaad0552', 'port' => 6379, 'ip' => '10.10.1.7', 'endpoint' => '10.10.1.7', 'role' => 'replica', 'replication-offset' => 98, 'health' => 'online' } ] }, { 'slots' => [10_923, 16_383], 'nodes' => [ { 'id' => 'ba85d0807043bb40f72bb4e1e8352b029c6e0082', 'port' => 6379, 'ip' => '10.10.1.8', 'endpoint' => '10.10.1.8', 'role' => 'master', 'replication-offset' => 98, 'health' => 'online' }, { 'id' => 'f2f36b472b187c577ccd93dd296e9045f473ae7a', 'port' => 6379, 'ip' => '10.10.1.3', 'endpoint' => '10.10.1.3', 'role' => 'replica', 'replication-offset' => 98, 'health' => 'online' } ] }, { 'slots' => [0, 5460], 'nodes' => [ { 'id' => '00c0d00f2a5eda22b2c8a8929ba27b454c4400fb', 'port' => 6379, 'ip' => '10.10.1.6', 'endpoint' => '10.10.1.6', 'role' => 'master', 'replication-offset' => 98, 'health' => 'online' }, { 'id' => 'b60c0672f257c01d76f27eacded14b6e6f4f990e', 'port' => 6379, 'ip' => '10.10.1.5', 'endpoint' => '10.10.1.5', 'role' => 'replica', 'replication-offset' => 98, 'health' => 'online' } ] } ] want = [ { id: '00c0d00f2a5eda22b2c8a8929ba27b454c4400fb', node_key: '10.10.1.6:6379', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[0, 5460]] }, { id: 'b60c0672f257c01d76f27eacded14b6e6f4f990e', node_key: '10.10.1.5:6379', role: 'slave', primary_id: '00c0d00f2a5eda22b2c8a8929ba27b454c4400fb', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] }, { id: '712b9a6656b38a5e002244903853fccb4d1eef4b', node_key: '10.10.1.4:6379', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[5461, 10_922]] }, { id: '7038691c545e7caa9147030ecfb4acf1eaad0552', node_key: '10.10.1.7:6379', role: 'slave', primary_id: '712b9a6656b38a5e002244903853fccb4d1eef4b', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] }, { id: 'ba85d0807043bb40f72bb4e1e8352b029c6e0082', node_key: '10.10.1.8:6379', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[10_923, 16_383]] }, { id: 'f2f36b472b187c577ccd93dd296e9045f473ae7a', node_key: '10.10.1.3:6379', role: 'slave', primary_id: 'ba85d0807043bb40f72bb4e1e8352b029c6e0082', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] } ] got = @test_node.send(:parse_cluster_shards_reply, reply) assert_equal(want.sort_by { |e| e.fetch(:id) }, got.sort_by(&:id).map(&:to_h)) end def test_parse_cluster_shards_reply_on_resp2 reply = [ [ 'slots', [10_923, 16_383], 'nodes', [ [ 'id', '90e1b98ef2ef598fa5a19966b345d4dfe7fe719d', 'port', 16_382, 'ip', '127.0.0.1', 'endpoint', '127.0.0.1', 'role', 'master', 'replication-offset', 642_515, 'health', 'online' ], [ 'id', '845d5784e64ab4576da6e7b35dc02b64b6c10af5', 'port', 16_385, 'ip', '127.0.0.1', 'endpoint', '127.0.0.1', 'role', 'replica', 'replication-offset', 642_515, 'health', 'online' ] ] ], [ 'slots', [5461, 10_922], 'nodes', [ [ 'id', 'd7d3328d5205a477337f1fdfaadcd2dacd8aecda', 'port', 16_381, 'ip', '127.0.0.1', 'endpoint', '127.0.0.1', 'role', 'master', 'replication-offset', 642_515, 'health', 'online' ], [ 'id', 'beeb7f1c33cde531e290c32c830f2401ab1b8dea', 'port', 16_384, 'ip', '127.0.0.1', 'endpoint', '127.0.0.1', 'role', 'replica', 'replication-offset', 642_515, 'health', 'online' ] ] ], [ 'slots', [0, 5460], 'nodes', [ [ 'id', 'ea3e0173888ef6d7d7bc488f5ceccc2a146bd898', 'port', 16_380, 'ip', '127.0.0.1', 'endpoint', '127.0.0.1', 'role', 'master', 'replication-offset', 642_515, 'health', 'online' ], [ 'id', '57b349e52d741ebebb12933fc081c45cbe1ea85a', 'port', 16_383, 'ip', '127.0.0.1', 'endpoint', '127.0.0.1', 'role', 'replica', 'replication-offset', 642_515, 'health', 'online' ] ] ] ] want = [ { id: 'ea3e0173888ef6d7d7bc488f5ceccc2a146bd898', node_key: '127.0.0.1:16380', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[0, 5460]] }, { id: '57b349e52d741ebebb12933fc081c45cbe1ea85a', node_key: '127.0.0.1:16383', role: 'slave', primary_id: 'ea3e0173888ef6d7d7bc488f5ceccc2a146bd898', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] }, { id: 'd7d3328d5205a477337f1fdfaadcd2dacd8aecda', node_key: '127.0.0.1:16381', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[5461, 10_922]] }, { id: 'beeb7f1c33cde531e290c32c830f2401ab1b8dea', node_key: '127.0.0.1:16384', role: 'slave', primary_id: 'd7d3328d5205a477337f1fdfaadcd2dacd8aecda', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] }, { id: '90e1b98ef2ef598fa5a19966b345d4dfe7fe719d', node_key: '127.0.0.1:16382', role: 'master', primary_id: '', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [[10_923, 16_383]] }, { id: '845d5784e64ab4576da6e7b35dc02b64b6c10af5', node_key: '127.0.0.1:16385', role: 'slave', primary_id: '90e1b98ef2ef598fa5a19966b345d4dfe7fe719d', ping_sent: nil, pong_recv: nil, config_epoch: nil, link_state: nil, slots: [] } ] got = @test_node.send(:parse_cluster_shards_reply, reply) assert_equal(want.sort_by { |e| e.fetch(:id) }, got.sort_by(&:id).map(&:to_h)) end def test_inspect assert_match(/^#$/, @test_node.inspect) end def test_enumerable refute(@test_node.any?(&:nil?)) end def test_node_keys want = @test_node_info_list.map(&:node_key) @test_node.node_keys.each do |got| assert_includes(want, got, "Case: #{got}") end end def test_find_by @test_node_info_list.each do |info| msg = "Case: primary only: #{info.node_key}" got = -> { @test_node.find_by(info.node_key) } if info.primary? assert_instance_of(::RedisClient, got.call, msg) else assert_raises(::RedisClient::Cluster::Node::ReloadNeeded, msg, &got) end msg = "Case: scale read: #{info.node_key}" got = @test_node_with_scale_read.find_by(info.node_key) assert_instance_of(::RedisClient, got, msg) end end def test_call_all want = (1..(@test_node_info_list.count(&:primary?))).map { |_| 'PONG' } got = @test_node.call_all(:call_v, ['PING'], []) assert_equal(want, got, 'Case: primary only') want = (1..(@test_node_info_list.count)).map { |_| 'PONG' } got = @test_node_with_scale_read.call_all(:call_v, ['PING'], []) assert_equal(want, got, 'Case: scale read') end def test_call_primaries want = (1..(@test_node_info_list.count(&:primary?))).map { |_| 'PONG' } got = @test_node.call_primaries(:call_v, ['PING'], []) assert_equal(want, got) got = @test_node_with_scale_read.call_primaries(:call_v, ['PING'], []) assert_equal(want, got, 'Case: scale read') end def test_call_replicas want = (1..(@test_node_info_list.count(&:primary?))).map { |_| 'PONG' } got = @test_node.call_replicas(:call_v, ['PING'], []) assert_equal(want, got, 'Case: primary only') got = @test_node_with_scale_read.call_replicas(:call_v, ['PING'], []) assert_equal(want, got, 'Case: scale read') end def test_send_ping want = (1..(@test_node_info_list.count(&:primary?))).map { |_| 'PONG' } got = @test_node.send_ping(:call_v, ['PING'], []) assert_equal(want, got, 'Case: primary only') want = (1..(@test_node_info_list.count)).map { |_| 'PONG' } got = @test_node_with_scale_read.send_ping(:call_v, ['PING'], []) assert_equal(want, got, 'Case: scale read') end def test_clients_for_scanning test_config = @test_node.instance_variable_get(:@config) want = @test_node_info_list.select(&:primary?) .map(&:node_key) # need to call client_config_for_node so that if we're using fixed_hostname in this test, # we get the actual hostname we're connecting to, not the one returned by the cluster API .map { |key| test_config.client_config_for_node(key) } .map { |cfg| "#{cfg[:host]}:#{cfg[:port]}" } .sort got = @test_node.clients_for_scanning.map { |client| "#{client.config.host}:#{client.config.port}" }.sort assert_equal(want, got, 'Case: primary only') want = @test_node_info_list.select(&:replica?) .map(&:node_key) # As per above, we need to get the real hostname, not that reported by Redis, # if fixed_hostname is set. .map { |key| test_config.client_config_for_node(key) } .map { |cfg| "#{cfg[:host]}:#{cfg[:port]}" } .sort got = @test_node_with_scale_read.clients_for_scanning.map { |client| "#{client.config.host}:#{client.config.port}" } got.each { |e| assert_includes(want, e, 'Case: scale read') } end def test_find_node_key_of_primary sample_node = @test_node_info_list.find(&:primary?) sample_slot = sample_node.slots.first.first got = @test_node.find_node_key_of_primary(sample_slot) assert_equal(sample_node.node_key, got, 'Case: sample slot') assert_nil(@test_node.find_node_key_of_primary(nil), 'Case: nil') end def test_find_node_key_of_replica sample_node = @test_node_info_list.find(&:primary?) sample_slot = sample_node.slots.first.first got = @test_node.find_node_key_of_replica(sample_slot) assert_equal(sample_node.node_key, got, 'Case: primary only') sample_replicas = @test_node_info_list.select(&:replica?) sample_primary = @test_node_info_list.find { |info| info.id == sample_replicas.first.primary_id } sample_slot = sample_primary.slots.first.first got = @test_node_with_scale_read.find_node_key_of_replica(sample_slot) want = sample_replicas.map(&:node_key) assert_includes(want, got, 'Case: scale read') assert_nil(@test_node.find_node_key_of_replica(nil), 'Case: nil') end def test_any_primary_node_key primary_node_keys = @test_node_info_list.select(&:primary?).map(&:node_key) got = @test_node.any_primary_node_key assert_includes(primary_node_keys, got, 'Case: primary only') got = @test_node_with_scale_read.any_primary_node_key assert_includes(primary_node_keys, got, 'Case: scale read') end def test_any_replica_node_key primary_node_keys = @test_node_info_list.select(&:primary?).map(&:node_key) replica_node_keys = @test_node_info_list.select(&:replica?).map(&:node_key) got = @test_node.any_replica_node_key assert_includes(primary_node_keys, got, 'Case: primary only') got = @test_node_with_scale_read.any_replica_node_key assert_includes(replica_node_keys, got, 'Case: scale read') end def test_update_slot sample_slot = 0 base_node_key = @test_node.find_node_key_of_primary(sample_slot) another_node_key = @test_node_info_list.find { |info| info.node_key != base_node_key && info.primary? }&.node_key @test_node.update_slot(sample_slot, another_node_key) assert_equal(another_node_key, @test_node.find_node_key_of_primary(sample_slot)) end def test_make_topology_class [ { with_replica: false, replica_affinity: :foo, want: ::RedisClient::Cluster::Node::PrimaryOnly }, { with_replica: true, replica_affinity: :foo, want: ::RedisClient::Cluster::Node::PrimaryOnly }, { with_replica: true, replica_affinity: :random, want: ::RedisClient::Cluster::Node::RandomReplica }, { with_replica: true, replica_affinity: :latency, want: ::RedisClient::Cluster::Node::LatencyReplica } ].each_with_index do |c, i| got = @test_node.send(:make_topology_class, c[:with_replica], c[:replica_affinity]) assert_equal(c[:want], got, "Case: #{i}") end end def test_build_slot_node_mappings node_info_list = [ { node_key: '127.0.0.1:7001', role: 'master', slots: [[0, 3000], [3002, 5460], [15_001, 15_001]] }, { node_key: '127.0.0.1:7002', role: 'master', slots: [[3001, 3001], [5461, 7000], [7002, 10_922]] }, { node_key: '127.0.0.1:7003', role: 'master', slots: [[7001, 7001], [10_923, 15_000], [15_002, 16_383]] }, { node_key: '127.0.0.1:7004', role: 'slave', slots: [] }, { node_key: '127.0.0.1:7005', role: 'slave', slots: [] }, { node_key: '127.0.0.1:7006', role: 'slave', slots: [] } ].map { |info| ::RedisClient::Cluster::Node::Info.new(**info) } got = @test_node.send(:build_slot_node_mappings, node_info_list) node_info_list.each do |info| next if info.slots.empty? info.slots.each do |range| (range[0]..range[1]).each { |slot| assert_same(info.node_key, got[slot], "Case: #{slot}") } end end end def test_make_array_for_slot_node_mappings_optimized node_info_list = Array.new(256) do |i| ::RedisClient::Cluster::Node::Info.new( node_key: "127.0.0.1:#{1024 + i + 1}", role: 'master' ) end want = node_info_list.first.node_key got = @test_node.send(:make_array_for_slot_node_mappings, node_info_list) assert_instance_of(USE_CHAR_ARRAY_SLOT ? ::RedisClient::Cluster::Node::CharArray : Array, got) SLOT_SIZE.times do |i| got[i] = want assert_equal(want, got[i], "Case: #{i}") end end def test_make_array_for_slot_node_mappings_unoptimized node_info_list = Array.new(257) do |i| ::RedisClient::Cluster::Node::Info.new( node_key: "127.0.0.1:#{1024 + i + 1}", role: 'master' ) end want = node_info_list.first.node_key got = @test_node.send(:make_array_for_slot_node_mappings, node_info_list) assert_instance_of(Array, got) SLOT_SIZE.times do |i| got[i] = want assert_equal(want, got[i], "Case: #{i}") end end def test_make_array_for_slot_node_mappings_max_shard_size node_info_list = Array.new(255) do |i| ::RedisClient::Cluster::Node::Info.new( node_key: "127.0.0.1:#{1024 + i + 1}", role: 'master' ) end got = @test_node.send(:make_array_for_slot_node_mappings, node_info_list) assert_instance_of(USE_CHAR_ARRAY_SLOT ? ::RedisClient::Cluster::Node::CharArray : Array, got) SLOT_SIZE.times { |i| got[i] = node_info_list.first.node_key } got[0] = 'newbie:6379' assert_equal('newbie:6379', got[0]) assert_raises(RangeError) { got[0] = 'zombie:6379' } if USE_CHAR_ARRAY_SLOT assert_raises(IndexError) { got[-1] = 'newbie:6379' } if USE_CHAR_ARRAY_SLOT assert_raises(IndexError) { got[-1] } if USE_CHAR_ARRAY_SLOT got[16_384] = 'newbie:6379' assert_nil(got[16_384]) if USE_CHAR_ARRAY_SLOT end def test_build_replication_mappings_regular node_key1 = '127.0.0.1:7001' node_key2 = '127.0.0.1:7002' node_key3 = '127.0.0.1:7003' node_key4 = '127.0.0.1:7004' node_key5 = '127.0.0.1:7005' node_key6 = '127.0.0.1:7006' node_key7 = '127.0.0.1:7007' node_key8 = '127.0.0.1:7008' node_key9 = '127.0.0.1:7009' node_info_list = [ { id: '1', node_key: node_key1, primary_id: '-' }, { id: '2', node_key: node_key2, primary_id: '-' }, { id: '3', node_key: node_key3, primary_id: '-' }, { id: '4', node_key: node_key4, primary_id: '1' }, { id: '5', node_key: node_key5, primary_id: '2' }, { id: '6', node_key: node_key6, primary_id: '3' }, { id: '7', node_key: node_key7, primary_id: '1' }, { id: '8', node_key: node_key8, primary_id: '2' }, { id: '9', node_key: node_key9, primary_id: '3' } ].map { |info| ::RedisClient::Cluster::Node::Info.new(**info) } got = @test_node.send(:build_replication_mappings, node_info_list) got.transform_values!(&:sort!) assert_same(node_key4, got[node_key1][0]) assert_same(node_key7, got[node_key1][1]) assert_same(node_key5, got[node_key2][0]) assert_same(node_key8, got[node_key2][1]) assert_same(node_key6, got[node_key3][0]) assert_same(node_key9, got[node_key3][1]) end def test_build_replication_mappings_lack_of_replica node_key1 = '127.0.0.1:7001' # node_key2 = '127.0.0.1:7002' # lack node_key3 = '127.0.0.1:7003' node_key4 = '127.0.0.1:7004' node_key5 = '127.0.0.1:7005' node_key6 = '127.0.0.1:7006' node_info_list = [ { id: '1', role: 'master', node_key: node_key1, primary_id: '-' }, { id: '3', role: 'master', node_key: node_key3, primary_id: '-' }, { id: '4', role: 'slave', node_key: node_key4, primary_id: '1' }, { id: '5', role: 'master', node_key: node_key5, primary_id: '-' }, { id: '6', role: 'slave', node_key: node_key6, primary_id: '3' } ].map { |info| ::RedisClient::Cluster::Node::Info.new(**info) } got = @test_node.send(:build_replication_mappings, node_info_list) got.transform_values!(&:sort!) assert_equal(3, got.size) assert_same(node_key4, got[node_key1][0]) assert_same(node_key6, got[node_key3][0]) assert_empty(got[node_key5]) end def test_try_map primary_node_keys = @test_node_info_list.select(&:primary?).map(&:node_key) [ { block: ->(_, client) { client.call('PING') }, results: primary_node_keys.to_h { |k| [k, 'PONG'] } }, { block: ->(_, client) { client.call('UNKNOWN') }, errors: ::RedisClient::CommandError } ].each_with_index do |c, idx| msg = "Case: #{idx}" clients = @test_node.instance_variable_get(:@topology).clients results, errors = @test_node.send(:try_map, clients, &c[:block]) if c.key?(:errors) errors.each_value { |e| assert_instance_of(c[:errors], e, msg) } else assert_equal(c[:results], results, msg) end end end def test_reload capture_buffer = ::Middlewares::CommandCapture::CommandBuffer.new test_node = make_node(replica: true, capture_buffer: capture_buffer) capture_buffer.clear test_node.try_reload! # It should have reloaded by calling CLUSTER NODES on three of the startup nodes subcmd = TEST_REDIS_MAJOR_VERSION >= 7 ? 'shards' : 'nodes' cluster_node_cmds = capture_buffer.to_a.select { |c| c.command == ['cluster', subcmd] } assert_equal MAX_STARTUP_SAMPLE, cluster_node_cmds.size # It should have connected to all of the clients. assert_equal TEST_NUMBER_OF_NODES, test_node.to_a.size # If we reload again, it should NOT change the redis client instances we have. original_client_ids = test_node.to_a.map(&:object_id).to_set test_node.try_reload! new_client_ids = test_node.to_a.map(&:object_id).to_set assert_equal original_client_ids, new_client_ids end def test_reload_with_original_config bootstrap_node = TEST_NODE_URIS.first capture_buffer = ::Middlewares::CommandCapture::CommandBuffer.new test_node = make_node( nodes: [bootstrap_node], replica: true, connect_with_original_config: true, capture_buffer: capture_buffer ) test_node.try_reload! # After reloading the first time, our Node object knows about all hosts, despite only starting with one: assert_equal TEST_NUMBER_OF_NODES, test_node.to_a.size # When we reload, it will only call CLUSTER NODES against a single node, the bootstrap node. capture_buffer.clear test_node.send(:bypass_reload!) subcmd = TEST_REDIS_MAJOR_VERSION >= 7 ? 'shards' : 'nodes' cluster_node_cmds = capture_buffer.to_a.select { |c| c.command == ['cluster', subcmd] } assert_equal 1, cluster_node_cmds.size assert_equal bootstrap_node, cluster_node_cmds.first.server_url end def test_reload_with_overriden_sample_size capture_buffer = ::Middlewares::CommandCapture::CommandBuffer.new test_node = make_node(replica: true, capture_buffer: capture_buffer, max_startup_sample: 1) capture_buffer.clear test_node.try_reload! # It should have reloaded by calling CLUSTER NODES on one of the startup nodes subcmd = TEST_REDIS_MAJOR_VERSION >= 7 ? 'shards' : 'nodes' cluster_node_cmds = capture_buffer.to_a.select { |c| c.command == ['cluster', subcmd] } assert_equal 1, cluster_node_cmds.size # It should have connected to all of the clients. assert_equal TEST_NUMBER_OF_NODES, test_node.to_a.size # If we reload again, it should NOT change the redis client instances we have. original_client_ids = test_node.to_a.map(&:object_id).to_set test_node.try_reload! new_client_ids = test_node.to_a.map(&:object_id).to_set assert_equal original_client_ids, new_client_ids end def test_reload_concurrently capture_buffer = ::Middlewares::CommandCapture::CommandBuffer.new test_node = make_node(replica: true, pool: { size: 2 }, capture_buffer: capture_buffer) # Simulate refetch_node_info_list taking a long time test_node.singleton_class.prepend(Module.new do def refetch_node_info_list(...) r = super sleep 2 r end end) capture_buffer.clear t1 = Thread.new { test_node.try_reload! } t2 = Thread.new { test_node.try_reload! } [t1, t2].each(&:join) # We should only have reloaded once, which is to say, we only called CLUSTER NODES command MAX_STARTUP_SAMPLE # times subcmd = TEST_REDIS_MAJOR_VERSION >= 7 ? 'shards' : 'nodes' cluster_node_cmds = capture_buffer.to_a.select { |c| c.command == ['cluster', subcmd] } assert_equal MAX_STARTUP_SAMPLE, cluster_node_cmds.size end end # rubocop:enable Metrics/ClassLength end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/cluster/test_node_key.rb000066400000000000000000000050121517214044400303230ustar00rootroot00000000000000# frozen_string_literal: true require 'uri' require 'testing_helper' class RedisClient class Cluster class TestNodeKey < TestingWrapper def test_hashify [ { node_key: '127.0.0.1:6379', want: { host: '127.0.0.1', port: '6379' } }, { node_key: '::1:6379', want: { host: '::1', port: '6379' } }, { node_key: 'foobar', want: { host: 'foobar', port: nil } }, { node_key: '', want: { host: '', port: nil } }, { node_key: nil, want: { host: nil, port: nil } } ].each_with_index do |c, idx| got = ::RedisClient::Cluster::NodeKey.hashify(c[:node_key]) assert_equal(c[:want], got, "Case: #{idx}") end end def test_split [ { node_key: '127.0.0.1:6379', want: ['127.0.0.1', '6379'] }, { node_key: '::1:6379', want: ['::1', '6379'] }, { node_key: 'foobar', want: ['foobar', nil] }, { node_key: '', want: ['', nil] }, { node_key: nil, want: [nil, nil] } ].each_with_index do |c, idx| got = ::RedisClient::Cluster::NodeKey.split(c[:node_key]) assert_equal(c[:want], got, "Case: #{idx}") end end def test_build_from_uri [ { uri: URI('redis://127.0.0.1:6379'), want: '127.0.0.1:6379' }, { uri: nil, want: '' } ].each_with_index do |c, idx| got = ::RedisClient::Cluster::NodeKey.build_from_uri(c[:uri]) assert_equal(c[:want], got, "Case: #{idx}") end end def test_build_from_host_port [ { params: { host: '127.0.0.1', port: 6379 }, want: '127.0.0.1:6379' }, { params: { host: nil, port: nil }, want: ':' } ].each_with_index do |c, idx| got = ::RedisClient::Cluster::NodeKey.build_from_host_port(c[:params][:host], c[:params][:port]) assert_equal(c[:want], got, "Case: #{idx}") end end def test_build_from_client dummy_client = Struct.new(:config, keyword_init: true) dummy_config = Struct.new(:host, :port, keyword_init: true) dummy = dummy_client.new(config: dummy_config.new(host: '127.0.0.1', port: '6379')) [ { client: dummy, want: '127.0.0.1:6379' }, { client: ::RedisClient.new(host: '127.0.0.1', port: '6379'), want: '127.0.0.1:6379' } ].each_with_index do |c, idx| got = ::RedisClient::Cluster::NodeKey.build_from_client(c[:client]) assert_equal(c[:want], got, "Case: #{idx}") end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/test_cluster.rb000066400000000000000000001151351517214044400265360ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' class RedisClient class TestCluster module Mixin def setup @captured_commands = ::Middlewares::CommandCapture::CommandBuffer.new @redirect_count = ::Middlewares::RedirectCount::Counter.new @client = new_test_client @client.call('FLUSHDB') wait_for_replication @captured_commands.clear @redirect_count.clear end def teardown @client&.call('FLUSHDB') wait_for_replication @client&.close flunk(@redirect_count.get) unless @redirect_count.zero? end def test_config refute_nil @client.config refute_nil @client.config.connect_timeout refute_nil @client.config.read_timeout refute_nil @client.config.write_timeout end def test_inspect assert_match(/^#$/, @client.inspect) end def test_call assert_raises(ArgumentError) { @client.call } 10.times do |i| assert_equal('OK', @client.call('SET', "key#{i}", i), "Case: SET: key#{i}") wait_for_replication assert_equal(i.to_s, @client.call('GET', "key#{i}"), "Case: GET: key#{i}") end assert(@client.call('PING') { |r| r == 'PONG' }) assert_equal(2, @client.call('HSET', 'hash', foo: 1, bar: 2)) wait_for_replication assert_equal(%w[1 2], @client.call('HMGET', 'hash', %w[foo bar])) end def test_call_once assert_raises(ArgumentError) { @client.call_once } 10.times do |i| assert_equal('OK', @client.call_once('SET', "key#{i}", i), "Case: SET: key#{i}") wait_for_replication assert_equal(i.to_s, @client.call_once('GET', "key#{i}"), "Case: GET: key#{i}") end assert(@client.call_once('PING') { |r| r == 'PONG' }) assert_equal(2, @client.call_once('HSET', 'hash', foo: 1, bar: 2)) wait_for_replication assert_equal(%w[1 2], @client.call_once('HMGET', 'hash', %w[foo bar])) end def test_blocking_call skip("FIXME: this case is buggy on #{RUBY_ENGINE}") if RUBY_ENGINE == 'truffleruby' # FIXME: buggy assert_raises(ArgumentError) { @client.blocking_call(TEST_TIMEOUT_SEC) } @client.call_v(%w[RPUSH foo hello]) @client.call_v(%w[RPUSH foo world]) wait_for_replication client_side_timeout = TEST_REDIS_MAJOR_VERSION < 6 ? 2.0 : 1.5 server_side_timeout = TEST_REDIS_MAJOR_VERSION < 6 ? '1' : '0.5' swap_timeout(@client, timeout: 0.1) do |client| assert_equal(%w[foo world], client.blocking_call(client_side_timeout, 'BRPOP', 'foo', server_side_timeout), 'Case: 1st') # FIXME: too flaky, just a workaround got = client.blocking_call(client_side_timeout, 'BRPOP', 'foo', server_side_timeout) if got.nil? assert_nil(got, 'Case: 2nd') else assert_equal(%w[foo hello], got, 'Case: 2nd') end assert_nil(client.blocking_call(client_side_timeout, 'BRPOP', 'foo', server_side_timeout), 'Case: 3rd') assert_raises(::RedisClient::ReadTimeoutError, 'Case: 4th') { client.blocking_call(0.1, 'BRPOP', 'foo', 0) } end end def test_scan 10.times { |i| @client.call('SET', "key#{i}", i) } wait_for_replication want = (0..9).map { |i| "key#{i}" } got = [] @client.scan('COUNT', '5') { |key| got << key } assert_equal(want, got.sort) end def test_sscan 10.times do |i| 10.times { |j| @client.call('SADD', "key#{i}", "member#{j}") } wait_for_replication want = (0..9).map { |j| "member#{j}" } got = [] @client.sscan("key#{i}", 'COUNT', '5') { |member| got << member } assert_equal(want, got.sort) end end def test_hscan 10.times do |i| 10.times { |j| @client.call('HSET', "key#{i}", "field#{j}", j) } wait_for_replication want = (0..9).map { |j| ["field#{j}", j.to_s] } got = [] @client.hscan("key#{i}", 'COUNT', '5') { |pair| got << pair } assert_equal(want, got.sort) end end def test_zscan 10.times do |i| 10.times { |j| @client.call('ZADD', "key#{i}", j, "member#{j}") } wait_for_replication want = (0..9).map { |j| ["member#{j}", j.to_s] } got = [] @client.zscan("key#{i}", 'COUNT', '5') { |pair| got << pair } assert_equal(want, got.sort) end end def test_pipelined assert_empty([], @client.pipelined { |_| 1 + 1 }) want = (0..9).map { 'OK' } + (1..3).to_a + %w[PONG] got = @client.pipelined do |pipeline| 10.times { |i| pipeline.call('SET', "string#{i}", i) } 3.times { |i| pipeline.call('RPUSH', 'list', i) } pipeline.call_once('PING') end assert_equal(want, got) wait_for_replication want = %w[PONG] + (0..9).map(&:to_s) + [%w[list 2]] client_side_timeout = TEST_REDIS_MAJOR_VERSION < 6 ? 1.5 : 1.0 server_side_timeout = TEST_REDIS_MAJOR_VERSION < 6 ? '1' : '0.5' swap_timeout(@client, timeout: 0.1) do |client| got = client.pipelined do |pipeline| pipeline.call_once('PING') 10.times { |i| pipeline.call('GET', "string#{i}") } pipeline.blocking_call(client_side_timeout, 'BRPOP', 'list', server_side_timeout) end assert_equal(want, got) end end def test_pipelined_with_errors assert_raises(RedisClient::Cluster::ErrorCollection) do @client.pipelined do |pipeline| 10.times do |i| pipeline.call('SET', "string#{i}", i) pipeline.call('SET', "string#{i}", i, 'too many args') pipeline.call('SET', "string#{i}", i + 10) end end end wait_for_replication 10.times { |i| assert_equal((i + 10).to_s, @client.call('GET', "string#{i}")) } end def test_pipelined_with_errors_as_is got = @client.pipelined(exception: false) do |pipeline| 10.times do |i| pipeline.call('SET', "string#{i}", i) pipeline.call('SET', "string#{i}", i, 'too many args') pipeline.call('SET', "string#{i}", i + 10) end end assert_equal(30, got.size) 10.times do |i| assert_equal('OK', got[(3 * i) + 0]) assert_instance_of(::RedisClient::CommandError, got[(3 * i) + 1]) assert_equal('OK', got[(3 * i) + 2]) end wait_for_replication 10.times { |i| assert_equal((i + 10).to_s, @client.call('GET', "string#{i}")) } end def test_pipelined_with_many_commands @client.pipelined { |pi| 1000.times { |i| pi.call('SET', i, i) } } wait_for_replication results = @client.pipelined { |pi| 1000.times { |i| pi.call('GET', i) } } results.each_with_index { |got, i| assert_equal(i.to_s, got) } end def test_transaction_with_single_key got = @client.multi do |t| t.call('SET', 'counter', '0') t.call('INCR', 'counter') t.call('INCR', 'counter') end assert_equal(['OK', 1, 2], got) wait_for_replication assert_equal('2', @client.call('GET', 'counter')) end def test_transaction_with_multiple_key assert_raises(::RedisClient::Cluster::Transaction::ConsistencyError) do @client.multi do |t| t.call('SET', 'key1', '1') t.call('SET', 'key2', '2') t.call('SET', 'key3', '3') end end (1..3).each do |i| assert_nil(@client.call('GET', "key#{i}")) end end def test_transaction_without_block assert_raises(LocalJumpError) { @client.multi } end def test_transaction_with_empty_block @captured_commands.clear assert_empty(@client.multi {}) assert_empty(@captured_commands.to_a.map(&:command).map(&:first)) end def test_transaction_with_empty_block_and_watch @captured_commands.clear assert_empty(@client.multi(watch: %w[key]) {}) assert_equal(%w[watch multi exec], @captured_commands.to_a.map(&:command).map(&:first)) end def test_transaction_with_early_return_block @captured_commands.clear condition = true got = @client.multi do |tx| next if condition tx.call('SET', 'key', 'value') end assert_empty(got) assert_empty(@captured_commands.to_a.map(&:command).map(&:first)) assert_nil(@client.call('GET', 'key')) end def test_transaction_with_early_return_block_in_watching @captured_commands.clear condition = true got = @client.multi(watch: %w[key]) do |tx| next if condition tx.call('SET', 'key', 'value') end assert_empty(got) assert_equal(%w[watch multi exec], @captured_commands.to_a.map(&:command).map(&:first)) assert_nil(@client.call('GET', 'key')) end def test_transaction_with_only_keyless_commands assert_raises(::RedisClient::Cluster::Transaction::ConsistencyError) do @client.multi do |t| t.call('ECHO', 'foo') t.call('ECHO', 'bar') end end end def test_transaction_with_hashtag got = @client.multi do |t| t.call('MSET', '{key}1', '1', '{key}2', '2') t.call('MSET', '{key}3', '3', '{key}4', '4') end assert_equal(%w[OK OK], got) wait_for_replication assert_equal(%w[1 2 3 4], @client.call('MGET', '{key}1', '{key}2', '{key}3', '{key}4')) end def test_transaction_without_hashtag assert_raises(::RedisClient::Cluster::Transaction::ConsistencyError) do @client.multi do |t| t.call('MSET', 'key1', '1', 'key2', '2') t.call('MSET', 'key3', '3', 'key4', '4') end end assert_raises(::RedisClient::Cluster::Transaction::ConsistencyError) do @client.multi do |t| t.call('MSET', 'key1', '1', 'key2', '2') t.call('MSET', 'key1', '1', 'key3', '3') t.call('MSET', 'key1', '1', 'key4', '4') end end (1..4).each do |i| assert_nil(@client.call('GET', "key#{i}")) end end def test_transaction_with_watch @client.call('MSET', '{key}1', '0', '{key}2', '0') got = @client.multi(watch: %w[{key}1 {key}2]) do |tx| tx.call('ECHO', 'START') tx.call('SET', '{key}1', '1') tx.call('SET', '{key}2', '2') tx.call('ECHO', 'FINISH') end assert_equal(%w[START OK OK FINISH], got) wait_for_replication assert_equal(%w[1 2], @client.call('MGET', '{key}1', '{key}2')) end def test_transaction_with_unsafe_watch @client.call('MSET', '{key}1', '0', '{key}2', '0') assert_raises(::RedisClient::Cluster::Transaction::ConsistencyError) do @client.multi(watch: %w[key1 key2]) do |tx| tx.call('SET', '{key}1', '1') tx.call('SET', '{key}2', '2') end end assert_raises(::RedisClient::Cluster::Transaction::ConsistencyError) do @client.multi(watch: %w[{hey}1 {hey}2]) do |tx| tx.call('SET', '{key}1', '1') tx.call('SET', '{key}2', '2') end end wait_for_replication assert_equal(%w[0 0], @client.call('MGET', '{key}1', '{key}2')) end def test_transaction_with_meaningless_watch @client.call('MSET', '{key}1', '0', '{key}2', '0') got = @client.multi(watch: %w[{key}3 {key}4]) do |tx| tx.call('ECHO', 'START') tx.call('SET', '{key}1', '1') tx.call('SET', '{key}2', '2') tx.call('ECHO', 'FINISH') end assert_equal(%w[START OK OK FINISH], got) wait_for_replication assert_equal(%w[1 2], @client.call('MGET', '{key}1', '{key}2')) end def test_transaction_does_not_pointlessly_unwatch_on_success @client.call('MSET', '{key}1', '0', '{key}2', '0') @captured_commands.clear @client.multi(watch: %w[{key}1 {key}2]) do |tx| tx.call('SET', '{key}1', '1') tx.call('SET', '{key}2', '2') end assert_equal(%w[watch multi SET SET exec], @captured_commands.to_a.map(&:command).map(&:first)) wait_for_replication assert_equal(%w[1 2], @client.call('MGET', '{key}1', '{key}2')) end def test_transaction_unwatches_on_error test_error = Class.new(StandardError) @captured_commands.clear assert_raises(test_error) do @client.multi(watch: %w[{key}1 {key}2]) do raise test_error, 'error!' end end assert_equal(%w[watch unwatch], @captured_commands.to_a.map(&:command).map(&:first)) end def test_transaction_does_not_unwatch_on_connection_error @captured_commands.clear assert_raises(RedisClient::ConnectionError) do @client.multi(watch: %w[{key}1 {key}2]) do |tx| tx.call('SET', '{key}1', 'x') tx.call('QUIT') end end command_list = @captured_commands.to_a.map(&:command).map(&:first) assert_includes(command_list, 'watch') refute_includes(command_list, 'unwatch') end def test_transaction_does_not_retry_without_rewatching client2 = new_test_client(middlewares: nil) @client.call('SET', 'key', 'original_value') assert_raises(RedisClient::ConnectionError) do @client.multi(watch: %w[key]) do |tx| # Simulate all the connections closing behind the router's back # Sending QUIT to redis makes the server side close the connection (and the client # side thus get a RedisClient::ConnectionError) node = @client.instance_variable_get(:@router).instance_variable_get(:@node) node.clients.each do |conn| conn.with(&:close) end # Now the second client sets the value, which should make this watch invalid client2.call('SET', 'key', 'client2_value') tx.call('SET', 'key', '@client_value') # Committing this transaction will fail, not silently reconnect (without the watch!) end end # The transaction did not commit. wait_for_replication assert_equal('client2_value', @client.call('GET', 'key')) end def test_transaction_with_watch_retries_block client2 = new_test_client(middlewares: nil) call_count = 0 @client.call('SET', 'key', 'original_value') @client.multi(watch: %w[key]) do |tx| if call_count == 0 # Simulate all the connections closing behind the router's back # Sending QUIT to redis makes the server side close the connection (and the client # side thus get a RedisClient::ConnectionError) node = @client.instance_variable_get(:@router).instance_variable_get(:@node) node.clients.each do |conn| conn.with(&:close) end # Now the second client sets the value, which should make this watch invalid client2.call('SET', 'key', 'client2_value') end call_count += 1 tx.call('SET', 'key', "@client_value_#{call_count}") end # The transaction did commit (but it was the second time) wait_for_replication assert_equal('@client_value_2', @client.call('GET', 'key')) assert_equal(2, call_count) end def test_transaction_with_error @client.call('SET', 'key1', 'x') assert_raises(::RedisClient::CommandError) do @client.multi do |tx| tx.call('SET', 'key1', 'aaa') tx.call('MYBAD', 'key1', 'bbb') end end wait_for_replication assert_equal('x', @client.call('GET', 'key1')) end def test_transaction_without_error_during_queueing @client.call('SET', 'key1', 'x') assert_raises(::RedisClient::CommandError) do @client.multi do |tx| tx.call('SET', 'key1', 'aaa') tx.call('INCR', 'key1') end end wait_for_replication assert_equal('aaa', @client.call('GET', 'key1')) end def test_transaction_with_block @client.call('MSET', '{key}1', 'a', '{key}2', 'b', '{key}3', 'c') got = @client.multi do |tx| tx.call('GET', '{key}1') { |x| "#{x}aa" } tx.call('GET', '{key}2') { |x| "#{x}bb" } tx.call('GET', '{key}3') { |x| "#{x}cc" } end assert_equal(%w[aaa bbb ccc], got) got = @client.multi(watch: %w[{key}1 {key}2 {key}3]) do |tx| tx.call('GET', '{key}1') { |x| "#{x}11" } tx.call('GET', '{key}2') { |x| "#{x}22" } tx.call('GET', '{key}3') { |x| "#{x}33" } end assert_equal(%w[a11 b22 c33], got) end def test_transaction_in_race_condition @client.call('MSET', '{key}1', '1', '{key}2', '2') another = Fiber.new do cli = new_test_client(middlewares: nil) cli.call('MSET', '{key}1', '3', '{key}2', '4') cli.close Fiber.yield end got = @client.multi(watch: %w[{key}1 {key}2]) do |tx| another.resume v1 = @client.call('GET', '{key}1') v2 = @client.call('GET', '{key}2') tx.call('SET', '{key}1', v2) tx.call('SET', '{key}2', v1) end assert_nil(got) wait_for_replication assert_equal(%w[3 4], @client.call('MGET', '{key}1', '{key}2')) end def test_transaction_with_dedicated_watch_command @client.call('MSET', '{key}1', '0', '{key}2', '0') got = @client.call('WATCH', '{key}1', '{key}2') do |tx| tx.call('ECHO', 'START') tx.call('SET', '{key}1', '1') tx.call('SET', '{key}2', '2') tx.call('ECHO', 'FINISH') end assert_equal(%w[START OK OK FINISH], got) wait_for_replication assert_equal(%w[1 2], @client.call('MGET', '{key}1', '{key}2')) end def test_transaction_with_dedicated_watch_command_without_block assert_raises(::RedisClient::Cluster::Transaction::ConsistencyError) do @client.call('WATCH', '{key}1', '{key}2') end end def test_pubsub_without_subscription pubsub = @client.pubsub assert_nil(pubsub.next_event(0.01)) pubsub.close end def test_pubsub_with_wrong_command pubsub = @client.pubsub assert_nil(pubsub.call('SUBWAY')) assert_nil(pubsub.call_v(%w[SUBSCRIBE])) assert_raises(::RedisClient::CommandError, 'unknown command') { pubsub.next_event } assert_raises(::RedisClient::CommandError, 'wrong number of arguments') { pubsub.next_event } assert_nil(pubsub.next_event(0.01)) pubsub.close end def test_global_pubsub pubsub = @client.pubsub channel = 'my-global-channel' pubsub.call('SUBSCRIBE', channel) assert_equal(['subscribe', channel, 1], pubsub.next_event(TEST_TIMEOUT_SEC)) publish_messages { |cli| cli.call('PUBLISH', channel, 'hello global world') } assert_equal(['message', channel, 'hello global world'], pubsub.next_event(TEST_TIMEOUT_SEC)) pubsub.call('UNSUBSCRIBE') unless RUBY_ENGINE == 'jruby' # FIXME: too slow in jruby ensure pubsub&.close end def test_global_pubsub_without_timeout pubsub = @client.pubsub pubsub.call('SUBSCRIBE', 'my-global-not-published-channel', 'my-global-published-channel') want = [%w[subscribe my-global-not-published-channel], %w[subscribe my-global-published-channel]] got = collect_messages(pubsub, size: 2, timeout: nil).map { |e| e.take(2) }.sort_by { |e| e[1].to_s } assert_equal(want, got) publish_messages { |cli| cli.call('PUBLISH', 'my-global-published-channel', 'hello global published world') } got = collect_messages(pubsub, size: 1, timeout: nil).first assert_equal(['message', 'my-global-published-channel', 'hello global published world'], got) pubsub.call('UNSUBSCRIBE') unless RUBY_ENGINE == 'jruby' # FIXME: too slow in jruby ensure pubsub&.close end def test_global_pubsub_with_multiple_channels pubsub = @client.pubsub pubsub.call('SUBSCRIBE', *Array.new(10) { |i| "g-chan#{i}" }) got = collect_messages(pubsub, size: 10).sort_by { |e| e[1].to_s } 10.times { |i| assert_equal(['subscribe', "g-chan#{i}", i + 1], got[i]) } publish_messages { |cli| cli.pipelined { |pi| 10.times { |i| pi.call('PUBLISH', "g-chan#{i}", i) } } } got = collect_messages(pubsub, size: 10).sort_by { |e| e[1].to_s } 10.times { |i| assert_equal(['message', "g-chan#{i}", i.to_s], got[i]) } pubsub.call('UNSUBSCRIBE') unless RUBY_ENGINE == 'jruby' # FIXME: too slow in jruby ensure pubsub&.close end def test_sharded_pubsub if TEST_REDIS_MAJOR_VERSION < 7 skip('Sharded Pub/Sub is supported by Redis 7+.') return end pubsub = @client.pubsub channel = 'my-sharded-channel' pubsub.call('SSUBSCRIBE', channel) assert_equal(['ssubscribe', channel, 1], pubsub.next_event(TEST_TIMEOUT_SEC)) publish_messages { |cli| cli.call('SPUBLISH', channel, 'hello sharded world') } assert_equal(['smessage', channel, 'hello sharded world'], pubsub.next_event(TEST_TIMEOUT_SEC)) pubsub.call('SUNSUBSCRIBE') unless RUBY_ENGINE == 'jruby' # FIXME: too slow in jruby ensure pubsub&.close end def test_sharded_pubsub_without_timeout if TEST_REDIS_MAJOR_VERSION < 7 skip('Sharded Pub/Sub is supported by Redis 7+.') return end pubsub = @client.pubsub pubsub.call('SSUBSCRIBE', 'my-sharded-not-published-channel') pubsub.call('SSUBSCRIBE', 'my-sharded-published-channel') want = [%w[ssubscribe my-sharded-not-published-channel], %w[ssubscribe my-sharded-published-channel]] got = collect_messages(pubsub, size: 2, timeout: nil).map { |e| e.take(2) }.sort_by { |e| e[1].to_s } assert_equal(want, got) publish_messages { |cli| cli.call('SPUBLISH', 'my-sharded-published-channel', 'hello sharded published world') } got = collect_messages(pubsub, size: 1, timeout: nil).first assert_equal(['smessage', 'my-sharded-published-channel', 'hello sharded published world'], got) pubsub.call('SUNSUBSCRIBE') unless RUBY_ENGINE == 'jruby' # FIXME: too slow in jruby ensure pubsub&.close end def test_sharded_pubsub_with_multiple_channels if TEST_REDIS_MAJOR_VERSION < 7 skip('Sharded Pub/Sub is supported by Redis 7+.') return end pubsub = @client.pubsub 10.times { |i| pubsub.call('SSUBSCRIBE', "s-chan#{i}") } got = collect_messages(pubsub, size: 10).sort_by { |e| e[1].to_s } 10.times { |i| assert_equal(['ssubscribe', "s-chan#{i}"], got[i].take(2)) } publish_messages { |cli| cli.pipelined { |pi| 10.times { |i| pi.call('SPUBLISH', "s-chan#{i}", i) } } } got = collect_messages(pubsub, size: 10).sort_by { |e| e[1].to_s } 10.times { |i| assert_equal(['smessage', "s-chan#{i}", i.to_s], got[i]) } pubsub.call('SUNSUBSCRIBE') unless RUBY_ENGINE == 'jruby' # FIXME: too slow in jruby ensure pubsub&.close end def test_other_pubsub_commands assert_instance_of(Array, @client.call('pubsub', 'channels')) assert_instance_of(Integer, @client.call('pubsub', 'numpat')) assert_instance_of(Hash, @client.call('pubsub', 'numsub')) assert_instance_of(Array, @client.call('pubsub', 'shardchannels')) if TEST_REDIS_MAJOR_VERSION >= 7 assert_instance_of(Hash, @client.call('pubsub', 'shardnumsub')) if TEST_REDIS_MAJOR_VERSION >= 7 ps = @client.pubsub assert_nil(ps.call('unsubscribe')) assert_nil(ps.call('punsubscribe')) assert_nil(ps.call('sunsubscribe')) if TEST_REDIS_MAJOR_VERSION >= 7 ps.close end def test_stream_commands @client.call('xadd', '{stream}1', '*', 'mesage', 'foo') @client.call('xadd', '{stream}1', '*', 'mesage', 'bar') @client.call('xadd', '{stream}2', '*', 'mesage', 'baz') @client.call('xadd', '{stream}2', '*', 'mesage', 'zap') wait_for_replication consumer = new_test_client got = consumer.call('xread', 'streams', '{stream}1', '{stream}2', '0', '0') consumer.close got = got.to_h if TEST_REDIS_MAJOR_VERSION < 6 assert_equal('foo', got.fetch('{stream}1')[0][1][1]) assert_equal('bar', got.fetch('{stream}1')[1][1][1]) assert_equal('baz', got.fetch('{stream}2')[0][1][1]) assert_equal('zap', got.fetch('{stream}2')[1][1][1]) end def test_stream_group_commands @client.call('xadd', '{stream}1', '*', 'task', 'data1') @client.call('xadd', '{stream}1', '*', 'task', 'data2') @client.call('xgroup', 'create', '{stream}1', 'worker', '0') wait_for_replication consumer1 = new_test_client consumer2 = new_test_client got1 = consumer1.call('xreadgroup', 'group', 'worker', 'consumer1', 'count', '1', 'streams', '{stream}1', '>') got2 = consumer2.call('xreadgroup', 'group', 'worker', 'consumer2', 'count', '1', 'streams', '{stream}1', '>') consumer1.close consumer2.close if TEST_REDIS_MAJOR_VERSION < 6 got1 = got1.to_h got2 = got2.to_h end assert_equal('data1', got1.fetch('{stream}1')[0][1][1]) assert_equal('data2', got2.fetch('{stream}1')[0][1][1]) end def test_dedicated_multiple_keys_command [ { command: %w[MSET key1 val1], want: 'OK', wait: true }, { command: %w[MGET key1], want: %w[val1] }, { command: %w[DEL key1], want: 1, wait: true }, { command: %w[MSET {key}1 val1 {key}2 val2], want: 'OK', wait: true }, { command: %w[MGET {key}1 {key}2], want: %w[val1 val2] }, { command: %w[DEL {key}1 {key}2], want: 2, wait: true }, { command: %w[MSET key1 val1 key2 val2], want: 'OK', wait: true }, { command: %w[MGET key1 key2], want: %w[val1 val2] }, { command: %w[DEL key1 key2], want: 2, wait: true }, { command: %w[MSET key1 val1 key2 val2], block: ->(r) { "#{r}!" }, want: 'OK!', wait: true }, { command: %w[MGET key1 key2], block: ->(r) { r.map { |e| "#{e}!" } }, want: %w[val1! val2!] }, { command: %w[DEL key1 key2], block: ->(r) { r == 2 }, want: true, wait: true } ].each_with_index do |c, i| block = c.key?(:block) ? c.fetch(:block) : nil assert_equal(c.fetch(:want), @client.call_v(c.fetch(:command), &block), i + 1) wait_for_replication if c.fetch(:wait, false) end end def test_dedicated_commands 10.times { |i| @client.call('SET', "key#{i}", i) } wait_for_replication [ { command: %w[ACL HELP], is_a: Array, supported_redis_version: 6 }, { command: ['WAIT', TEST_REPLICA_SIZE, '1'], is_a: Integer }, { command: %w[KEYS *], want: (0..9).map { |i| "key#{i}" } }, { command: %w[DBSIZE], want: (0..9).size }, { command: %w[SCAN], is_a: Array }, { command: %w[LASTSAVE], is_a: Array }, { command: %w[ROLE], is_a: Array }, { command: %w[CONFIG RESETSTAT], want: 'OK' }, { command: %w[CONFIG GET maxmemory], is_a: TEST_REDIS_MAJOR_VERSION < 6 ? Array : Hash }, { command: %w[CLIENT LIST], blk: ->(r) { r.lines("\n", chomp: true).map(&:split).map { |e| Hash[e.map { |x| x.split('=') }] } }, is_a: Array }, { command: %w[CLIENT PAUSE 100], want: 'OK' }, { command: %w[CLIENT INFO], is_a: String, supported_redis_version: 6 }, { command: %w[CLUSTER SET-CONFIG-EPOCH 0], error: ::RedisClient::Cluster::OrchestrationCommandNotSupported }, { command: %w[CLUSTER SAVECONFIG], want: 'OK' }, { command: %w[CLUSTER GETKEYSINSLOT 13252 1], want: %w[key0] }, { command: %w[CLUSTER NODES], is_a: String }, { command: %w[READONLY], error: ::RedisClient::Cluster::OrchestrationCommandNotSupported }, { command: %w[MEMORY STATS], is_a: Array }, { command: %w[MEMORY PURGE], want: 'OK' }, { command: %w[MEMORY USAGE key0], is_a: Integer }, { command: %w[SCRIPT DEBUG NO], want: 'OK' }, { command: %w[SCRIPT FLUSH], want: 'OK' }, { command: %w[SCRIPT EXISTS b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c], want: [0] }, { command: %w[SCRIPT EXISTS 5b9fb3410653a731f8ddfeff39a0c061 31b6de18e43fe980ed07d8b0f5a8cabe], want: [0, 0] }, { command: %w[SCRIPT EXISTS b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c], blk: ->(reply) { reply.map { |r| !r.zero? } }, want: [false] }, { command: %w[SCRIPT EXISTS 5b9fb3410653a731f8ddfeff39a0c061 31b6de18e43fe980ed07d8b0f5a8cabe], blk: ->(reply) { reply.map { |r| !r.zero? } }, want: [false, false] }, { command: %w[PUBSUB CHANNELS test-channel*], want: [] }, { command: %w[PUBSUB NUMSUB test-channel], want: { 'test-channel' => 0 } }, { command: %w[PUBSUB NUMPAT], want: 0 }, { command: %w[PUBSUB HELP], is_a: Array }, { command: %w[MULTI], error: ::RedisClient::Cluster::AmbiguousNodeError }, { command: %w[FLUSHDB], want: 'OK' } ].each do |c| next if c.key?(:supported_redis_version) && c[:supported_redis_version] > TEST_REDIS_MAJOR_VERSION msg = "Case: #{c[:command].join(' ')}" got = -> { @client.call_v(c[:command], &c[:blk]) } if c.key?(:error) assert_raises(c[:error], msg, &got) elsif c.key?(:is_a) assert_instance_of(c[:is_a], got.call, msg) else assert_equal(c[:want], got.call, msg) end end end def test_compatibility_with_redis_gem assert_equal('OK', @client.set('foo', 100)) wait_for_replication assert_equal('100', @client.get('foo')) assert_raises(NoMethodError) { @client.densaugeo('1m') } end def test_compatibility_with_ring assert_equal([@client], @client.nodes) assert_equal(@client, @client.node_for('key')) assert_equal({ @client => %w[a b c] }, @client.nodes_for('a', 'b', ['c'])) end def test_compatibility_with_pooled count = 0 @client.with { count += 1 } @client.with(timeout: 2) { count += 1 } @client.then { count += 1 } assert_equal 3, count end def test_circuit_breakers cli = ::RedisClient.cluster( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, # This option is important - need to make sure that the reloads happen on different connections # to the timeouts, so that they don't count against the circuit breaker (they'll have their own breakers). connect_with_original_config: true, **TEST_GENERIC_OPTIONS.merge( circuit_breaker: { # Also important - the retry_count on resharding errors is set to 3, so we have to allow at lest # that many errors to avoid tripping the breaker in the first call. error_threshold: 4, error_timeout: 60, success_threshold: 10 } ) ).new_client cli.call('echo', 'init') swap_timeout(cli, timeout: 0.1) do |c| assert_raises(::RedisClient::ReadTimeoutError) { c.blocking_call(0.1, 'BRPOP', 'foo', 0) } assert_raises(::RedisClient::CircuitBreaker::OpenCircuitError) { c.blocking_call(0.1, 'BRPOP', 'foo', 0) } end cli&.close end def test_only_reshards_own_errors slot = ::RedisClient::Cluster::KeySlotConverter.convert('testkey') router = @client.instance_variable_get(:@router) correct_primary_key = router.find_node_key_by_key('testkey', primary: true) broken_primary_key = (router.node_keys - [correct_primary_key]).first client1 = new_test_client( middlewares: [ ::RedisClient::Cluster::ErrorIdentification::Middleware ] ) client2 = new_test_client( middlewares: [ ::RedisClient::Cluster::ErrorIdentification::Middleware, ::Middlewares::RedirectFake ], custom: { redirect_fake: ::Middlewares::RedirectFake::Setting.new( slot: slot, to: broken_primary_key, command: %w[set testkey client2] ) } ) assert_raises(RedisClient::CommandError) do client1.call('set', 'testkey', 'client1') do |got| assert_equal('OK', got) client2.call('set', 'testkey', 'client2') end end # The exception should not have causes client1 to update its shard mappings, because it didn't # come from a RedisClient instance that client1 knows about. assert_equal( correct_primary_key, client1.instance_variable_get(:@router).find_node_key_by_key('testkey', primary: true) ) client1.close client2.close end def test_initialization_delayed config = ::RedisClient::ClusterConfig.new(nodes: 'redis://127.0.0.1:11211') client = ::RedisClient::Cluster.new(config) assert_instance_of(::RedisClient::Cluster, client) assert_raises(RedisClient::Cluster::InitialSetupError) { client.call('PING') } end private def wait_for_replication client_side_timeout = TEST_TIMEOUT_SEC + 1.0 server_side_timeout = (TEST_TIMEOUT_SEC * 1000).to_i swap_timeout(@client, timeout: 0.1) do |client| client&.blocking_call(client_side_timeout, 'WAIT', TEST_REPLICA_SIZE, server_side_timeout) rescue RedisClient::Cluster::ErrorCollection => e # FIXME: flaky in jruby on #test_pubsub_with_wrong_command raise unless e.errors.values.all? { |err| err.is_a?(::RedisClient::ConnectionError) } end end def collect_messages(pubsub, size:, max_attempts: 30, timeout: 1.0) messages = [] attempts = 0 loop do attempts += 1 break if attempts > max_attempts reply = pubsub.next_event(timeout) break if reply.nil? messages << reply break messages if messages.size == size end end def publish_messages client = new_test_client(middlewares: nil) yield client client.close end def hiredis_used? ::RedisClient.const_defined?(:HiredisConnection) && ::RedisClient.default_driver == ::RedisClient::HiredisConnection end end class PrimaryOnly < TestingWrapper include Mixin def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) config = ::RedisClient::ClusterConfig.new( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, slow_command_timeout: TEST_TIMEOUT_SEC, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ) ::RedisClient::Cluster.new(config) end end class ScaleReadRandom < TestingWrapper include Mixin def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) config = ::RedisClient::ClusterConfig.new( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random, fixed_hostname: TEST_FIXED_HOSTNAME, slow_command_timeout: TEST_TIMEOUT_SEC, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ) ::RedisClient::Cluster.new(config) end end class ScaleReadRandomWithPrimary < TestingWrapper include Mixin def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) config = ::RedisClient::ClusterConfig.new( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random_with_primary, fixed_hostname: TEST_FIXED_HOSTNAME, slow_command_timeout: TEST_TIMEOUT_SEC, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ) ::RedisClient::Cluster.new(config) end end class ScaleReadLatency < TestingWrapper include Mixin def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) config = ::RedisClient::ClusterConfig.new( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :latency, fixed_hostname: TEST_FIXED_HOSTNAME, slow_command_timeout: TEST_TIMEOUT_SEC, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ) ::RedisClient::Cluster.new(config) end end class Pooled < TestingWrapper include Mixin def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) config = ::RedisClient::ClusterConfig.new( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, slow_command_timeout: TEST_TIMEOUT_SEC, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ) ::RedisClient::Cluster.new(config, pool: { timeout: TEST_TIMEOUT_SEC, size: 2 }) end end end end redis-rb-redis-cluster-client-aaf9e2e/test/redis_client/test_cluster_config.rb000066400000000000000000000234121517214044400300570ustar00rootroot00000000000000# frozen_string_literal: true require 'uri' require 'testing_helper' class RedisClient class TestClusterConfig < TestingWrapper def test_inspect cfg = { host: '127.0.0.1', port: 6379 } want = "#" got = ::RedisClient::ClusterConfig.new.inspect assert_equal(want, got) end def test_new_pool assert_instance_of( ::RedisClient::Cluster, ::RedisClient::ClusterConfig.new( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_pool ) end def test_new_client assert_instance_of( ::RedisClient::Cluster, ::RedisClient::ClusterConfig.new( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_client ) end def test_startup_nodes [ { config: ::RedisClient::ClusterConfig.new, want: { '127.0.0.1:6379' => { host: '127.0.0.1', port: 6379, command_builder: ::RedisClient::Cluster::NoopCommandBuilder } } }, { config: ::RedisClient::ClusterConfig.new(replica: true), want: { '127.0.0.1:6379' => { host: '127.0.0.1', port: 6379, command_builder: ::RedisClient::Cluster::NoopCommandBuilder } } }, { config: ::RedisClient::ClusterConfig.new(fixed_hostname: 'endpoint.example.com'), want: { '127.0.0.1:6379' => { host: 'endpoint.example.com', port: 6379, command_builder: ::RedisClient::Cluster::NoopCommandBuilder } } }, { config: ::RedisClient::ClusterConfig.new(timeout: 1), want: { '127.0.0.1:6379' => { host: '127.0.0.1', port: 6379, timeout: 1, command_builder: ::RedisClient::Cluster::NoopCommandBuilder } } }, { config: ::RedisClient::ClusterConfig.new(db: 1), want: { '127.0.0.1:6379' => { host: '127.0.0.1', port: 6379, db: 1, command_builder: ::RedisClient::Cluster::NoopCommandBuilder } } }, { config: ::RedisClient::ClusterConfig.new(nodes: ['redis://1.2.3.4:1234/0', 'rediss://5.6.7.8:5678/1']), want: { '1.2.3.4:1234' => { host: '1.2.3.4', port: 1234, db: 0, command_builder: ::RedisClient::Cluster::NoopCommandBuilder }, '5.6.7.8:5678' => { host: '5.6.7.8', port: 5678, db: 1, ssl: true, command_builder: ::RedisClient::Cluster::NoopCommandBuilder } } }, { config: ::RedisClient::ClusterConfig.new(custom: { foo: 'bar' }), want: { '127.0.0.1:6379' => { host: '127.0.0.1', port: 6379, custom: { foo: 'bar' }, command_builder: ::RedisClient::Cluster::NoopCommandBuilder } } } ].each_with_index do |c, idx| assert_equal(c[:want], c[:config].startup_nodes, "Case: #{idx}") end end def test_use_replica? assert_predicate(::RedisClient::ClusterConfig.new(replica: true), :use_replica?) refute_predicate(::RedisClient::ClusterConfig.new(replica: false), :use_replica?) refute_predicate(::RedisClient::ClusterConfig.new, :use_replica?) end def test_replica_affinity [ { value: :random, want: :random }, { value: 'random', want: :random }, { value: :latency, want: :latency }, { value: 'latency', want: :latency }, { value: :unknown, want: :unknown }, { value: 'unknown', want: :unknown }, { value: 0, want: :'0' }, { value: nil, want: :'' } ].each do |c| cfg = ::RedisClient::ClusterConfig.new(replica_affinity: c[:value]) assert_equal(c[:want], cfg.replica_affinity) end end def test_command_builder assert_equal(::RedisClient::CommandBuilder, ::RedisClient::ClusterConfig.new.command_builder) end def test_concurrency [ { value: nil, want: { model: :none } }, { value: { model: :none }, want: { model: :none } }, { value: { model: :on_demand, size: 3 }, want: { model: :on_demand, size: 3 } }, { value: { model: :pooled, size: 6 }, want: { model: :pooled, size: 6 } } ].each do |c| cfg = ::RedisClient::ClusterConfig.new(concurrency: c[:value]) assert_equal(c[:want], cfg.instance_variable_get(:@concurrency)) end end def test_build_node_configs config = ::RedisClient::ClusterConfig.new [ { addrs: %w[redis://127.0.0.1], want: [{ host: '127.0.0.1', port: 6379 }] }, { addrs: %w[redis://127.0.0.1:6379], want: [{ host: '127.0.0.1', port: 6379 }] }, { addrs: %w[redis://127.0.0.1:6379/1], want: [{ host: '127.0.0.1', port: 6379, db: 1 }] }, { addrs: %w[redis://127.0.0.1:6379 redis://127.0.0.2:6380], want: [{ host: '127.0.0.1', port: 6379 }, { host: '127.0.0.2', port: 6380 }] }, { addrs: %w[rediss://foo:bar@127.0.0.1:6379], want: [{ ssl: true, username: 'foo', password: 'bar', host: '127.0.0.1', port: 6379 }] }, { addrs: %w[redis://foo@127.0.0.1:6379], want: [{ host: '127.0.0.1', port: 6379, username: 'foo' }] }, { addrs: %w[redis://foo:@127.0.0.1:6379], want: [{ host: '127.0.0.1', port: 6379, username: 'foo' }] }, { addrs: %w[redis://:bar@127.0.0.1:6379], want: [{ host: '127.0.0.1', port: 6379, password: 'bar' }] }, { addrs: %W[redis://#{URI.encode_www_form_component('!&<123-abc>')}:@127.0.0.1:6379], want: [{ host: '127.0.0.1', port: 6379, username: '!&<123-abc>' }] }, { addrs: %W[redis://:#{URI.encode_www_form_component('!&<123-abc>')}@127.0.0.1:6379], want: [{ host: '127.0.0.1', port: 6379, password: '!&<123-abc>' }] }, { addrs: [{ host: '127.0.0.1', port: 6379 }], want: [{ host: '127.0.0.1', port: 6379 }] }, { addrs: [{ host: '127.0.0.1', port: 6379 }, { host: '127.0.0.2', port: '6380' }], want: [{ host: '127.0.0.1', port: 6379 }, { host: '127.0.0.2', port: 6380 }] }, { addrs: [{ host: '127.0.0.1', port: 6379, username: 'foo', password: 'bar', ssl: true }], want: [{ ssl: true, username: 'foo', password: 'bar', host: '127.0.0.1', port: 6379 }] }, { addrs: [{ host: '127.0.0.1', port: 6379, db: 1 }], want: [{ host: '127.0.0.1', port: 6379, db: 1 }] }, { addrs: 'redis://127.0.0.1:6379', want: [{ host: '127.0.0.1', port: 6379 }] }, { addrs: { host: '127.0.0.1', port: 6379 }, want: [{ host: '127.0.0.1', port: 6379 }] }, { addrs: [{ host: '127.0.0.1' }], want: [{ host: '127.0.0.1', port: 6379 }] }, { addrs: %w[http://127.0.0.1:80], error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: [{ host: '127.0.0.1', port: 'foo' }], error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: %w[redis://127.0.0.1:foo], error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: [6379], error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: ['foo'], error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: [''], error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: [{}], error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: [], error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: {}, error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: '', error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { addrs: nil, error: ::RedisClient::ClusterConfig::InvalidClientConfigError } ].each_with_index do |c, idx| msg = "Case: #{idx}: #{c}" got = -> { config.send(:build_node_configs, c[:addrs]) } if c.key?(:error) assert_raises(c[:error], msg, &got) else assert_equal(c.fetch(:want), got.call, msg) end end end def test_merge_generic_config config = ::RedisClient::ClusterConfig.new [ { params: { client_config: { ssl: false, username: 'foo', password: 'bar', timeout: 1 }, node_configs: [{ ssl: true, username: 'baz', password: 'zap', host: '127.0.0.1' }] }, want: { ssl: true, username: 'baz', password: 'zap', timeout: 1 } }, { params: { client_config: { ssl: false, timeout: 1 }, node_configs: [{ ssl: true, host: '127.0.0.1' }] }, want: { ssl: true, timeout: 1 } }, { params: { client_config: { timeout: 1 }, node_configs: [{ ssl: true }] }, want: { ssl: true, timeout: 1 } }, { params: { client_config: { db: 1 }, node_configs: [{ db: 2 }] }, want: { db: 2 } }, { params: { client_config: {}, node_configs: [], keys: [] }, want: {} } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = config.send(:merge_generic_config, c[:params][:client_config], c[:params][:node_configs]) assert_equal(c[:want], got, msg) end end def test_client_config_for_node config = ::RedisClient::ClusterConfig.new( nodes: ['rediss://username:password@1.2.3.4:1234/0', 'redis://5.6.7.8:5678/1'], custom: { foo: 'bar' } ) want = { host: '9.9.9.9', port: 9999, username: 'username', password: 'password', ssl: true, db: 0, custom: { foo: 'bar' }, command_builder: ::RedisClient::Cluster::NoopCommandBuilder } assert_equal(want, config.client_config_for_node('9.9.9.9:9999')) end def test_client_config_id assert_equal('foo-cluster', ::RedisClient::ClusterConfig.new(id: 'foo-cluster').id) assert_nil(::RedisClient::ClusterConfig.new.id) end end end redis-rb-redis-cluster-client-aaf9e2e/test/ssl_certs/000077500000000000000000000000001517214044400230205ustar00rootroot00000000000000redis-rb-redis-cluster-client-aaf9e2e/test/ssl_certs/redis-rb-ca.crt000066400000000000000000000027351517214044400256310ustar00rootroot00000000000000-----BEGIN CERTIFICATE----- MIIEKDCCAxCgAwIBAgIUQ4fcBhPuBH8/dhxQYWk3VGtV6jUwDQYJKoZIhvcNAQEL BQAwdzELMAkGA1UEBhMCVVMxEDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0 bGFudGExETAPBgNVBAoMCHJlZGlzLXJiMR0wGwYDVQQLDBRyZWRpcy1jbHVzdGVy LWNsaWVudDESMBAGA1UEAwwJMTI3LjAuMC4xMCAXDTIyMTAwODAzMTcwOVoYDzIz MjIwNzI4MDMxNzA5WjB3MQswCQYDVQQGEwJVUzEQMA4GA1UECAwHR2VvcmdpYTEQ MA4GA1UEBwwHQXRsYW50YTERMA8GA1UECgwIcmVkaXMtcmIxHTAbBgNVBAsMFHJl ZGlzLWNsdXN0ZXItY2xpZW50MRIwEAYDVQQDDAkxMjcuMC4wLjEwggEiMA0GCSqG SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCxkiVbVfVVyJtGwxQbzvfXMJ+J3VRZCqIS /S5UshqkmThn2e4ZvrQYJqEax55KSQqW87wTMZg83s+tDM4mPbDgk5Xg6P2fvfQH TvrF5kVDmrAjq+qvVUoijPrnoTiWXNv+h1J13UJjRhVo3co3W7jSZQvIsPwOs7Ir LBXor3EwQjNoFLI+T9vDfVW47zXz9eHwuPt5um4XM9ODpR8jWSUdYrI5vEjNlbUn 7AR/oSarRLKEyvZaXPHfT0yNARSUmoYSS2EOnhGtOV3nCQEQZKv+hETFPKkHoKor L/6DKuaAt5lNOz1ykq4EISfYKh/8OxxJ8QXH/Nj9gpdHJnAZjgy/AgMBAAGjgakw gaYwHQYDVR0OBBYEFAtY+UXH5hu9LpScPBHNy5SHU2JwMB8GA1UdIwQYMBaAFAtY +UXH5hu9LpScPBHNy5SHU2JwMA8GA1UdEwEB/wQFMAMBAf8wUwYDVR0RBEwwSoIJ bG9jYWxob3N0ggVub2RlMYIFbm9kZTKCBW5vZGUzggVub2RlNIIFbm9kZTWCBW5v ZGU2ggVub2RlN4IFbm9kZTiCBW5vZGU5MA0GCSqGSIb3DQEBCwUAA4IBAQAlsNBL jMp1QSU/S4ot3pFUJTZ2iJohcTJlR5yvWqikMVve2kwWtIlDglmg+JSlZ3DqBdOR AF6tvrXBNF4XlPFmB3C2dS3NWqaOwT0J20GVs5+jyF2cqZUWx+5szg+5afVlBVIX CNyGFUwJrIq6nVaCEclg8f7nIp9WycYRlHrkF3euntDZY5cj5f2/fywTtRiWYd3N ez+BWa1z/PFJ9JyVgi4aet5cqlEpHPV1bkIsi8HObgfFgg7pR1fE2VsW0nhIYzVi NOhl9jTKB1TYz0IM9VEogqhlEQHj0JLPoZDgZ8AGvPwt7LkxtgPQ0yOc/ZyHoaaU 2igVxvCoy5Xjcrhl -----END CERTIFICATE----- redis-rb-redis-cluster-client-aaf9e2e/test/ssl_certs/redis-rb-cert.crt000066400000000000000000000110241517214044400261720ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 43:87:dc:06:13:ee:04:7f:3f:76:1c:50:61:69:37:54:6b:55:ea:36 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=Georgia, L=Atlanta, O=redis-rb, OU=redis-cluster-client, CN=127.0.0.1 Validity Not Before: Oct 8 03:17:10 2022 GMT Not After : Jul 28 03:17:10 2322 GMT Subject: C=US, ST=Georgia, O=redis-rb, OU=redis-cluster-client, CN=127.0.0.1 Subject Public Key Info: Public Key Algorithm: rsaEncryption RSA Public-Key: (2048 bit) Modulus: 00:d6:4f:ad:73:24:2f:c8:0d:19:6b:39:a4:f3:9a: 8a:c6:8e:cf:6c:bf:2a:63:90:ee:ed:52:3b:fe:41: eb:dd:04:99:bc:39:35:66:b6:70:66:d7:d7:55:1d: 3d:71:33:a1:3a:d6:e5:45:51:43:36:da:bc:62:99: 85:38:07:c4:20:97:58:6f:39:1c:9e:a2:3b:e9:ae: 5f:b5:79:18:53:9a:dd:b5:9a:2a:9a:8f:e1:76:2c: a7:2f:a0:1e:59:8d:99:2a:d2:92:e3:a8:77:d1:4f: 50:14:dd:43:e4:cf:3c:84:b6:ad:01:2f:b0:ff:b2: af:08:56:73:39:09:48:7e:b4:a8:41:c4:e5:a3:0b: 0b:1f:a6:ab:27:cd:b5:6c:5a:a8:45:ed:e5:6e:76: 49:58:9c:ce:77:fb:81:6a:37:91:67:8c:78:b6:ef: 73:29:da:bb:c2:19:84:97:2b:c4:ed:b5:52:b9:47: 64:70:9c:ac:af:5b:20:e3:1c:da:2c:ca:9c:26:32: af:72:06:4d:d0:a3:6e:3d:48:e5:00:08:68:0f:1b: 81:74:16:fe:d1:32:78:c8:16:56:04:88:c1:3f:da: a5:a9:3b:13:2a:c8:ae:ee:c0:ef:ab:45:a7:f7:1a: 6b:e0:fb:2a:fa:40:98:75:a1:04:10:97:40:65:54: 15:09 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Comment: OpenSSL Generated Certificate X509v3 Subject Key Identifier: 02:53:41:2F:20:D1:67:C8:7C:64:B8:BE:3E:0A:E7:9C:CC:D3:F2:1C X509v3 Authority Key Identifier: keyid:0B:58:F9:45:C7:E6:1B:BD:2E:94:9C:3C:11:CD:CB:94:87:53:62:70 Signature Algorithm: sha256WithRSAEncryption 04:88:63:a4:0d:93:df:2d:a3:56:e2:e6:31:2a:53:6b:bc:dc: d2:b5:1c:81:e6:c9:dc:44:78:27:9e:76:b2:60:bc:ae:96:6f: 33:19:df:08:24:fb:cd:ae:6a:e0:56:63:dd:a8:ed:c9:93:7e: ae:9a:14:16:e9:16:20:fe:4f:00:82:6c:32:39:95:58:8d:22: 97:a9:76:79:45:aa:90:8e:c8:78:08:92:eb:ab:14:c9:5a:b0: 04:c7:81:00:81:2c:f0:1c:f7:54:e4:d5:db:b3:b9:bf:a2:6a: 70:45:8b:2d:f1:fb:20:d6:ef:ea:80:15:d8:db:07:65:7d:f6: 69:dc:cf:21:00:24:01:40:10:b2:c2:4f:c4:12:70:50:d3:18: 37:89:fc:80:99:21:20:bb:1a:6c:44:f1:f2:31:03:4c:5d:b8: 1a:ec:2e:43:53:9b:8a:af:c1:7c:08:e8:a2:6a:44:37:d8:78: 6f:f5:14:4e:1c:16:66:b3:69:ec:aa:4c:d3:8c:55:b7:0c:a2: 69:c2:cc:0c:d0:27:f1:14:d4:16:51:8f:c4:f1:74:47:fc:d7: 1d:d0:03:41:7a:cb:c2:c7:99:9c:ca:4b:3f:ac:4a:7d:77:3b: b9:bb:7b:73:3f:0f:1b:e6:52:c2:d8:3f:88:be:d3:42:5f:f5: cc:b5:99:f5 -----BEGIN CERTIFICATE----- MIID5zCCAs+gAwIBAgIUQ4fcBhPuBH8/dhxQYWk3VGtV6jYwDQYJKoZIhvcNAQEL BQAwdzELMAkGA1UEBhMCVVMxEDAOBgNVBAgMB0dlb3JnaWExEDAOBgNVBAcMB0F0 bGFudGExETAPBgNVBAoMCHJlZGlzLXJiMR0wGwYDVQQLDBRyZWRpcy1jbHVzdGVy LWNsaWVudDESMBAGA1UEAwwJMTI3LjAuMC4xMCAXDTIyMTAwODAzMTcxMFoYDzIz MjIwNzI4MDMxNzEwWjBlMQswCQYDVQQGEwJVUzEQMA4GA1UECAwHR2VvcmdpYTER MA8GA1UECgwIcmVkaXMtcmIxHTAbBgNVBAsMFHJlZGlzLWNsdXN0ZXItY2xpZW50 MRIwEAYDVQQDDAkxMjcuMC4wLjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQDWT61zJC/IDRlrOaTzmorGjs9svypjkO7tUjv+QevdBJm8OTVmtnBm19dV HT1xM6E61uVFUUM22rximYU4B8Qgl1hvORyeojvprl+1eRhTmt21miqaj+F2LKcv oB5ZjZkq0pLjqHfRT1AU3UPkzzyEtq0BL7D/sq8IVnM5CUh+tKhBxOWjCwsfpqsn zbVsWqhF7eVudklYnM53+4FqN5FnjHi273Mp2rvCGYSXK8TttVK5R2RwnKyvWyDj HNosypwmMq9yBk3Qo249SOUACGgPG4F0Fv7RMnjIFlYEiME/2qWpOxMqyK7uwO+r Raf3Gmvg+yr6QJh1oQQQl0BlVBUJAgMBAAGjezB5MAkGA1UdEwQCMAAwLAYJYIZI AYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQW BBQCU0EvINFnyHxkuL4+CueczNPyHDAfBgNVHSMEGDAWgBQLWPlFx+YbvS6UnDwR zcuUh1NicDANBgkqhkiG9w0BAQsFAAOCAQEABIhjpA2T3y2jVuLmMSpTa7zc0rUc gebJ3ER4J552smC8rpZvMxnfCCT7za5q4FZj3ajtyZN+rpoUFukWIP5PAIJsMjmV WI0il6l2eUWqkI7IeAiS66sUyVqwBMeBAIEs8Bz3VOTV27O5v6JqcEWLLfH7INbv 6oAV2NsHZX32adzPIQAkAUAQssJPxBJwUNMYN4n8gJkhILsabETx8jEDTF24Guwu Q1Obiq/BfAjoompEN9h4b/UUThwWZrNp7KpM04xVtwyiacLMDNAn8RTUFlGPxPF0 R/zXHdADQXrLwseZnMpLP6xKfXc7ubt7cz8PG+ZSwtg/iL7TQl/1zLWZ9Q== -----END CERTIFICATE----- redis-rb-redis-cluster-client-aaf9e2e/test/ssl_certs/redis-rb-cert.key000066400000000000000000000032541517214044400262000ustar00rootroot00000000000000-----BEGIN PRIVATE KEY----- MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDWT61zJC/IDRlr OaTzmorGjs9svypjkO7tUjv+QevdBJm8OTVmtnBm19dVHT1xM6E61uVFUUM22rxi mYU4B8Qgl1hvORyeojvprl+1eRhTmt21miqaj+F2LKcvoB5ZjZkq0pLjqHfRT1AU 3UPkzzyEtq0BL7D/sq8IVnM5CUh+tKhBxOWjCwsfpqsnzbVsWqhF7eVudklYnM53 +4FqN5FnjHi273Mp2rvCGYSXK8TttVK5R2RwnKyvWyDjHNosypwmMq9yBk3Qo249 SOUACGgPG4F0Fv7RMnjIFlYEiME/2qWpOxMqyK7uwO+rRaf3Gmvg+yr6QJh1oQQQ l0BlVBUJAgMBAAECggEBANKGRNH1+0YesBin8MUozCiPQ24FQGO8jSjufmafZU6h ZVAENtQmZbZxU2OWLLRWTozZazGzyT5Kk4KcYsSOxHhrlAD2bonavtYoaHaBdYcz e5YY0r8wlf+bj8R2GzpHoe3yGz+uT716lyVX0okjlsScGskui0YzxkN/gQLHfsKP cf2nPDm3u2ArrCjHYQT7YjEvCU5fAVGSHSuvVgHD+PIckE3irLaWRgCiM5uKtXce l20ju3zFYQMyt3rEmndKBr8AwUFW2Ku7CYgnP89rRVt3XBSVGhVly4MH6A6xRnA1 O2uF56trS82zCXRHCkBURng6eipzbJF3B5tNKBk3I+0CgYEA+m9mgicKfMTd5cyr TiTY/wsTjnTU6ZCexPb5dbtUzvBxezot4YfrSPF30jws8TivIVMdH6jYmNzAgJUN z+rZrujX089uW97zHW1QaWzOmH7lsK9hOgMR2HVz6GjFlZciJ1emXuPVP5bi2zpl oviiFMZLSO7qW2nhLtKJ3eKmxSMCgYEA2xLJTMuA6FmedMiISvk3bIn9x6h/YXT9 eTcXeNQR1YIWHANEhN361a+TFxmR0QYRapPamhjasT+EIPL8WY1slRHWrwLRvSJv 7YwmLiSZwWmEOJPwKQqmR3Yyvb9aslsAknPSILCZ9bxC+UfO7DIFzquQt8x6jy4R VokQpNcVjeMCgYEA2Qp/DsGDJ0r+/M/6jwkEP1V8J3Q9qga6cv2QiWZHQ+nCkAeG B/XiBh+vtraMRKrZrMn5bZzJywFWnJmRlOZ2rk4B7wHRJTH+BTzd+eBg1Gz158C3 RK2wY6a3Q265/sEyymH+QDK4eBnulgzwVOOipNqOGLFmzr7ed9PjxDdQTX0CgYEA weEOZfh0TS2DHreaZz/H3TcCcgCdOxLegLhQ/Y4xelN2XbRGn5AUvah09Kycb/B+ 2WOgw1/bq6IavU5OJrMStZrj9F76X/hqNkEiSRP7P0Cy05+Zm7jhD717ipIfIlmH WBVIkcW5e9DxNMxoRIDAwvbzTLaagLy0e3EyWbBAUyECgYAoXjATX0vwfokPmYV9 z69VjGhxruQ1j2Uh6AaJ6COn61Cnh8Y0cT67yXgRA2uuYChBoyL0jvLby70K2rvv EBtJvC9WWVIZeacTytuNQhozLm18AmfeUTMxegrxeUwm/nxJZfqXkPb8wcZw79N9 taQ0a7+rmJRu0Fja03y7VROy8Q== -----END PRIVATE KEY----- redis-rb-redis-cluster-client-aaf9e2e/test/test_against_cluster_broken.rb000066400000000000000000000225001517214044400271310ustar00rootroot00000000000000# frozen_string_literal: true require 'logger' require 'json' require 'testing_helper' require 'securerandom' class TestAgainstClusterBroken < TestingWrapper WAIT_SEC = 0.1 MAX_ATTEMPTS = 300 NUMBER_OF_KEYS = 16_000 MAX_PIPELINE_SIZE = 40 HASH_TAG_GRAIN = 5 SLICED_NUMBERS = (0...NUMBER_OF_KEYS).each_slice(MAX_PIPELINE_SIZE).freeze def setup @controller = build_controller @captured_commands = ::Middlewares::CommandCapture::CommandBuffer.new @redirect_count = ::Middlewares::RedirectCount::Counter.new @cluster_down_error_count = 0 @logger = Logger.new($stdout) print "\n" @logger.info('setup: test') prepare_test_data @clients = Array.new(3) { build_client.tap { |c| c.call('echo', 'init') } } end def teardown @logger.info('teardown: test') revive_dead_nodes @clients.each(&:close) @controller&.close end def test_client_patience do_manual_failover wait_for_cluster_to_be_ready do_assertions(offset: 0) # a replica sacrifice_replica = @controller.select_sacrifice_of_replica kill_a_node(sacrifice_replica) wait_for_cluster_to_be_ready(ignore: [sacrifice_replica]) do_assertions(offset: 1) # a primary sacrifice_primary = @controller.select_sacrifice_of_primary kill_a_node(sacrifice_primary) wait_for_cluster_to_be_ready(ignore: [sacrifice_replica, sacrifice_primary]) do_assertions(offset: 2) # recovery revive_dead_nodes wait_for_cluster_to_be_ready do_assertions(offset: 3) end def test_reloading_on_connection_error sacrifice = @controller.select_sacrifice_of_primary # Find a key which lives on the sacrifice node test_key = generate_key_for_node(sacrifice) @clients[0].call('SET', test_key, 'foobar1') # Shut the node down. kill_a_node_and_wait_for_failover(sacrifice) # When we try and fetch the key, it'll attempt to connect to the broken node, and # thus trigger a reload of the cluster topology. assert_equal 'OK', @clients[0].call('SET', test_key, 'foobar2') end def test_transaction_retry_on_connection_error sacrifice = @controller.select_sacrifice_of_primary # Find a key which lives on the sacrifice node test_key = generate_key_for_node(sacrifice) @clients[0].call('SET', test_key, 'foobar1') call_count = 0 # Begin a transaction, but shut the node down after the WATCH is issued res = @clients[0].multi(watch: [test_key]) do |tx| kill_a_node_and_wait_for_failover(sacrifice) if call_count == 0 call_count += 1 tx.call('SET', test_key, 'foobar2') end # The transaction should have retried once and successfully completed # the second time. assert_equal ['OK'], res assert_equal 'foobar2', @clients[0].call('GET', test_key) assert_equal 2, call_count end private def prepare_test_data client = build_client(custom: nil, middlewares: nil) client.call('FLUSHDB') SLICED_NUMBERS.each do |numbers| client.pipelined do |pi| numbers.each do |i| pi.call('SET', "single:#{i}", i) pi.call('SET', "pipeline:#{i}", i) pi.call('SET', "{group#{i / HASH_TAG_GRAIN}}:transaction:#{i}", i) end end end wait_for_replication(client) client.close end def do_assertions(offset:) @captured_commands.clear @redirect_count.clear @cluster_down_error_count = 0 log_info('assertions') do log_info('assertions: single') do NUMBER_OF_KEYS.times do |i| want = (i + offset).to_s got = retryable { @clients[0].call_once('GET', "single:#{i}") } assert_equal(want, got, 'Case: Single GET') want = 'OK' got = retryable { @clients[0].call_once('SET', "single:#{i}", i + offset + 1) } assert_equal(want, got, 'Case: Single SET') end end log_info('assertions: pipeline') do SLICED_NUMBERS.each do |numbers| want = numbers.map { |i| (i + offset).to_s } got = retryable do @clients[1].pipelined do |pi| numbers.each { |i| pi.call('GET', "pipeline:#{i}") } end end assert_equal(want, got, 'Case: Pipeline GET') want = numbers.map { 'OK' } got = retryable do @clients[1].pipelined do |pi| numbers.each { |i| pi.call('SET', "pipeline:#{i}", i + offset + 1) } end end assert_equal(want, got, 'Case: Pipeline SET') end end log_info('assertions: transaction') do NUMBER_OF_KEYS.times.group_by { |i| i / HASH_TAG_GRAIN }.each do |group, numbers| want = numbers.map { 'OK' } keys = numbers.map { |i| "{group#{group}}:transaction:#{i}" } got = retryable do @clients[2].multi(watch: group.odd? ? nil : keys) do |tx| keys.each_with_index { |key, i| tx.call('SET', key, numbers[i] + offset + 1) } end end assert_equal(want, got, 'Case: Transaction: SET') end end log_metrics end end def generate_key_for_node(conn) # Figure out a slot on the the sacrifice node, and a key in that slot. conn_id = conn.call('CLUSTER', 'MYID') conn_slots = conn.call('CLUSTER', 'SLOTS') .select { |res| res[2][2] == conn_id } .flat_map { |res| (res[0]..res[1]).to_a } loop do test_key = SecureRandom.hex return test_key if conn_slots.include?(conn.call('CLUSTER', 'KEYSLOT', test_key)) end end def wait_for_replication(client) client_side_timeout = TEST_TIMEOUT_SEC + 1.0 server_side_timeout = (TEST_TIMEOUT_SEC * 1000).to_i swap_timeout(client, timeout: 0.1) do |c| c.blocking_call(client_side_timeout, 'WAIT', TEST_REPLICA_SIZE, server_side_timeout) end end def wait_for_cluster_to_be_ready(ignore: []) log_info('wait for the cluster to be stable') do @controller.wait_for_cluster_to_be_ready(skip_clients: ignore) end end def do_manual_failover log_info('failover') do @controller.failover end end def kill_a_node(sacrifice) log_info("kill #{sacrifice.config.host}:#{sacrifice.config.port}") do refute_nil(sacrifice, "#{sacrifice.config.host}:#{sacrifice.config.port}") `docker compose ps --format json`.lines.map { |line| JSON.parse(line) }.each do |service| published_ports = service.fetch('Publishers').map { |e| e.fetch('PublishedPort') }.uniq next unless published_ports.include?(sacrifice.config.port) service_name = service.fetch('Service') system("docker compose --progress quiet pause #{service_name}", exception: true) break end assert_raises(::RedisClient::ConnectionError) { sacrifice.call_once('PING') } end end def revive_dead_nodes log_info('revive dead nodes') do `docker compose ps --format json --status paused`.lines.map { |line| JSON.parse(line) }.each do |service| service_name = service.fetch('Service') system("docker compose --progress quiet unpause #{service_name}", exception: true) end end end def log_info(message) @logger.info("start: #{message}") yield @logger.info(" done: #{message}") end def log_metrics print "#{@redirect_count.get}, "\ "ClusterShardsCall: #{@captured_commands.count('cluster', 'shards')}, "\ "ClusterDownError: #{@cluster_down_error_count}\n" end def retryable(attempts: MAX_ATTEMPTS, wait_sec: WAIT_SEC) loop do raise MaxRetryExceeded if attempts <= 0 attempts -= 1 break yield rescue ::RedisClient::ConnectionError, ::RedisClient::Cluster::NodeMightBeDown @cluster_down_error_count += 1 sleep wait_sec rescue ::RedisClient::CommandError => e raise unless e.message.start_with?('CLUSTERDOWN') @cluster_down_error_count += 1 sleep wait_sec rescue ::RedisClient::Cluster::ErrorCollection => e raise unless e.errors.values.all? do |err| err.message.start_with?('CLUSTERDOWN') || err.is_a?(::RedisClient::ConnectionError) end @cluster_down_error_count += 1 sleep wait_sec end end def kill_a_node_and_wait_for_failover(sacrifice) other_client = @controller.clients.reject { _1 == sacrifice }.first sacrifice_id = sacrifice.call('CLUSTER', 'MYID') kill_a_node(sacrifice) failover_checks = 0 loop do raise 'Timed out waiting for failover in kill_a_node_and_wait_for_failover' if failover_checks > 30 # Wait for the sacrifice node to not be a primary according to CLUSTER SLOTS. cluster_slots = other_client.call('CLUSTER', 'SLOTS') break unless cluster_slots.any? { _1[2][2] == sacrifice_id } sleep 1 failover_checks += 1 end end def build_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, fixed_hostname: TEST_FIXED_HOSTNAME, custom: custom, middlewares: middlewares, **TEST_GENERIC_OPTIONS.merge(timeout: 0.1), **opts ).new_client end def build_controller ClusterController.new( TEST_NODE_URIS, replica_size: TEST_REPLICA_SIZE, **TEST_GENERIC_OPTIONS.merge(timeout: 0.1) ) end end redis-rb-redis-cluster-client-aaf9e2e/test/test_against_cluster_down.rb000066400000000000000000000156711517214044400266330ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' class TestAgainstClusterDown < TestingWrapper WAIT_SEC = 0.1 CASES = %w[Single Pipeline Transaction Subscriber Publisher].freeze SINGLE_KEYS = %w[single1 single3 single4].freeze PIPELINE_KEYS = %w[pipeline1 pipeline2 pipeline4].freeze TRANSACTION_KEYS = %w[transaction1 transaction3 transaction4].freeze CHANNELS = %w[chan1 chan2 chan3].freeze NUMBER_OF_JOBS = SINGLE_KEYS.size + PIPELINE_KEYS.size + TRANSACTION_KEYS.size + CHANNELS.size * 2 def setup @captured_commands = ::Middlewares::CommandCapture::CommandBuffer.new @redirect_count = ::Middlewares::RedirectCount::Counter.new @clients = Array.new(NUMBER_OF_JOBS) { build_client } @threads = [] @controller = nil @cluster_down_counter = Counter.new @recorders = Array.new(NUMBER_OF_JOBS) { Recorder.new } @captured_commands.clear @redirect_count.clear end def teardown @controller&.close @threads&.each(&:exit) @clients&.each(&:close) print "#{@redirect_count.get}, "\ "ClusterShardsCall: #{@captured_commands.count('cluster', 'shards')}, "\ "ClusterDownError: #{@cluster_down_counter.get} = " end def test_recoverability_from_cluster_down SINGLE_KEYS.each_with_index { |key, i| @threads << spawn_single(@clients[i], @recorders[i], key) } PIPELINE_KEYS.each_with_index { |key, i| @threads << spawn_pipeline(@clients[i + 3], @recorders[i + 3], key) } TRANSACTION_KEYS.each_with_index { |key, i| @threads << spawn_transaction(@clients[i + 6], @recorders[i + 6], key) } CHANNELS.each_with_index do |channel, i| @threads << spawn_subscriber(@clients[i + 9], @recorders[i + 9], channel) @threads << spawn_publisher(@clients[i + 12], @recorders[i + 12], channel) end wait_for_jobs_to_be_stable system('docker compose --progress quiet down', exception: true) system('docker system prune --force --volumes', exception: true, out: File::NULL) system('docker compose --progress quiet up --detach', exception: true) @controller = build_controller @controller.wait_for_cluster_to_be_ready wait_for_jobs_to_be_stable refute(@cluster_down_counter.get.zero?, 'Case: cluster down count') refute(@captured_commands.count('cluster', 'shards').zero?, 'Case: cluster shards calls') @values_a = @recorders.map { |r| r.get.to_i } wait_for_jobs_to_be_stable @values_b = @recorders.map { |r| r.get.to_i } @recorders.each_with_index do |_, i| assert(@values_a[i] < @values_b[i], "#{CASES[i]}: #{@values_a[i]} < #{@values_b[i]}") end end private def build_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) ::RedisClient.cluster( nodes: TEST_NODE_URIS, connect_with_original_config: true, fixed_hostname: TEST_FIXED_HOSTNAME, custom: custom, middlewares: middlewares, **TEST_GENERIC_OPTIONS, **opts ).new_client end def build_controller ClusterController.new( TEST_NODE_URIS, replica_size: TEST_REPLICA_SIZE, **TEST_GENERIC_OPTIONS.merge(timeout: 30.0) ) end def spawn_single(client, recorder, key) Thread.new(client, recorder, key) do |cli, rec, k| loop do handle_errors do reply = cli.call('incr', k) rec.set(reply) end ensure sleep WAIT_SEC end end end def spawn_pipeline(client, recorder, key) Thread.new(client, recorder, key) do |cli, rec, k| loop do handle_errors do reply = cli.pipelined do |pi| pi.call('incr', k) pi.call('incr', k) end rec.set(reply.last) end ensure sleep WAIT_SEC end end end def spawn_transaction(client, recorder, key) Thread.new(client, recorder, key) do |cli, rec, k| i = 0 loop do handle_errors do reply = cli.multi(watch: i.odd? ? [k] : nil) do |tx| tx.call('incr', k) tx.call('incr', k) end rec.set(reply.last) i += 1 end ensure sleep WAIT_SEC end end end def spawn_publisher(client, recorder, channel) Thread.new(client, recorder, channel) do |cli, rec, chan| i = 0 loop do handle_errors do cli.call('spublish', chan, i) end rec.set(i) i += 1 ensure sleep WAIT_SEC end end end def spawn_subscriber(client, recorder, channel) Thread.new(client, recorder, channel) do |cli, rec, chan| ps = nil loop do ps = cli.pubsub ps.call('ssubscribe', chan) break rescue StandardError ps&.close ensure sleep WAIT_SEC end loop do handle_errors do event = ps.next_event(WAIT_SEC) case event&.first when 'smessage' then rec.set(event[2]) when 'sunsubscribe' then ps.call('ssubscribe', chan) end end end rescue StandardError, SignalException ps&.close raise end end def handle_errors yield rescue ::RedisClient::ConnectionError, ::RedisClient::Cluster::InitialSetupError, ::RedisClient::Cluster::NodeMightBeDown @cluster_down_counter.increment rescue ::RedisClient::CommandError => e raise unless e.message.start_with?('CLUSTERDOWN') @cluster_down_counter.increment rescue ::RedisClient::Cluster::ErrorCollection => e raise unless e.errors.values.all? do |err| err.message.start_with?('CLUSTERDOWN') || err.is_a?(::RedisClient::ConnectionError) end @cluster_down_counter.increment end def wait_for_jobs_to_be_stable(attempts: 100) start = Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) sleep_sec = WAIT_SEC * (@threads.size * 2) @recorders.each do |recorder| loop do raise MaxRetryExceeded if attempts <= 0 attempts -= 1 next sleep(sleep_sec) unless recorder.updated?(start) value_a = recorder.get.to_i sleep sleep_sec value_b = recorder.get.to_i break if value_a < value_b end end end class Counter def initialize @count = 0 @mutex = Mutex.new end def increment @mutex.synchronize { @count += 1 } end def get @mutex.synchronize { @count } end end class Recorder def initialize @last_value = nil @updated_at = nil @mutex = Mutex.new end def set(value) @mutex.synchronize do @last_value = value @updated_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) end end def get @mutex.synchronize { @last_value } end def updated?(since) @mutex.synchronize do if @updated_at.nil? false else since < @updated_at end end end end end redis-rb-redis-cluster-client-aaf9e2e/test/test_against_cluster_scale.rb000066400000000000000000000156701517214044400267520ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' module TestAgainstClusterScale PATTERN = ENV.fetch('TEST_CLASS_PATTERN', '') module Mixin WAIT_SEC = 1 MAX_ATTEMPTS = 20 NUMBER_OF_KEYS = 20_000 MAX_PIPELINE_SIZE = 40 HASH_TAG_GRAIN = 5 SLICED_NUMBERS = (0...NUMBER_OF_KEYS).each_slice(MAX_PIPELINE_SIZE).freeze def setup @captured_commands = ::Middlewares::CommandCapture::CommandBuffer.new @redirect_count = ::Middlewares::RedirectCount::Counter.new @client = ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, fixed_hostname: TEST_FIXED_HOSTNAME, custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **TEST_GENERIC_OPTIONS ).new_client @client.call('echo', 'init') @captured_commands.clear @redirect_count.clear @cluster_down_error_count = 0 end def teardown @client&.close @controller&.close print "#{@redirect_count.get}, "\ "ClusterShardsCall: #{@captured_commands.count('cluster', 'shards')}, "\ "ClusterDownError: #{@cluster_down_error_count} = " end def test_01_scale_out SLICED_NUMBERS.each do |numbers| @client.pipelined do |pi| numbers.each do |i| pi.call('SET', "key#{i}", i) pi.call('SET', "{group#{i / HASH_TAG_GRAIN}}:key#{i}", i) end end end wait_for_replication primary_url, replica_url = build_additional_node_urls @controller = build_cluster_controller(TEST_NODE_URIS, shard_size: 3) @controller.scale_out(primary_url: primary_url, replica_url: replica_url) do_test_after_scaled_out want = (TEST_NODE_URIS + build_additional_node_urls).size got = @client.instance_variable_get(:@router) .instance_variable_get(:@node) .instance_variable_get(:@topology) .instance_variable_get(:@clients) .size assert_equal(want, got, 'Case: number of nodes') refute(@captured_commands.count('cluster', 'shards').zero?, @captured_commands.to_a.map(&:command)) end def test_02_scale_in @controller = build_cluster_controller(TEST_NODE_URIS + build_additional_node_urls, shard_size: 4) @controller.scale_in do_test_after_scaled_in want = TEST_NODE_URIS.size got = @client.instance_variable_get(:@router) .instance_variable_get(:@node) .instance_variable_get(:@topology) .instance_variable_get(:@clients) .size assert_equal(want, got, 'Case: number of nodes') refute(@captured_commands.count('cluster', 'shards').zero?, @captured_commands.to_a.map(&:command)) end private def wait_for_replication client_side_timeout = TEST_TIMEOUT_SEC + 1.0 server_side_timeout = (TEST_TIMEOUT_SEC * 1000).to_i swap_timeout(@client, timeout: 0.1) do |client| client.blocking_call(client_side_timeout, 'WAIT', TEST_REPLICA_SIZE, server_side_timeout) end end def build_cluster_controller(nodes, shard_size:) ClusterController.new( nodes, shard_size: shard_size, replica_size: TEST_REPLICA_SIZE, **TEST_GENERIC_OPTIONS.merge(timeout: 30.0) ) end def build_additional_node_urls max = TEST_REDIS_PORTS.max (max + 1..max + 2).map { |port| "#{TEST_REDIS_SCHEME}://#{TEST_REDIS_HOST}:#{port}" } end def retryable(attempts:) loop do raise MaxRetryExceeded if attempts <= 0 attempts -= 1 break yield rescue ::RedisClient::CommandError => e raise unless e.message.start_with?('CLUSTERDOWN') @cluster_down_error_count += 1 sleep WAIT_SEC end end end if PATTERN == 'Single' || PATTERN.empty? class Single < TestingWrapper include Mixin def self.run_order :alpha end def do_test_after_scaled_out NUMBER_OF_KEYS.times do |i| assert_equal(i.to_s, @client.call('GET', "key#{i}"), "Case: key#{i}") end end def do_test_after_scaled_in NUMBER_OF_KEYS.times do |i| got = retryable(attempts: MAX_ATTEMPTS) { @client.call('GET', "key#{i}") } assert_equal(i.to_s, got, "Case: key#{i}") end end end end if PATTERN == 'Pipeline' || PATTERN.empty? class Pipeline < TestingWrapper include Mixin def self.run_order :alpha end def do_test_after_scaled_out SLICED_NUMBERS.each do |numbers| got = @client.pipelined do |pi| numbers.each { |i| pi.call('GET', "key#{i}") } end assert_equal(numbers.map(&:to_s), got, 'Case: GET') end end def do_test_after_scaled_in SLICED_NUMBERS.each do |numbers| got = retryable(attempts: MAX_ATTEMPTS) do @client.pipelined do |pi| numbers.each { |i| pi.call('GET', "key#{i}") } end end assert_equal(numbers.map(&:to_s), got, 'Case: GET') end end end end if PATTERN == 'Transaction' || PATTERN.empty? class Transaction < TestingWrapper include Mixin def self.run_order :alpha end def do_test_after_scaled_out NUMBER_OF_KEYS.times.group_by { |i| i / HASH_TAG_GRAIN }.each do |group, numbers| keys = numbers.map { |i| "{group#{group}}:key#{i}" } got = @client.multi(watch: group.odd? ? nil : keys) do |tx| keys.each { |key| tx.call('INCR', key) } end want = numbers.map { |i| i + 1 } assert_equal(want, got, 'Case: INCR') end end def do_test_after_scaled_in NUMBER_OF_KEYS.times.group_by { |i| i / HASH_TAG_GRAIN }.each do |group, numbers| keys = numbers.map { |i| "{group#{group}}:key#{i}" } got = retryable(attempts: MAX_ATTEMPTS) do @client.multi(watch: group.odd? ? nil : keys) do |tx| keys.each { |key| tx.call('INCR', key) } end end want = numbers.map { |i| i + 2 } assert_equal(want, got, 'Case: INCR') end end end end if PATTERN == 'PubSub' || PATTERN.empty? class PubSub < TestingWrapper include Mixin def self.run_order :alpha end def do_test_after_scaled_out 1000.times do |i| pubsub = @client.pubsub pubsub.call('SSUBSCRIBE', "chan#{i}") event = pubsub.next_event(0.01) event = pubsub.next_event(0.01) if event.nil? # state changed assert_equal(['ssubscribe', "chan#{i}", 1], event) assert_nil(pubsub.next_event(0.01)) ensure pubsub&.close end end alias do_test_after_scaled_in do_test_after_scaled_out end end end redis-rb-redis-cluster-client-aaf9e2e/test/test_against_cluster_state.rb000066400000000000000000000257761517214044400270130ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' module TestAgainstClusterState PATTERN = ENV.fetch('TEST_CLASS_PATTERN', '') module Mixin SLOT_SIZE = 16_384 def setup @controller = ClusterController.new( TEST_NODE_URIS, replica_size: TEST_REPLICA_SIZE, **TEST_GENERIC_OPTIONS ) @controller.rebuild @captured_commands = ::Middlewares::CommandCapture::CommandBuffer.new @redirect_count = ::Middlewares::RedirectCount::Counter.new @client = new_test_client.tap { |c| c.call('echo', 'init') } @captured_commands.clear @redirect_count.clear end def teardown @controller&.close @client&.close print "#{@redirect_count.get}, "\ "ClusterShardsCall: #{@captured_commands.count('cluster', 'shards')} = " end def test_the_state_of_cluster_resharding resharded_keys = nil do_resharding_test do |keys| resharded_keys = keys keys.each do |key| want = key got = @client.call('GET', key) assert_equal(want, got, "Case: GET: #{key}") end end refute(@redirect_count.zero?, @redirect_count.get) resharded_keys.each do |key| want = key got = @client.call('GET', key) assert_equal(want, got, "Case: GET: #{key}") end end def test_the_state_of_cluster_resharding_with_pipelining resharded_keys = nil do_resharding_test do |keys| resharded_keys = keys values = @client.pipelined do |pipeline| keys.each { |key| pipeline.call('GET', key) } end keys.each_with_index do |key, i| want = key got = values[i] assert_equal(want, got, "Case: GET: #{key}") end end values = @client.pipelined do |pipeline| resharded_keys.each { |key| pipeline.call('GET', key) } end resharded_keys.each_with_index do |key, i| want = key got = values[i] assert_equal(want, got, "Case: GET: #{key}") end # Since redirections are handled by #call_pipelined_aware_of_redirection, # we can't trace them in pipelining processes. # # refute(@redirect_count.zero?, @redirect_count.get) end def test_the_state_of_cluster_resharding_with_transaction call_cnt = 0 resharded_keys = nil do_resharding_test do |keys| resharded_keys = keys @client.multi do |tx| call_cnt += 1 keys.each do |key| tx.call('SET', key, '0') tx.call('INCR', key) end end keys.each do |key| want = '1' got = @client.call('GET', key) assert_equal(want, got, "Case: GET: #{key}") end end refute(@redirect_count.zero?, @redirect_count.get) @client.multi do |tx| call_cnt += 1 resharded_keys.each do |key| tx.call('SET', key, '2') tx.call('INCR', key) end end resharded_keys.each do |key| want = '3' got = @client.call('GET', key) assert_equal(want, got, "Case: GET: #{key}") end assert_equal(2, call_cnt) end def test_the_state_of_cluster_resharding_with_transaction_and_watch call_cnt = 0 resharded_keys = nil do_resharding_test do |keys| resharded_keys = keys @client.multi(watch: keys) do |tx| call_cnt += 1 keys.each do |key| tx.call('SET', key, '0') tx.call('INCR', key) end end keys.each do |key| want = '1' got = @client.call('GET', key) assert_equal(want, got, "Case: GET: #{key}") end end refute(@redirect_count.zero?, @redirect_count.get) @client.multi(watch: resharded_keys) do |tx| call_cnt += 1 resharded_keys.each do |key| tx.call('SET', key, '2') tx.call('INCR', key) end end resharded_keys.each do |key| want = '3' got = @client.call('GET', key) assert_equal(want, got, "Case: GET: #{key}") end assert_equal(2, call_cnt) end def test_the_state_of_cluster_resharding_with_reexecuted_watch client2 = new_test_client(middlewares: nil) call_cnt = 0 @client.call('SET', 'watch_key', 'original_value') @client.multi(watch: %w[watch_key]) do |tx| # Use client2 to change the value of watch_key, which would cause this transaction to fail if call_cnt == 0 client2.call('SET', 'watch_key', 'client2_value') # Now perform (and _finish_) a reshard, which should make this transaction receive a MOVED # redirection when it goes to commit. That should result in the entire block being retried slot = ::RedisClient::Cluster::KeySlotConverter.convert('watch_key') src, dest = @controller.select_resharding_target(slot) @controller.start_resharding(slot: slot, src_node_key: src, dest_node_key: dest) @controller.finish_resharding(slot: slot, src_node_key: src, dest_node_key: dest) end call_cnt += 1 tx.call('SET', 'watch_key', "@client_value_#{call_cnt}") end # It should have retried the entire transaction block. assert_equal(2, call_cnt) # The second call succeeded assert_equal('@client_value_2', @client.call('GET', 'watch_key')) refute(@redirect_count.zero?, @redirect_count.get) ensure client2&.close end def test_the_state_of_cluster_resharding_with_pipelining_on_new_connection # This test is excercising a very delicate race condition; i think the use of @client to set # the keys in do_resharding_test is actually causing the race condition not to happen, so this # test is actually performing the resharding on its own. key_count = 10 key_count.times do |i| key = "key#{i}" slot = ::RedisClient::Cluster::KeySlotConverter.convert(key) src, dest = @controller.select_resharding_target(slot) @controller.start_resharding(slot: slot, src_node_key: src, dest_node_key: dest) @controller.finish_resharding(slot: slot, src_node_key: src, dest_node_key: dest) end res = @client.pipelined do |p| key_count.times do |i| p.call_v(['SET', "key#{i}", "value#{i}"]) end end key_count.times do |i| assert_equal('OK', res[i]) assert_equal("value#{i}", @client.call_v(['GET', "key#{i}"])) end end private def wait_for_replication client_side_timeout = TEST_TIMEOUT_SEC + 1.0 server_side_timeout = (TEST_TIMEOUT_SEC * 1000).to_i swap_timeout(@client, timeout: 0.1) do |client| client.blocking_call(client_side_timeout, 'WAIT', TEST_REPLICA_SIZE, server_side_timeout) rescue RedisClient::Cluster::ErrorCollection => e raise unless e.errors.values.all? { |err| err.message.start_with?('ERR WAIT cannot be used with replica instances') } end end def do_resharding_test(number_of_keys: 1000) @client.pipelined { |pipeline| number_of_keys.times { |i| pipeline.call('SET', "key#{i}", "key#{i}") } } wait_for_replication count, slot = @client.pipelined { |pi| SLOT_SIZE.times { |i| pi.call('CLUSTER', 'COUNTKEYSINSLOT', i) } } .each_with_index.max_by { |c, _| c } refute_equal(0, count) keys = @client.call('CLUSTER', 'GETKEYSINSLOT', slot, count) refute_empty(keys) src, dest = @controller.select_resharding_target(slot) @controller.start_resharding(slot: slot, src_node_key: src, dest_node_key: dest) yield(keys) @controller.finish_resharding(slot: slot, src_node_key: src, dest_node_key: dest) end end if PATTERN == 'PrimaryOnly' || PATTERN.empty? class PrimaryOnly < TestingWrapper include Mixin private def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) ::RedisClient.cluster( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ).new_client end end end if PATTERN == 'Pooled' || PATTERN.empty? class Pooled < TestingWrapper include Mixin private def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) ::RedisClient.cluster( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ).new_pool(timeout: TEST_TIMEOUT_SEC, size: 2) end end end if PATTERN == 'ScaleReadRandom' || PATTERN.empty? class ScaleReadRandom < TestingWrapper include Mixin private def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random, fixed_hostname: TEST_FIXED_HOSTNAME, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ).new_client end end end if PATTERN == 'ScaleReadRandomWithPrimary' || PATTERN.empty? class ScaleReadRandomWithPrimary < TestingWrapper include Mixin private def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :random_with_primary, fixed_hostname: TEST_FIXED_HOSTNAME, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ).new_client end end end if PATTERN == 'ScaleReadLatency' || PATTERN.empty? class ScaleReadLatency < TestingWrapper include Mixin private def new_test_client( custom: { captured_commands: @captured_commands, redirect_count: @redirect_count }, middlewares: [::Middlewares::CommandCapture, ::Middlewares::RedirectCount], **opts ) ::RedisClient.cluster( nodes: TEST_NODE_URIS, replica: true, replica_affinity: :latency, fixed_hostname: TEST_FIXED_HOSTNAME, middlewares: middlewares, custom: custom, **TEST_GENERIC_OPTIONS, **opts ).new_client end end end end redis-rb-redis-cluster-client-aaf9e2e/test/test_concurrency.rb000066400000000000000000000151351517214044400247420ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' class TestConcurrency < TestingWrapper MAX_THREADS = Integer(ENV.fetch('REDIS_CLIENT_MAX_THREADS', 5)) ATTEMPTS = 1000 WANT = '1' def setup @client = new_test_client MAX_THREADS.times { |i| @client.call('SET', "key#{i}", WANT) } end def teardown @client&.close end def test_forking skip("fork is not available on #{RUBY_ENGINE}") if %w[jruby truffleruby].include?(RUBY_ENGINE) pids = Array.new(MAX_THREADS) do Process.fork do ATTEMPTS.times { |i| @client.call('INCR', "key#{i}") } end end pids += Array.new(MAX_THREADS) do Process.fork do ATTEMPTS.times { |i| @client.call('DECR', "key#{i}") } end end pids.each do |pid| _, status = Process.waitpid2(pid) assert_predicate(status, :success?) end MAX_THREADS.times { |i| assert_equal(WANT, @client.call('GET', "key#{i}")) } end def test_forking_with_pipelining skip("fork is not available on #{RUBY_ENGINE}") if %w[jruby truffleruby].include?(RUBY_ENGINE) pids = Array.new(MAX_THREADS) do Process.fork do @client.pipelined { |pi| ATTEMPTS.times { |i| pi.call('INCR', "key#{i}") } } end end pids += Array.new(MAX_THREADS) do Process.fork do @client.pipelined { |pi| ATTEMPTS.times { |i| pi.call('DECR', "key#{i}") } } end end pids.each do |pid| _, status = Process.waitpid2(pid) assert_predicate(status, :success?) end MAX_THREADS.times { |i| assert_equal(WANT, @client.call('GET', "key#{i}")) } end def test_forking_with_transaction skip("fork is not available on #{RUBY_ENGINE}") if %w[jruby truffleruby].include?(RUBY_ENGINE) @client.call('SET', '{key}1', WANT) pids = Array.new(MAX_THREADS) do Process.fork do @client.multi(watch: %w[{key}1]) do |tx| ATTEMPTS.times do tx.call('INCR', '{key}1') tx.call('DECR', '{key}1') end end end end pids.each do |pid| _, status = Process.waitpid2(pid) assert_predicate(status, :success?) end assert_equal(WANT, @client.call('GET', '{key}1')) end def test_threading threads = Array.new(MAX_THREADS) do Thread.new do ATTEMPTS.times { |i| @client.call('INCR', "key#{i}") } nil rescue StandardError => e e end end threads += Array.new(MAX_THREADS) do Thread.new do ATTEMPTS.times { |i| @client.call('DECR', "key#{i}") } nil rescue StandardError => e e end end threads.each { |t| assert_nil(t.value) } MAX_THREADS.times { |i| assert_equal(WANT, @client.call('GET', "key#{i}")) } end def test_threading_with_pipelining threads = Array.new(MAX_THREADS) do Thread.new do @client.pipelined { |pi| ATTEMPTS.times { |i| pi.call('INCR', "key#{i}") } } nil rescue StandardError => e e end end threads += Array.new(MAX_THREADS) do Thread.new do @client.pipelined { |pi| ATTEMPTS.times { |i| pi.call('DECR', "key#{i}") } } nil rescue StandardError => e e end end threads.each { |t| assert_nil(t.value) } MAX_THREADS.times { |i| assert_equal(WANT, @client.call('GET', "key#{i}")) } end def test_threading_with_transaction @client.call('SET', '{key}1', WANT) threads = Array.new(MAX_THREADS) do Thread.new do @client.multi(watch: %w[{key}1]) do |tx| ATTEMPTS.times do tx.call('INCR', '{key}1') tx.call('DECR', '{key}1') end end rescue StandardError => e e end end threads.each { |t| refute_instance_of(StandardError, t.value) } assert_equal(WANT, @client.call('GET', '{key}1')) end def test_ractor skip('Ractor is not available') unless Object.const_defined?(:Ractor, false) skip("#{RedisClient.default_driver} is not safe for Ractor") if RedisClient.default_driver != RedisClient::RubyConnection skip('OpenSSL gem has non-shareable objects') if TEST_REDIS_SSL skip('unstable ractor') if RUBY_ENGINE == 'ruby' && RUBY_ENGINE_VERSION.split('.').take(2).join('.').to_f < 4.0 ractors = Array.new(MAX_THREADS) do |i| c = ::RedisClient.cluster(nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS).new_client Ractor.new(i, c) do |i, c| c.call('get', "key#{i}") rescue StandardError => e e ensure c&.close end end ractors.each { |r| assert_equal(WANT, r.value) } end def test_ractor_with_pipelining skip('Ractor is not available') unless Object.const_defined?(:Ractor, false) skip("#{RedisClient.default_driver} is not safe for Ractor") if RedisClient.default_driver != RedisClient::RubyConnection skip('OpenSSL gem has non-shareable objects') if TEST_REDIS_SSL skip('unstable ractor') if RUBY_ENGINE == 'ruby' && RUBY_ENGINE_VERSION.split('.').take(2).join('.').to_f < 4.0 ractors = Array.new(MAX_THREADS) do |i| c = ::RedisClient.cluster(nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS).new_client Ractor.new(i, c) do |i, c| c.pipelined do |pi| pi.call('get', "key#{i}") pi.call('echo', 'hi') end rescue StandardError => e e ensure c&.close end end ractors.each { |r| assert_equal([WANT, 'hi'], r.value) } end def test_ractor_with_transaction skip('Ractor is not available') unless Object.const_defined?(:Ractor, false) skip("#{RedisClient.default_driver} is not safe for Ractor") if RedisClient.default_driver != RedisClient::RubyConnection skip('OpenSSL gem has non-shareable objects') if TEST_REDIS_SSL skip('unstable ractor') if RUBY_ENGINE == 'ruby' && RUBY_ENGINE_VERSION.split('.').take(2).join('.').to_f < 4.0 ractors = Array.new(MAX_THREADS) do |i| c = ::RedisClient.cluster(nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS).new_client Ractor.new(i, c) do |i, c| c.multi(watch: ["key#{i}"]) do |tx| tx.call('incr', "key#{i}") tx.call('incr', "key#{i}") end rescue StandardError => e e ensure c&.close end end ractors.each { |r| assert_equal([2, 3], r.value) } end private def new_test_client ::RedisClient.cluster( nodes: TEST_NODE_URIS, fixed_hostname: TEST_FIXED_HOSTNAME, **TEST_GENERIC_OPTIONS ).new_pool( timeout: TEST_TIMEOUT_SEC, size: MAX_THREADS ) end end redis-rb-redis-cluster-client-aaf9e2e/test/test_redis_cluster_client.rb000066400000000000000000000030261517214044400266110ustar00rootroot00000000000000# frozen_string_literal: true require 'testing_helper' class TestRedisClient < TestingWrapper def test_cluster [ { kwargs: {}, error: nil }, { kwargs: { nodes: 'redis://127.0.0.1:6379' }, error: nil }, { kwargs: { nodes: { host: '127.0.0.1' } }, error: nil }, { kwargs: { nodes: { port: 6379 } }, error: nil }, { kwargs: { nodes: { host: '127.0.0.1', port: 6379 } }, error: nil }, { kwargs: { nodes: [{ host: '127.0.0.1', port: 6379 }] }, error: nil }, { kwargs: { nodes: %w[redis://127.0.0.1:6379] }, error: nil }, { kwargs: { nodes: %w[redis://127.0.0.1:6379], replica: true }, error: nil }, { kwargs: { nodes: %w[redis://127.0.0.1:6379], replica: true, fixed_hostname: 'endpoint.example.com' }, error: nil }, { kwargs: { nodes: %w[redis://127.0.0.1:6379], replica: 1, fixed_hostname: '' }, error: nil }, { kwargs: { nodes: %w[redis://127.0.0.1:6379], foo: 'bar' }, error: nil }, { kwargs: { nodes: 'http://127.0.0.1:80' }, error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { kwargs: { nodes: [] }, error: ::RedisClient::ClusterConfig::InvalidClientConfigError }, { kwargs: { nodes: nil }, error: ::RedisClient::ClusterConfig::InvalidClientConfigError } ].each_with_index do |c, idx| msg = "Case: #{idx}" got = -> { ::RedisClient.cluster(**c[:kwargs]) } if c[:error].nil? assert_instance_of(::RedisClient::ClusterConfig, got.call, msg) else assert_raises(c[:error], msg, &got) end end end end redis-rb-redis-cluster-client-aaf9e2e/test/testing_constants.rb000066400000000000000000000062111517214044400251150ustar00rootroot00000000000000# frozen_string_literal: true # rubocop:disable Lint/UnderscorePrefixedVariableName require 'redis_client' TEST_REDIS_HOST = ENV.fetch('REDIS_HOST', '127.0.0.1') TEST_REDIS_PORT = 6379 TEST_TIMEOUT_SEC = 5.0 TEST_RECONNECT_ATTEMPTS = 3 _new_raw_cli = ->(**opts) { ::RedisClient.config(host: TEST_REDIS_HOST, port: TEST_REDIS_PORT, **opts).new_client } _test_cert_path = ->(f) { File.expand_path(File.join('ssl_certs', f), __dir__) } TEST_SSL_PARAMS = { ca_file: _test_cert_path.call('redis-rb-ca.crt'), cert: _test_cert_path.call('redis-rb-cert.crt'), key: _test_cert_path.call('redis-rb-cert.key') }.freeze _base_opts = { timeout: TEST_TIMEOUT_SEC, reconnect_attempts: TEST_RECONNECT_ATTEMPTS } _ssl_opts = { ssl: true, ssl_params: TEST_SSL_PARAMS }.freeze _redis_scheme = 'redis' begin _tmp_cli = _new_raw_cli.call(**_base_opts) _tmp_cli.call('PING') rescue ::RedisClient::UnsupportedServer _base_opts.merge!(protocol: 2) rescue ::RedisClient::ConnectionError => e raise unless e.message.include?('Connection reset by peer') || e.message.include?('EOFError') _redis_scheme = 'rediss' rescue ::RedisClient::CommandError => e raise unless e.message.include?('NOAUTH') _base_opts.merge!(password: '!&<123-abc>') ensure _tmp_cli&.close end TEST_REDIS_SCHEME = _redis_scheme TEST_REDIS_SSL = TEST_REDIS_SCHEME == 'rediss' TEST_FIXED_HOSTNAME = TEST_REDIS_SSL ? TEST_REDIS_HOST : nil TEST_SHARD_SIZE = ENV.fetch('REDIS_SHARD_SIZE', '3').to_i TEST_REPLICA_SIZE = ENV.fetch('REDIS_REPLICA_SIZE', '1').to_i TEST_NUMBER_OF_REPLICAS = TEST_REPLICA_SIZE * TEST_SHARD_SIZE TEST_NUMBER_OF_NODES = TEST_SHARD_SIZE + TEST_NUMBER_OF_REPLICAS case TEST_REDIS_HOST when '127.0.0.1', 'localhost' TEST_REDIS_PORTS = TEST_REDIS_PORT.upto(TEST_REDIS_PORT + TEST_NUMBER_OF_NODES - 1).to_a.freeze TEST_NODE_URIS = TEST_REDIS_PORTS.map { |v| "#{TEST_REDIS_SCHEME}://#{TEST_REDIS_HOST}:#{v}" }.freeze TEST_NODE_OPTIONS = TEST_REDIS_PORTS.to_h { |v| ["#{TEST_REDIS_HOST}:#{v}", { host: TEST_REDIS_HOST, port: v }] }.freeze when 'node1' TEST_REDIS_PORTS = Array.new(TEST_NUMBER_OF_NODES) { TEST_REDIS_PORT }.freeze TEST_NODE_URIS = Array.new(TEST_NUMBER_OF_NODES) do |i| host = "node#{i + 1}" "#{TEST_REDIS_SCHEME}://#{host}:#{TEST_REDIS_PORT}" end.freeze TEST_NODE_OPTIONS = Array.new(TEST_NUMBER_OF_NODES) do |i| host = "node#{format("%#{TEST_NUMBER_OF_NODES}d", i + 1)}" ["#{host}:#{TEST_REDIS_PORT}", { host: host, port: TEST_REDIS_PORT }] end.to_h.freeze else raise NotImplementedError, TEST_REDIS_HOST end TEST_GENERIC_OPTIONS = (TEST_REDIS_SSL ? _base_opts.merge(_ssl_opts) : _base_opts).freeze _tmp_cli = _new_raw_cli.call(**TEST_GENERIC_OPTIONS) TEST_REDIS_VERSION = _tmp_cli.call('INFO', 'SERVER').split("\r\n").grep(/redis_version.+/).first.split(':')[1] TEST_REDIS_MAJOR_VERSION = Integer(TEST_REDIS_VERSION.split('.').first) _tmp_cli.close BENCH_ENVOY_OPTIONS = { port: 7000, protocol: 2 }.freeze BENCH_REDIS_CLUSTER_PROXY_OPTIONS = { port: 7001, protocol: 2 }.freeze Ractor.make_shareable(TEST_NODE_URIS) if Object.const_defined?(:Ractor, false) && Ractor.respond_to?(:make_shareable) # rubocop:enable Lint/UnderscorePrefixedVariableName redis-rb-redis-cluster-client-aaf9e2e/test/testing_helper.rb000066400000000000000000000020301517214044400243530ustar00rootroot00000000000000# frozen_string_literal: true # @see https://docs.ruby-lang.org/en/2.1.0/MiniTest/Assertions.html require 'minitest/autorun' require 'redis-cluster-client' require 'testing_constants' require 'cluster_controller' require 'middlewares/command_capture' require 'middlewares/redirect_count' require 'middlewares/redirect_fake' case ENV.fetch('REDIS_CONNECTION_DRIVER', 'ruby') when 'hiredis' then require 'hiredis-client' end MaxRetryExceeded = Class.new(StandardError) class TestingWrapper < Minitest::Test private def swap_timeout(client, timeout:) return if client.nil? node = client.instance_variable_get(:@router)&.instance_variable_get(:@node) raise 'The client must be initialized.' if node.nil? updater = lambda do |c, t| c.read_timeout = t c.config.instance_variable_set(:@read_timeout, t) end regular_timeout = node.first.read_timeout node.each { |cli| updater.call(cli, timeout) } result = yield client node.each { |cli| updater.call(cli, regular_timeout) } result end end