i am trying to ruun a cluster using docker, managed to setup a1 master and slave and was able to access its redis-cli. Whoever when i run SET FOO BAR i get an error:
my docker-compose file:
version: '3.8'
networks:
app-tier:
driver: bridge
services:
mongodb_container:
networks:
- app-tier
image: mongo:latest
ports:
- 27017:27017
restart: unless-stopped
volumes:
- mongodb_data_container:/data/db
# Start Redis
master-rooms:
networks:
- app-tier
restart: always
image: docker.io/bitnami/redis-cluster:7.0
environment:
- 'REDIS_PORT_NUMBER=6379'
- 'REDIS_REPLICATION_MODE=master'
- 'ALLOW_EMPTY_PASSWORD=yes'
- 'REDIS_NODES=master-rooms replica-rooms master-data replica-data master-private replica-private cluster-setup'
ports:
- '6379:6379'
volumes:
- redis-cluster_data-0:/bitnami/redis/data
expose:
- 6379
replica-rooms:
networks:
- app-tier
restart: always
image: docker.io/bitnami/redis-cluster:7.0
environment:
- 'REDIS_REPLICATION_MODE=slave'
- 'REDIS_MASTER_HOST=master-rooms'
- 'REDIS_PORT_NUMBER=6380'
- 'REDIS_MASTER_PORT_NUMBER=6379'
- 'ALLOW_EMPTY_PASSWORD=yes'
- 'REDIS_NODES=master-rooms replica-rooms master-data replica-data master-private replica-private cluster-setup'
ports:
- '6380:6380'
depends_on:
- master-rooms
volumes:
- redis-cluster_data-1:/bitnami/redis/data
expose:
- 6380
master-data:
networks:
- app-tier
restart: always
environment:
- 'REDIS_PORT_NUMBER=6179'
- 'REDIS_REPLICATION_MODE=master'
- 'ALLOW_EMPTY_PASSWORD=yes'
- 'REDIS_NODES=master-rooms replica-rooms master-data replica-data master-private replica-private cluster-setup'
image: docker.io/bitnami/redis-cluster:7.0
ports:
- '6179:6179'
volumes:
- redis-cluster_data-2:/bitnami/redis/data
expose:
- 6179
replica-data:
networks:
- app-tier
restart: always
environment:
- 'REDIS_PORT_NUMBER=6180'
- 'REDIS_REPLICATION_MODE=slave'
- 'REDIS_MASTER_HOST=master-data'
- 'REDIS_MASTER_PORT_NUMBER=6179'
- 'ALLOW_EMPTY_PASSWORD=yes'
- 'REDIS_NODES=master-rooms replica-rooms master-data replica-data master-private replica-private cluster-setup'
image: docker.io/bitnami/redis-cluster:7.0
ports:
- '6180:6180'
depends_on:
- master-data
volumes:
- redis-cluster_data-3:/bitnami/redis/data
expose:
- 6180
master-private:
networks:
- app-tier
restart: always
environment:
- 'REDIS_PORT_NUMBER=6079'
- 'REDIS_REPLICATION_MODE=master'
- 'ALLOW_EMPTY_PASSWORD=yes'
- 'REDIS_NODES=master-rooms replica-rooms master-data replica-data master-private replica-private cluster-setup'
image: docker.io/bitnami/redis-cluster:7.0
ports:
- '6079:6079'
volumes:
- redis-cluster_data-4:/bitnami/redis/data
expose:
- 6079
replica-private:
networks:
- app-tier
restart: always
environment:
- 'REDIS_PORT_NUMBER=6080'
- 'REDIS_REPLICATION_MODE=slave'
- 'REDIS_MASTER_HOST=master-private'
- 'REDIS_MASTER_PORT_NUMBER=6079'
- 'ALLOW_EMPTY_PASSWORD=yes'
- 'REDIS_NODES=master-rooms replica-rooms master-data replica-data master-private replica-private cluster-setup'
image: docker.io/bitnami/redis-cluster:7.0
ports:
- '6080:6080'
depends_on:
- master-private
volumes:
- redis-cluster_data-5:/bitnami/redis/data
expose:
- 6080
cluster-setup:
networks:
- app-tier
image: docker.io/bitnami/redis-cluster:7.0
ports:
- '6378:6378'
volumes:
- redis-cluster_creator:/bitnami/redis/data
depends_on:
- master-rooms
- master-data
- master-private
# - master-allowedaa
# - master-passwords
- replica-rooms
- replica-data
- replica-private
environment:
- 'REDIS_PORT_NUMBER=6378'
- 'ALLOW_EMPTY_PASSWORD=yes'
- 'REDIS_CLUSTER_REPLICAS=1'
- 'REDIS_NODES=master-rooms replica-rooms master-data replica-data master-private replica-private cluster-setup'
- 'REDIS_CLUSTER_CREATOR=yes'
expose:
- 6378
# End Redis
volumes:
redis-cluster_data-0:
driver: local
redis-cluster_data-1:
driver: local
redis-cluster_data-2:
driver: local
redis-cluster_data-3:
driver: local
redis-cluster_data-4:
driver: local
redis-cluster_data-5:
driver: local
redis-cluster_creator:
driver: local
The error:
=== REDIS BUG REPORT START: Cut & paste starting from here === | 1:M 30 Sep 2022 00:36:09.952 # Redis 7.0.5 crashed by signal: 11, si_code: 1 | 1:M 30 Sep 2022 00:36:09.952 # Accessing address: 0x3334626531377e | 1:M 30 Sep 2022 00:36:09.952 # Crashed running the instruction at: 0x40000e9c0c | | ------ STACK TRACE ------ | EIP: | redis-server 0.0.0.0:6179 cluster[0x40000e9c0c] | | Backtrace: | /lib/x86_64-linux-gnu/libpthread.so.0(+0x13140)[0x4002217140] | redis-server 0.0.0.0:6179 cluster[0x40000e9c0c] | redis-server 0.0.0.0:6179 cluster[0x400009efc3] | redis-server 0.0.0.0:6179 cluster[0x40000a0bf9] | redis-server 0.0.0.0:6179 cluster[0x40000b4681] | redis-server 0.0.0.0:6179 cluster[0x40000b4915] | redis-server 0.0.0.0:6179 cluster[0x400007a6cb] | redis-server 0.0.0.0:6179 cluster[0x400007cf2d] | redis-server 0.0.0.0:6179 cluster[0x4000093fc0] | redis-server 0.0.0.0:6179 cluster[0x4000097c88] | redis-server 0.0.0.0:6179 cluster[0x4000144b31] | redis-server 0.0.0.0:6179 cluster[0x4000071a9a] | redis-server 0.0.0.0:6179 cluster[0x4000071e45] | redis-server 0.0.0.0:6179 cluster[0x400006d5ae] | /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xea)[0x4002249d0a] | redis-server 0.0.0.0:6179 cluster[0x400006dc0a] | | ------ REGISTERS ------ | 1:M 30 Sep 2022 00:36:09.961 # | RAX:00000040038afde0 RBX:0000004002a1e550 | RCX:0000004002a5926c RDX:3233346265313766 | RDI:0000004002a5926c RSI:000000400020f980 | RBP:0000004001cfefa0 RSP:0000004001cfef90 | R8 :0000000000000003 R9 :0000000000000028 | R10:0000004002a02f40 R11:00000040023fdf40 | R12:0000004002a9b000 R13:0000004002a1e550 | R14:0000004002a643d8 R15:0000004002bbd600 | RIP:00000040000e9c0c EFL:0000000000000206 | CSGSFS:002b000000000033 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef9f) -> 0000004002bbd600 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef9e) -> 0000000000000000 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef9d) -> 0000004002a643d8 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef9c) -> 0000004002a643c0 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef9b) -> 0000000000000000 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef9a) -> 0000004001cff010 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef99) -> 00000040000a0bf9 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef98) -> 0000004001cff010 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef97) -> 0000004002a9b000 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef96) -> 0000004002a643d8 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef95) -> 0000004002a643c0 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef94) -> 0000000000000008 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef93) -> 000000400009efc3 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef92) -> 0000004001cfefd0 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef91) -> 0000004002a643c0 | 1:M 30 Sep 2022 00:36:09.961 # (0000004001cfef90) -> 0000004002a9b000 | | ------ INFO OUTPUT ------ | # Server | redis_version:7.0.5 | redis_git_sha1:00000000 | redis_git_dirty:0 | redis_build_id:150190119493720b | redis_mode:cluster | os:Linux 5.10.104-linuxkit x86_64 | arch_bits:64 | monotonic_clock:POSIX clock_gettime | multiplexing_api:epoll | atomicvar_api:c11-builtin | gcc_version:10.2.1 | process_id:1 | process_supervised:no | run_id:a918e2603f64a78561bdcd3eed4afd3d133703ec | tcp_port:6179 | server_time_usec:1664498169951715 | uptime_in_seconds:26 | uptime_in_days:0 | hz:10 | configured_hz:10 | lru_clock:3553785 | executable:/redis-server | config_file: | io_threads_active:0 | | # Clients | connected_clients:1 | cluster_connections:10 | maxclients:10000 | client_recent_max_input_buffer:20480 | client_recent_max_output_buffer:20504 | blocked_clients:0 | tracking_clients:0 | clients_in_timeout_table:0 | | # Memory | used_memory:1775616 | used_memory_human:1.69M | used_memory_rss:0 | used_memory_rss_human:0B | used_memory_peak:1790152 | used_memory_peak_human:1.71M | used_memory_peak_perc:99.19% | used_memory_overhead:1630636 | used_memory_startup:1600432 | used_memory_dataset:144980 | used_memory_dataset_perc:82.76% | allocator_allocated:1933728 | allocator_active:2322432 | allocator_resident:5337088 | total_system_memory:8232747008 | total_system_memory_human:7.67G | used_memory_lua:31744 | used_memory_vm_eval:31744 | used_memory_lua_human:31.00K | used_memory_scripts_eval:0 | number_of_cached_scripts:0 | number_of_functions:0 | number_of_libraries:0 | used_memory_vm_functions:32768 | used_memory_vm_total:64512 | used_memory_vm_total_human:63.00K | used_memory_functions:184 | used_memory_scripts:184 | used_memory_scripts_human:184B | maxmemory:0 | maxmemory_human:0B | maxmemory_policy:noeviction | allocator_frag_ratio:1.20 | allocator_frag_bytes:388704 | allocator_rss_ratio:2.30 | allocator_rss_bytes:3014656 | rss_overhead_ratio:0.00 | rss_overhead_bytes:-5337088 | mem_fragmentation_ratio:0.00 | mem_fragmentation_bytes:-1753064 | mem_not_counted_for_evict:8 | mem_replication_backlog:20508 | mem_total_replication_buffers:20504 | mem_clients_slaves:0 | mem_clients_normal:1800 | mem_cluster_links:7616 | mem_aof_buffer:8 | mem_allocator:jemalloc-5.2.1 | active_defrag_running:0 | lazyfree_pending_objects:0 | lazyfreed_objects:0 | | # Persistence | loading:0 | async_loading:0 | current_cow_peak:0 | current_cow_size:0 | current_cow_size_age:0 | current_fork_perc:0.00 | current_save_keys_processed:0 | current_save_keys_total:0 | rdb_changes_since_last_save:0 | rdb_bgsave_in_progress:0 | rdb_last_save_time:1664498143 | rdb_last_bgsave_status:ok | rdb_last_bgsave_time_sec:0 | rdb_current_bgsave_time_sec:-1 | rdb_saves:0 | rdb_last_cow_size:1806336 | rdb_last_load_keys_expired:0 | rdb_last_load_keys_loaded:0 | aof_enabled:1 | aof_rewrite_in_progress:0 | aof_rewrite_scheduled:0 | aof_last_rewrite_time_sec:-1 | aof_current_rewrite_time_sec:-1 | aof_last_bgrewrite_status:ok | aof_rewrites:0 | aof_rewrites_consecutive_failures:0 | aof_last_write_status:ok | aof_last_cow_size:0 | module_fork_in_progress:0 | module_fork_last_cow_size:0 | aof_current_size:88 | aof_base_size:88 | aof_pending_rewrite:0 | aof_buffer_length:0 | aof_pending_bio_fsync:0 | aof_delayed_fsync:0 | | # Stats | total_connections_received:3 | total_commands_processed:25 | instantaneous_ops_per_sec:1 | total_net_input_bytes:963 | total_net_output_bytes:833 | total_net_repl_input_bytes:0 | total_net_repl_output_bytes:286 | instantaneous_input_kbps:0.04 | instantaneous_output_kbps:0.00 | instantaneous_input_repl_kbps:0.00 | instantaneous_output_repl_kbps:0.00 | rejected_connections:0 | sync_full:1 | sync_partial_ok:0 | sync_partial_err:1 | expired_keys:0 | expired_stale_perc:0.00 | expired_time_cap_reached_count:0 | expire_cycle_cpu_milliseconds:3 | evicted_keys:0 | evicted_clients:0 | total_eviction_exceeded_time:0 | current_eviction_exceeded_time:0 | keyspace_hits:0 | keyspace_misses:0 | pubsub_channels:0 | pubsub_patterns:0 | pubsubshard_channels:0 | latest_fork_usec:2580 | total_forks:1 | migrate_cached_sockets:0 | slave_expires_tracked_keys:0 | active_defrag_hits:0 | active_defrag_misses:0 | active_defrag_key_hits:0 | active_defrag_key_misses:0 | total_active_defrag_time:0 | current_active_defrag_time:0 | tracking_total_keys:0 | tracking_total_items:0 | tracking_total_prefixes:0 | unexpected_error_replies:0 | total_error_replies:1 | dump_payload_sanitizations:0 | total_reads_processed:27 | total_writes_processed:7 | io_threaded_reads_processed:0 | io_threaded_writes_processed:0 | reply_buffer_shrinks:2 | reply_buffer_expands:0 | | # Replication | role:master | connected_slaves:1 | slave0:ip=172.24.0.6,port=6180,state=online,offset=42,lag=0 | master_failover_state:no-failover | master_replid:7117ba435e53fb6a287f3696351c02a22cb28b20 | master_replid2:0000000000000000000000000000000000000000 | master_repl_offset:42 | second_repl_offset:-1 | repl_backlog_active:1 | repl_backlog_size:1048576 | repl_backlog_first_byte_offset:1 | repl_backlog_histlen:42 | | # CPU | used_cpu_sys:0.184315 | used_cpu_user:0.882482 | used_cpu_sys_children:0.390085 | used_cpu_user_children:2.357764 | used_cpu_sys_main_thread:0.179068 | used_cpu_user_main_thread:0.876434 | | # Modules | | # Commandstats | cmdstat_ping:calls=1,usec=236,usec_per_call=236.00,rejected_calls=1,failed_calls=0 | cmdstat_cluster|info:calls=1,usec=2219,usec_per_call=2219.00,rejected_calls=0,failed_calls=0 | cmdstat_psync:calls=1,usec=1254,usec_per_call=1254.00,rejected_calls=0,failed_calls=0 | cmdstat_replconf:calls=22,usec=2227,usec_per_call=101.23,rejected_calls=0,failed_calls=0 | | # Errorstats | errorstat_LOADING:count=1 | | # Latencystats | latency_percentiles_usec_ping:p50=236.543,p99=236.543,p99.9=236.543 | latency_percentiles_usec_cluster|info:p50=0.000,p99=0.000,p99.9=0.000 | latency_percentiles_usec_psync:p50=1261.567,p99=1261.567,p99.9=1261.567 | latency_percentiles_usec_replconf:p50=22.015,p99=1515.519,p99.9=1515.519 | | # Cluster | cluster_enabled:1 | | # Keyspace | db0:keys=1,expires=0,avg_ttl=0 | | ------ CLIENT LIST OUTPUT ------ | id=5 addr=172.24.0.6:37030 laddr=172.24.0.5:6179 fd=19 name= age=25 idle=0 flags=S db=0 sub=0 psub=0 ssub=0 multi=-1 qbuf=0 qbuf-free=20474 argv-mem=0 multi-mem=0 rbs=1024 rbp=0 obl=0 oll=1 omem=20504 tot-mem=42776 events=r cmd=replconf user=default redir=-1 resp=2 | id=6 addr=172.24.0.1:56364 laddr=172.24.0.5:6179 fd=18 name= age=10 idle=0 flags=N db=0 sub=0 psub=0 ssub=0 multi=-1 qbuf=31 qbuf-free=20443 argv-mem=9 multi-mem=0 rbs=1024 rbp=0 obl=0 oll=0 omem=0 tot-mem=22305 events=r cmd=set user=default redir=-1 resp=2 | | ------ CURRENT CLIENT INFO ------ | id=6 addr=172.24.0.1:56364 laddr=172.24.0.5:6179 fd=18 name= age=10 idle=0 flags=N db=0 sub=0 psub=0 ssub=0 multi=-1 qbuf=31 qbuf-free=20443 argv-mem=9 multi-mem=0 rbs=1024 rbp=0 obl=0 oll=0 omem=0 tot-mem=22305 events=r cmd=set user=default redir=-1 resp=2 | argv[0]: '"set"' | argv[1]: '"foo"' | argv[2]: '"bar"' | 1:M 30 Sep 2022 00:36:09.967 # key 'foo' found in DB containing the following object: | 1:M 30 Sep 2022 00:36:09.967 # Object type: 0 | 1:M 30 Sep 2022 00:36:09.967 # Object encoding: 8 | 1:M 30 Sep 2022 00:36:09.967 # Object refcount: 1 | | ------ MODULES INFO OUTPUT ------ | | ------ CONFIG DEBUG OUTPUT ------ | io-threads 1 | replica-read-only yes | lazyfree-lazy-user-flush no | repl-diskless-load disabled | lazyfree-lazy-eviction no | slave-read-only yes | lazyfree-lazy-server-del no | sanitize-dump-payload no | client-query-buffer-limit 1gb | io-threads-do-reads no | lazyfree-lazy-user-del no | activedefrag no | list-compress-depth 0 | lazyfree-lazy-expire no | proto-max-bulk-len 512mb | repl-diskless-sync yes | | ------ FAST MEMORY TEST ------ | 1:M 30 Sep 2022 00:36:09.969 # Bio thread for job type #0 terminated | 1:M 30 Sep 2022 00:36:09.969 # Bio thread for job type #1 terminated | 1:M 30 Sep 2022 00:36:09.970 # Bio thread for job type #2 terminated | *** Preparing to test memory region 40002cd000 (2572288 bytes) | *** Preparing to test memory region 4001d2c000 (12288 bytes) | *** Preparing to test memory region 4001e75000 (8192 bytes) | *** Preparing to test memory region 4002200000 (16384 bytes) | *** Preparing to test memory region 4002222000 (16384 bytes) | *** Preparing to test memory region 40023f7000 (40960 bytes) | *** Preparing to test memory region 4002800000 (2097152 bytes) | *** Preparing to test memory region 4002a01000 (2097152 bytes) | *** Preparing to test memory region 4003200000 (9961472 bytes) | *** Preparing to test memory region 4003b81000 (8388608 bytes) | *** Preparing to test memory region 4004382000 (8388608 bytes) | *** Preparing to test memory region 4004b83000 (8388608 bytes) | *** Preparing to test memory region 4005384000 (8388608 bytes) | .O.O.O.O.O.O.O.O.O.O.O.O.O | Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible. | | ------ DUMPING CODE AROUND EIP ------ | Symbol: slotToKeyAddEntry (base: 0x40000e9bb0) | Module: redis-server 0.0.0.0:6179 [cluster] (base 0x4000000000) | $ xxd -r -p /tmp/dump.hex /tmp/dump.bin | $ objdump --adjust-vma=0x40000e9bb0 -D -b binary -m i386:x86-64 /tmp/dump.bin | ------ | 1:M 30 Sep 2022 00:36:10.255 # dump of function (hexdump of 220 bytes): | 554889e541544989f4534889fb488b3f0fb677ff89f083e0073c040f87142ef8ff488d15085c12000fb6c0486304824801d0ffe00f1f40008b77f7e85070ffff89c048c1e0044903442448488b500848830001488953204885d2740b48837a1800755b48895a1848837b18007538488958085b415c5dc3660f1f8400000000008b77efebb60f1f0040c0ee03400fb6f6eba9660f1f4400000fb677fdeb9d66900fb777fbeb95ba1c1b0000488d35032a1200488d3d67491200e8f20dffffba191b0000488d35eb291200488d3d27491200e8da0dffff662e0f1f8400 | | === REDIS BUG REPORT END. Make sure to include from START to END. === | | Please report the crash by opening an issue on github: | | http://github.com/redis/redis/issues | | If a Redis module was involved, please open in the module's repo instead. | | Suspect RAM error? Use redis-server --test-memory to verify it. | | Some other issues could be detected by redis-server --check-system | qemu: uncaught target signal 11 (Segmentation fault) - core dumped | 1:S 30 Sep 2022 00:36:26.396 # Cluster state changed: fail | 1:S 30 Sep 2022 00:36:27.694 # Cluster state changed: fail
Comment From: jgregory-apogee
We're seeing this error, too at slotKeyToAddEntry(). We're running a redis cluster from helm redis/redis-cluster. Were you ever able to resolve it?
Comment From: sundb
qemu was found in the log.
Please try to use the docker images in the comment https://github.com/redis/redis/issues/10162#issuecomment-1072359526.
~~Could you specify the docker image you using?~~(show in the docker-compose.yml)
Comment From: jgregory-apogee
We're using helm for this. Here are the commands we're using:
helm repo add redis https://charts.bitnami.com/bitnami
helm install redis-cluster -f environments/local/values-redis.yml redis/redis-cluster
values-redis.yml:
## Redis Cluster settings
fullnameOverride: redis-service
cluster:
init: true
nodes: 3
replicas: 0
usePassword: true
password: "redacted"