Crash report
Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context.
=== REDIS BUG REPORT START: Cut & paste starting from here ===
6662:S 25 Jul 2020 16:13:57.299 # Redis 6.0.6 crashed by signal: 11
6662:S 25 Jul 2020 16:13:57.299 # Crashed running the instruction at: 0x7f912a86deba
6662:S 25 Jul 2020 16:13:57.299 # Accessing address: 0x20
6662:S 25 Jul 2020 16:13:57.299 # Failed assertion: <no assertion failed> (<no file>:0)
------ STACK TRACE ------
EIP:
/lib64/libssl.so.10(SSL_set_bio+0xa)[0x7f912a86deba]
Backtrace:
//usr/local/bin/redis-server 0.0.0.0:6379(logStackTrace+0x2e)[0x47751e]
//usr/local/bin/redis-server 0.0.0.0:6379(sigsegvHandler+0x96)[0x477ba6]
/lib64/libpthread.so.0(+0xf630)[0x7f912a1bc630]
/lib64/libssl.so.10(SSL_set_bio+0xa)[0x7f912a86deba]
/lib64/libssl.so.10(SSL_set_fd+0x3f)[0x7f912a86e06f]
//usr/local/bin/redis-server 0.0.0.0:6379[0x4c24b0]
//usr/local/bin/redis-server 0.0.0.0:6379(aeProcessEvents+0x28a)[0x42f41a]
//usr/local/bin/redis-server 0.0.0.0:6379(aeMain+0x1d)[0x42f76d]
//usr/local/bin/redis-server 0.0.0.0:6379(main+0x4b9)[0x42c0b9]
/lib64/libc.so.6(__libc_start_main+0xf5)[0x7f9129e01555]
//usr/local/bin/redis-server 0.0.0.0:6379[0x42c369]
------ INFO OUTPUT ------
# Server
redis_version:6.0.6
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:451087295f496f75
redis_mode:standalone
os:Linux 4.19.76-linuxkit x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:8.3.1
process_id:6662
run_id:fa946606553c4a2892850ddd2050a54d942b8980
tcp_port:6379
uptime_in_seconds:0
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:1858117
executable://usr/local/bin/redis-server
config_file:/etc/redis/6379.conf
# Clients
connected_clients:0
client_recent_max_input_buffer:0
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
# Memory
used_memory:806568
used_memory_human:787.66K
used_memory_rss:5459968
used_memory_rss_human:5.21M
used_memory_peak:806568
used_memory_peak_human:787.66K
used_memory_peak_perc:100.20%
used_memory_overhead:804936
used_memory_startup:804936
used_memory_dataset:1632
used_memory_dataset_perc:100.00%
allocator_allocated:1257008
allocator_active:1536000
allocator_resident:4050944
total_system_memory:8353480704
total_system_memory_human:7.78G
used_memory_lua:37888
used_memory_lua_human:37.00K
used_memory_scripts:0
used_memory_scripts_human:0B
number_of_cached_scripts:0
maxmemory:524288000
maxmemory_human:500.00M
maxmemory_policy:volatile-lru
allocator_frag_ratio:1.22
allocator_frag_bytes:278992
allocator_rss_ratio:2.64
allocator_rss_bytes:2514944
rss_overhead_ratio:1.35
rss_overhead_bytes:1409024
mem_fragmentation_ratio:6.78
mem_fragmentation_bytes:4655032
mem_not_counted_for_evict:0
mem_replication_backlog:0
mem_clients_slaves:0
mem_clients_normal:0
mem_aof_buffer:0
mem_allocator:jemalloc-5.1.0
active_defrag_running:0
lazyfree_pending_objects:0
# Persistence
loading:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1595693637
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:0
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0
aof_current_size:0
aof_base_size:0
aof_pending_rewrite:0
aof_buffer_length:0
aof_rewrite_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:0
# Stats
total_connections_received:0
total_commands_processed:0
instantaneous_ops_per_sec:0
total_net_input_bytes:0
total_net_output_bytes:0
instantaneous_input_kbps:0.00
instantaneous_output_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:0
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
# Replication
role:slave
master_host:172.17.0.2
master_port:6379
master_link_status:down
master_last_io_seconds_ago:-1
master_sync_in_progress:0
slave_repl_offset:1
master_link_down_since_seconds:1595693637
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:6221de48e9a463b603671d5aee8fde4b83646f7c
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:0
second_repl_offset:-1
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
# CPU
used_cpu_sys:0.002772
used_cpu_user:0.001386
used_cpu_sys_children:0.000000
used_cpu_user_children:0.000000
# Modules
# Commandstats
# Cluster
cluster_enabled:0
# Keyspace
------ CLIENT LIST OUTPUT ------
------ REGISTERS ------
6662:S 25 Jul 2020 16:13:57.301 #
RAX:0000000000000001 RBX:0000000002088760
RCX:00007ffd9aff2c44 RDX:0000000002088760
RDI:0000000000000000 RSI:0000000002088760
RBP:0000000002088760 RSP:00007ffd9aff2c40
R8 :0000000000000000 R9 :0000000000000000
R10:00007ffd9aff26e0 R11:00007f912a4e4b60
R12:0000000002088760 R13:0000000000000008
R14:0000000000000000 R15:0000000000000002
RIP:00007f912a86deba EFL:0000000000010202
CSGSFS:002b000000000033
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c4f) -> 000000000042f41a
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c4e) -> 0000000000000008
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c4d) -> 0000000000000001
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c4c) -> 0000000000000100
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c4b) -> 00007f912820b480
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c4a) -> 00007f912820b480
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c49) -> 000000000042eb4b
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c48) -> 00007f9128223150
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c47) -> 00000000004c24b0
6662:S 25 Jul 2020 16:13:57.301 # (00007ffd9aff2c46) -> 0000000000000001
6662:S 25 Jul 2020 16:13:57.302 # (00007ffd9aff2c45) -> 0000000000000002
6662:S 25 Jul 2020 16:13:57.302 # (00007ffd9aff2c44) -> 00007f912820d580
6662:S 25 Jul 2020 16:13:57.302 # (00007ffd9aff2c43) -> 00007f912a86e06f
6662:S 25 Jul 2020 16:13:57.302 # (00007ffd9aff2c42) -> 0000000000000008
6662:S 25 Jul 2020 16:13:57.302 # (00007ffd9aff2c41) -> 0000000000000000
6662:S 25 Jul 2020 16:13:57.302 # (00007ffd9aff2c40) -> 0000000002088760
------ MODULES INFO OUTPUT ------
------ FAST MEMORY TEST ------
6662:S 25 Jul 2020 16:13:57.302 # Bio thread for job type #0 terminated
6662:S 25 Jul 2020 16:13:57.302 # Bio thread for job type #1 terminated
6662:S 25 Jul 2020 16:13:57.303 # Bio thread for job type #2 terminated
*** Preparing to test memory region 7bf000 (2281472 bytes)
*** Preparing to test memory region 2061000 (135168 bytes)
*** Preparing to test memory region 2082000 (135168 bytes)
*** Preparing to test memory region 7f9125dfd000 (8388608 bytes)
*** Preparing to test memory region 7f91265fe000 (8388608 bytes)
*** Preparing to test memory region 7f9126dff000 (8388608 bytes)
*** Preparing to test memory region 7f9127600000 (8388608 bytes)
*** Preparing to test memory region 7f9127e00000 (8388608 bytes)
*** Preparing to test memory region 7f9128c2c000 (8192 bytes)
*** Preparing to test memory region 7f9128e46000 (8192 bytes)
*** Preparing to test memory region 7f912a1a8000 (20480 bytes)
*** Preparing to test memory region 7f912a3c5000 (16384 bytes)
*** Preparing to test memory region 7f912a828000 (16384 bytes)
*** Preparing to test memory region 7f912b3bd000 (45056 bytes)
*** Preparing to test memory region 7f912b3cb000 (4096 bytes)
*** Preparing to test memory region 7f912b3cc000 (4096 bytes)
*** Preparing to test memory region 7f912b3cf000 (4096 bytes)
.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O.O
Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.
------ DUMPING CODE AROUND EIP ------
Symbol: SSL_set_bio (base: 0x7f912a86deb0)
Module: /lib64/libssl.so.10 (base 0x7f912a82c000)
$ xxd -r -p /tmp/dump.hex /tmp/dump.bin
$ objdump --adjust-vma=0x7f912a86deb0 -D -b binary -m i386:x86-64 /tmp/dump.bin
------
6662:S 25 Jul 2020 16:13:57.448 # dump of function (hexdump of 138 bytes):
41544989f4554889d553488b47204889fb4885c07406483b4718743c488b7b104885ff740a4c39e77405e84173fdff488b7b184885ff74104839ef740b483b7b107405e82873fdff4c89631048896b185b5d415cc30f1f00488b50384889571848c7403800000000ebb2660f1f440000488b4710c390662e0f1f840000000000488b4718c390662e0f1f
=== REDIS BUG REPORT END. Make sure to include from START to END. ===
Aditional information
- OS distribution and version
- Steps to reproduce (if any)
Comment From: oranagra
@daixijun can you please add info about your OS. seems that it crashed at srartup, is that right? Does it reproduce? do you have any idea how we can reproduce it?
Comment From: daixijun
@oranagra run redis 6.0.0 in docker, image: geerlingguy/docker-centos7-ansible:latest it crashed at srartup, is right.
manage redis daemon using systemd
[Unit]
Description=Redis persistent key-value database - instance %i
After=network.target
After=network-online.target
Wants=network-online.target
AssertPathExists=/etc/redis/%i.conf
[Service]
ExecStart=/usr/local/bin/redis-server /etc/redis/%i.conf
ExecStop=/usr/local/libexec/redis-shutdown redis/%i
Type=forking
User=redis
Group=redis
LimitNOFILE=10032
NoNewPrivileges=yes
UMask=0077
TimeoutStartSec=30
TimeoutStopSec=30
# RuntimeDirectory=redis
# RuntimeDirectoryMode=0755
[Install]
WantedBy=multi-user.target
Comment From: oranagra
@daixijun redis 6.0.x depends on C11 _Atomic feature which requires GCC 4.9 and newer. Centos 7 (and docker image you used) has GCC 4.8.5. I'm not sure how the redis binary you're using was built, but it seems it's not compatible with your system.
you can have a look at #7509 and #6286, there are some ways to work around this.
Comment From: daixijun
@ocadaruma i build redis 6.0.x using gcc 8.3.1 (scl devtoolset)
Comment From: yossigo
@daixijun This doesn't seem to easily reproduce. I suspect it could be a mismatch between the openssl headers built with and the runtime libraries. A few questions:
1. Did you build and run this on the same container?
2. Can you provide the configuration file you use?
3. Does this reproduce also when running redis-server manually and not via systemd?
Comment From: daixijun
@yossigo
- yes, build and run on same container, but not Dockerfile, test ansible role using molecule
- redis.conf
bind 0.0.0.0
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 60
tls-auth-clients yes
tls-replication yes
tls-cluster yes
tls-protocols TLSv1.2
tls-prefer-server-ciphers yes
daemonize yes
supervised systemd
pidfile /var/run/redis_6379.pid
loglevel notice
logfile /var/log/redis/6379.log
databases 16
always-show-logo yes
save ""
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
rdb-del-sync-files no
dir /var/lib/redis/6379
replicaof 172.17.0.2 6379
masterauth test
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-diskless-load disabled
repl-disable-tcp-nodelay no
replica-priority 100
requirepass test
maxclients 10000
maxmemory 500mb
maxmemory-policy volatile-lru
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
appendonly yes
appendfilename appendonly.aof
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
- Running
redis-servermanually is the same crash
Comment From: daixijun
@yossigo @oranagra I think I have found the problem. The Redis instance does not have TLS enabled, But redis.conf contains TLS configuration.
tls-auth-clients yes
tls-replication yes
tls-cluster yes
tls-protocols TLSv1.2
tls-prefer-server-ciphers yes
After removing the above configuration from the redis.conf,It is working properly.
Comment From: yossigo
@daixijun The configuration you're using was overlooked - having cluster or (outgoing) replication TLS-enabled but not accepting inbound client/replica connections on a tls-port. As a workaround, make sure you have tls-port defined to some arbitrary port (even if you don't really want to support inbound TLS).
Will fix that.