I am trying to make Sentinel work, but it's not connecting and I have no errors. The setup is as follows:

  • M1-S1
  • R1-S2
  • R2-S3
  • R3-S4

Sentinel and Redis are on the same box. However, S1 won't connect to M1; it is always seen as sdown. Is there anything I am missing? Even the logs, which are set to loglevel debug, are not helpful.

The following commands work: 

redis-cli -h x.x.x.x -p 26379 --cert redis.local.ca.pem --key redis.local.ca.key --cacert localchain.crt --tls
redis-cli -h x.x.x.x -p 6379 --cert redis.local.ca.pem --key redis.local.ca.key --cacert localchain.crt --tls

The certificates are okay, as verified with: 

openssl verify -CAfile localchain.crt redis.local.ca.pem

The only issue I can see is that there are a high number of password failures, but this is not showing in the redis-sentinel.log of M1. This could mean that the other boxes are trying to connect.

Thank you.

root@redis1:/var/log/redis# tail -f redis-sentinel.log
12679:X 14 Mar 2024 15:36:33.355 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
12679:X 14 Mar 2024 15:36:33.355 * Redis version=7.2.4, bits=64, commit=00000000, modified=0, pid=12679, just started
12679:X 14 Mar 2024 15:36:33.355 * Configuration loaded
12679:X 14 Mar 2024 15:36:33.356 * monotonic clock: POSIX clock_gettime
12679:X 14 Mar 2024 15:36:33.357 * Running mode=sentinel, port=26379.
12679:X 14 Mar 2024 15:36:33.361 * Sentinel ID is 8083be1d0c1fe54debd1aa300539b203c752c9e6
12679:X 14 Mar 2024 15:36:33.361 # +monitor master mymaster x.x.x.x 6379 quorum 3
12679:X 14 Mar 2024 15:37:03.387 # +sdown master mymaster x.x.x.x 6379

redis.conf


protected-mode no 
port 0
tcp-backlog 511
timeout 0
tcp-keepalive 300
tls-port 6379
tls-cert-file '/etc/redis/redis.local.ca.pem'
tls-key-file '/etc/redis/redis.local.ca.key'
tls-ca-cert-file '/etc/redis/localchain.crt'
tls-auth-clients yes
tls-replication yes
tls-cluster yes
daemonize yes
supervised systemd
pidfile "/run/redis/redis-server.pid"
loglevel notice
logfile "/var/log/redis/redis-server.log"
databases 16
always-show-logo no
set-proc-title yes
proc-title-template "{title} {listen-addr} {server-mode}"
locale-collate ""
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename "dump.rdb"
rdb-del-sync-files no
dir "/var/lib/redis"
masterauth "somepassword"
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync yes
repl-diskless-sync-delay 5
repl-diskless-sync-max-replicas 0
repl-diskless-load disabled
repl-disable-tcp-nodelay no
replica-priority 100
acllog-max-len 128
aclfile /etc/redis/users.acl
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
lazyfree-lazy-user-del no
lazyfree-lazy-user-flush no
oom-score-adj no
oom-score-adj-values 0 200 800
disable-thp yes
appendonly no
appendfilename "appendonly.aof"
appenddirname "appendonlydir"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
aof-timestamp-enabled no
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-listpack-entries 512
hash-max-listpack-value 64
list-max-listpack-size -2
list-compress-depth 0
set-max-intset-entries 512
set-max-listpack-entries 128
set-max-listpack-value 64
zset-max-listpack-entries 128
zset-max-listpack-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4kb
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
jemalloc-bg-thread yes

sentinel.conf


protected-mode no
port 0
daemonize yes
supervised systemd
pidfile "/run/sentinel/redis-sentinel.pid"
loglevel debug 
logfile "/var/log/redis/redis-sentinel.log"
dir "/var/lib/redis"
sentinel monitor mymaster x.x.x.x 6379 3
sentinel auth-pass mymaster mypassword
sentinel auth-user mymaster sentinel-user
acllog-max-len 128
aclfile "/etc/redis/sentinel-users.acl"
sentinel deny-scripts-reconfig yes
sentinel resolve-hostnames no
sentinel announce-hostnames no
tls-replication yes
tls-cert-file '/etc/redis/redis.bellsolutionstech.ca.pem'
tls-key-file '/etc/redis/redis.bellsolutionstech.ca.key'
tls-ca-cert-file '/etc/redis/bellchain.crt'
tls-port 26379

# Generated by CONFIG REWRITE
latency-tracking-info-percentiles 50 99 99.9
sentinel myid 8083be1d0c1fe54debd1aa300539b203c752c9e6
sentinel config-epoch mymaster 0
sentinel leader-epoch mymaster 0
sentinel current-epoch 0

Info form Redis


# Server
redis_version:7.2.4
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:44edf16bebe6fed2
redis_mode:standalone
os:Linux 5.15.0-100-generic x86_64
arch_bits:64
monotonic_clock:POSIX clock_gettime
multiplexing_api:epoll
atomicvar_api:c11-builtin
gcc_version:11.4.0
process_id:12727
process_supervised:systemd
run_id:238943f2efbe2b569573341fe569e7e0274b047e
tcp_port:6379
server_time_usec:1710450175365459
uptime_in_seconds:4775
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:15951359
executable:/usr/bin/redis-server
config_file:/etc/redis/redis.conf
io_threads_active:0
listener2:name=tls,bind=*,bind=-::*,port=6379

# Clients
connected_clients:2
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:20480
client_recent_max_output_buffer:20504
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
total_blocking_keys:0
total_blocking_keys_on_nokey:0

# Memory
used_memory:1058568
used_memory_human:1.01M
used_memory_rss:14028800
used_memory_rss_human:13.38M
used_memory_peak:1118128
used_memory_peak_human:1.07M
used_memory_peak_perc:94.67%
used_memory_overhead:932260
used_memory_startup:866664
used_memory_dataset:126308
used_memory_dataset_perc:65.82%
allocator_allocated:1180312
allocator_active:1462272
allocator_resident:4227072
total_system_memory:8323686400
total_system_memory_human:7.75G
used_memory_lua:31744
used_memory_vm_eval:31744
used_memory_lua_human:31.00K
used_memory_scripts_eval:0
number_of_cached_scripts:0
number_of_functions:0
number_of_libraries:0
used_memory_vm_functions:32768
used_memory_vm_total:64512
used_memory_vm_total_human:63.00K
used_memory_functions:184
used_memory_scripts:184
used_memory_scripts_human:184B
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.24
allocator_frag_bytes:281960
allocator_rss_ratio:2.89
allocator_rss_bytes:2764800
rss_overhead_ratio:3.32
rss_overhead_bytes:9801728
mem_fragmentation_ratio:13.28
mem_fragmentation_bytes:12972056
mem_not_counted_for_evict:0
mem_replication_backlog:20508
mem_total_replication_buffers:20504
mem_clients_slaves:0
mem_clients_normal:44800
mem_cluster_links:0
mem_aof_buffer:0
mem_allocator:jemalloc-5.3.0
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

# Persistence
loading:0
async_loading:0
current_cow_peak:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1710445400
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_saves:0
rdb_last_cow_size:0
rdb_last_load_keys_expired:0
rdb_last_load_keys_loaded:1
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_rewrites:0
aof_rewrites_consecutive_failures:0
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0

# Stats
total_connections_received:5071
total_commands_processed:9835
instantaneous_ops_per_sec:2
total_net_input_bytes:2080733
total_net_output_bytes:1187278
total_net_repl_input_bytes:0
total_net_repl_output_bytes:6650
instantaneous_input_kbps:0.50
instantaneous_output_kbps:0.28
instantaneous_input_repl_kbps:0.00
instantaneous_output_repl_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:1
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:42
evicted_keys:0
evicted_clients:0
total_eviction_exceeded_time:0
current_eviction_exceeded_time:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
pubsubshard_channels:0
latest_fork_usec:0
total_forks:0
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
total_active_defrag_time:0
current_active_defrag_time:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:30091
dump_payload_sanitizations:0
total_reads_processed:19976
total_writes_processed:10619
io_threaded_reads_processed:0
io_threaded_writes_processed:0
reply_buffer_shrinks:10
reply_buffer_expands:1
eventloop_cycles:85002
eventloop_duration_sum:26214632
eventloop_duration_cmd_sum:93047
instantaneous_eventloop_cycles_per_sec:17
instantaneous_eventloop_duration_usec:174
acl_access_denied_auth:5066
acl_access_denied_cmd:0
acl_access_denied_key:0
acl_access_denied_channel:0

# Replication
role:master
connected_slaves:1
slave0:ip=x.x.x.x,port=6379,state=online,offset=89026,lag=0
master_failover_state:no-failover
master_replid:07414e54dcf2cdbddd69ef84c071b35d80cd0631
master_replid2:326623742c3b526c62fdf16d518ba51150a58d2c
master_repl_offset:89026
second_repl_offset:82377
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:82377
repl_backlog_histlen:6650

# CPU
used_cpu_sys:5.164023
used_cpu_user:21.537500
used_cpu_sys_children:0.000000
used_cpu_user_children:0.000000
used_cpu_sys_main_thread:5.158951
used_cpu_user_main_thread:21.540887

# Modules

# Errorstats
errorstat_ERR:count=1
errorstat_NOAUTH:count=25024
errorstat_WRONGPASS:count=5066

# Cluster
cluster_enabled:0

# Keyspace
db0:keys=1,expires=0,avg_ttl=0

Info from Sentinel

# Server
redis_version:7.2.4
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:44edf16bebe6fed2
redis_mode:sentinel
os:Linux 5.15.0-100-generic x86_64
arch_bits:64
monotonic_clock:POSIX clock_gettime
multiplexing_api:epoll
atomicvar_api:c11-builtin
gcc_version:11.4.0
process_id:12679
process_supervised:systemd
run_id:11bf9efbafaaf6729ab3cf232cd03b129d5cfded
tcp_port:26379
server_time_usec:1710450377671348
uptime_in_seconds:5384
uptime_in_days:0
hz:18
configured_hz:10
lru_clock:15951561
executable:/usr/bin/redis-sentinel
config_file:/etc/redis/sentinel.conf
io_threads_active:0
listener2:name=tls,bind=*,bind=-::*,port=26379

# Clients
connected_clients:1
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:20480
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
total_blocking_keys:0
total_blocking_keys_on_nokey:0

# Stats
total_connections_received:4
total_commands_processed:4
instantaneous_ops_per_sec:0
total_net_input_bytes:803
total_net_output_bytes:4082
total_net_repl_input_bytes:0
total_net_repl_output_bytes:0
instantaneous_input_kbps:0.00
instantaneous_output_kbps:0.00
instantaneous_input_repl_kbps:0.00
instantaneous_output_repl_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:70
evicted_keys:0
evicted_clients:0
total_eviction_exceeded_time:0
current_eviction_exceeded_time:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
pubsubshard_channels:0
latest_fork_usec:0
total_forks:0
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
total_active_defrag_time:0
current_active_defrag_time:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:13
dump_payload_sanitizations:0
total_reads_processed:20
total_writes_processed:17
io_threaded_reads_processed:0
io_threaded_writes_processed:0
reply_buffer_shrinks:5
reply_buffer_expands:1
eventloop_cycles:115668
eventloop_duration_sum:25907087
eventloop_duration_cmd_sum:133
instantaneous_eventloop_cycles_per_sec:21
instantaneous_eventloop_duration_usec:122
acl_access_denied_auth:0
acl_access_denied_cmd:0
acl_access_denied_key:0
acl_access_denied_channel:0

# CPU
used_cpu_sys:7.448594
used_cpu_user:18.878832
used_cpu_sys_children:0.000000
used_cpu_user_children:0.000000
used_cpu_sys_main_thread:7.448011
used_cpu_user_main_thread:18.877352

# Sentinel
sentinel_masters:1
sentinel_tilt:0
sentinel_tilt_since_seconds:-1
sentinel_running_scripts:0
sentinel_scripts_queue_length:0
sentinel_simulate_failure_flags:0
master0:name=mymaster,status=sdown,address=x.x.x.x:6379,slaves=0,sentinels=1

Comment From: zulander1

Finally, it was a wrong user/password