I was trying to set up the notification-script for warning level events in sentinel, but even though the script was getting properly invoked manually, the sentinel was not triggering it status of redis server :
# Replication
role:master
connected_slaves:1
slave0:ip=10.45.16.154,port=6379,state=online,offset=1974075,lag=0
master_replid:f8525cb9113a6258d555205202e8888935a9321d
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:1974219
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:2097152
repl_backlog_first_byte_offset:1970183
repl_backlog_histlen:4037
# CPU
used_cpu_sys:58.32
used_cpu_user:38.95
used_cpu_sys_children:0.00
used_cpu_user_children:0.00
# Cluster
cluster_enabled:0
# Keyspace
[root@redis-notify-test-1-z2 :~]
[root@redis-notify-test-1-z2 :~] date ; systemctl stop redis
Tue Feb 11 05:29:54 UTC 2020
[root@redis-notify-test-1-z2 :~] date ; systemctl start redis
Tue Feb 11 05:31:28 UTC 2020
[root@redis-notify-test-1-z2 :~]
[root@redis-notify-test-2-z2 :~] redis-cli info server
# Server
redis_version:4.0.11
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:862464d33a44958a
redis_mode:standalone
os:Linux 3.10.0-1062.1.2.el7.x86_64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:4.8.5
process_id:52883
run_id:3bf51023de2b3aac5f457e532bb7c2b54835234d
tcp_port:6379
uptime_in_seconds:11
uptime_in_days:0
hz:10
lru_clock:4349280
executable:/mnt1/redis-4.0.11/src/redis-server
config_file:/mnt1/config/redis-server.conf
[root@redis-notify-test-2-z2 :~]
logs of the sentinel server , when i killed the dummy redis master process
sentinel 1
95853:X 11 Feb 05:30:24.237 # +sdown master notify-test-2 10.45.16.155 6379
95853:X 11 Feb 05:30:24.305 # +new-epoch 6
95853:X 11 Feb 05:30:24.311 # +vote-for-leader e48c0b40157d7b78f4c1512962247b25854fb448 6
95853:X 11 Feb 05:30:24.313 # +odown master notify-test-2 10.45.16.155 6379 #quorum 3/2
95853:X 11 Feb 05:30:24.313 # Next failover delay: I will not start a failover before Tue Feb 11 05:36:24 2020
95853:X 11 Feb 05:30:24.741 # +config-update-from sentinel e48c0b40157d7b78f4c1512962247b25854fb448 10.45.16.163 26379 @ notify-test-2 10.45.16.155 6379
95853:X 11 Feb 05:30:24.741 # +switch-master notify-test-2 10.45.16.155 6379 10.45.16.154 6379
95853:X 11 Feb 05:30:24.741 * +slave slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
95853:X 11 Feb 05:30:54.812 # +sdown slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
95853:X 11 Feb 05:31:28.992 # -sdown slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
95853:X 11 Feb 05:31:38.942 * +convert-to-slave slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
^C
[root@azrqa-sentinel-notify-test-sentinel-2-1-z2 10.45.16.167:~]
sentinel 2
59915:X 11 Feb 05:30:24.175 # +sdown master notify-test-2 10.45.16.155 6379
59915:X 11 Feb 05:30:24.353 # +new-epoch 6
59915:X 11 Feb 05:30:24.359 # +vote-for-leader e48c0b40157d7b78f4c1512962247b25854fb448 6
59915:X 11 Feb 05:30:24.740 # +config-update-from sentinel e48c0b40157d7b78f4c1512962247b25854fb448 10.45.16.163 26379 @ notify-test-2 10.45.16.155 6379
59915:X 11 Feb 05:30:24.740 # +switch-master notify-test-2 10.45.16.155 6379 10.45.16.154 6379
59915:X 11 Feb 05:30:24.741 * +slave slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
59915:X 11 Feb 05:30:54.742 # +sdown slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
59915:X 11 Feb 05:31:29.055 # -sdown slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
^C
[root@azrqa-sentinel-notify-test-sentinel-2-2-z1 10.45.16.168:~]
sentinel 3
60728:X 11 Feb 05:30:24.182 # +sdown master notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.283 # +odown master notify-test-2 10.45.16.155 6379 #quorum 2/2
60728:X 11 Feb 05:30:24.283 # +new-epoch 6
60728:X 11 Feb 05:30:24.283 # +try-failover master notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.294 # +vote-for-leader e48c0b40157d7b78f4c1512962247b25854fb448 6
60728:X 11 Feb 05:30:24.321 # 2681bfc94a6df96bab3348db4a1a65d3f523918e voted for e48c0b40157d7b78f4c1512962247b25854fb448 6
60728:X 11 Feb 05:30:24.366 # f6a931722ea76a80784496755e3a2de731e77343 voted for e48c0b40157d7b78f4c1512962247b25854fb448 6
60728:X 11 Feb 05:30:24.366 # +elected-leader master notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.366 # +failover-state-select-slave master notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.421 # +selected-slave slave 10.45.16.154:6379 10.45.16.154 6379 @ notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.421 * +failover-state-send-slaveof-noone slave 10.45.16.154:6379 10.45.16.154 6379 @ notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.578 * +failover-state-wait-promotion slave 10.45.16.154:6379 10.45.16.154 6379 @ notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.657 # +promoted-slave slave 10.45.16.154:6379 10.45.16.154 6379 @ notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.657 # +failover-state-reconf-slaves master notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.732 # +failover-end master notify-test-2 10.45.16.155 6379
60728:X 11 Feb 05:30:24.732 # +switch-master notify-test-2 10.45.16.155 6379 10.45.16.154 6379
60728:X 11 Feb 05:30:24.732 * +slave slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
60728:X 11 Feb 05:30:54.777 # +sdown slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
60728:X 11 Feb 05:31:29.083 # -sdown slave 10.45.16.155:6379 10.45.16.155 6379 @ notify-test-2 10.45.16.154 6379
^C
[root@azrqa-sentinel-notify-test-sentinel-2-3-z3 10.45.16.163:~]
[root@azrqa-sentinel-notify-test-sentinel-2-2-z1 10.45.16.168:~] senti-cli info server
# Server
redis_version:4.0.11
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:862464d33a44958a
redis_mode:sentinel
os:Linux 3.10.0-1062.1.2.el7.x86_64 x86_64
arch_bits:64
in the conf i have added the location of the script
[root@azrqa-sentinel-notify-test-sentinel-2-2-z1 10.45.16.168:~] cat /mnt1/config/redis-sentinel.conf
protected-mode no
port 26379
dir "/mnt1/data/redis-sentinel"
logfile "/mnt1/logs/redis-sentinel.log"
sentinel myid f6a931722ea76a80784496755e3a2de731e77343
# Generated by CONFIG REWRITE
maxclients 4064
supervised systemd
sentinel deny-scripts-reconfig no
sentinel monitor notify-test-2 10.45.16.155 6379 2
sentinel notification-script notify-test-2 /mnt1/scripts/notify.sh
sentinel client-reconfig-script notify-test-2 /mnt1/scripts/notify.sh
sentinel config-epoch notify-test-2 12
sentinel leader-epoch notify-test-2 12
sentinel known-slave notify-test-2 10.45.16.154 6379
sentinel known-sentinel notify-test-2 10.45.16.167 26379 2681bfc94a6df96bab3348db4a1a65d3f523918e
sentinel known-sentinel notify-test-2 10.45.16.163 26379 e48c0b40157d7b78f4c1512962247b25854fb448
sentinel current-epoch 12
[root@azrqa-sentinel-notify-test-sentinel-2-2-z1 10.45.16.168:~]
sentinel notification-script notify-test-2 /mnt1/scripts/notify.sh
but still there is no mention in the logs "/mnt1/logs/redis-sentinel.log"
I m not aware of any other logging location , in this regard to dig further in this regard apart from the one mentioned above.
Comment From: hakimkartik
I also tried upgrading the redis binary version
[root@azrqa-sentinel-notify-test-sentinel-2-1-z2 10.45.16.167:~] senti-cli info server
# Server
redis_version:5.0.7
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:91145edf25c40cd7
redis_mode:sentinel
after that though the script was getting invoked whenever I was killing the master ```32028:X 11 Feb 2020 07:46:42.586 # -script-error /mnt1/scripts/notify.sh 0 2 32028:X 11 Feb 2020 07:46:42.586 # -script-error /mnt1/scripts/notify.sh 0 2 32028:X 11 Feb 2020 07:46:42.586 # -script-error /mnt1/scripts/notify.sh 0 2 32028:X 11 Feb 2020 07:46:42.586 # -script-error /mnt1/scripts/notify.sh 0 2 32028:X 11 Feb 2020 07:46:42.657 # -script-error /mnt1/scripts/notify.sh 0 2
32028:X 11 Feb 2020 07:47:12.586 # +sdown slave 10.45.16.154:6379 10.45.16.154 6379 @ notify-test-2 10.45.16.155 6379 32028:X 11 Feb 2020 07:47:12.649 # -script-error /mnt1/scripts/notify.sh 0 2
^C [root@azrqa-sentinel-notify-test-sentinel-2-1-z2 10.45.16.167:~]
but it was showing exit code as `2`.
Even when the script was practically empty and exitting with `exit 0;`
[root@azrqa-sentinel-notify-test-sentinel-2-1-z2 10.45.16.167:~] cat /mnt1/scripts/notify.sh
exit 0; [root@azrqa-sentinel-notify-test-sentinel-2-1-z2 10.45.16.167:~] ```
Comment From: hakimkartik
And user permissions of the sentinel process, as well as notification scripts, are identical