Notice! - If a Redis module was involved, please open an issue in the module's repo instead! - If you're using docker on Apple M1, please make sure the image you're using was compiled for ARM!
Crash report
Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context.
(gdb) bt full
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
set = {__val = {0, 139748469180824, 0, 140734866257039, 3616731346761019888, 128, 140734866256976, 2, 63, 140734866256368,
94322896223653, 140734866256752, 140734866256976, 139748471443354, 140734866256944, 139748472313904}}
pid = <optimized out>
tid = <optimized out>
ret = <optimized out>
#1 0x00007f19b9f8f859 in __GI_abort () at abort.c:79
save_stage = 1
act = {__sigaction_handler = {sa_handler = 0x7fff63b5f43f, sa_sigaction = 0x7fff63b5f43f}, sa_mask = {__val = {
4199936040484592316, 18446744073709545184, 94322896266112, 139743462238400, 140734866262240, 4199936040296897212,
4288720840637012668, 0, 94322919711168, 0, 0, 94322896243096, 47, 140734866262240, 139748471467637, 0}},
sa_flags = -330405376, sa_restorer = 0x7fff00000001}
sigs = {__val = {32, 0 <repeats 15 times>}}
#2 0x00007f19b9ff9fcc in __libc_message (action=do_abort, fmt=0x7f19ba123ebe "%s", fmt=0x7f19ba123ebe "%s", action=do_abort)
at ../sysdeps/posix/libc_fatal.c:155
ap = {{gp_offset = 24, fp_offset = 32537, overflow_arg_area = 0x7fff63b5f480, reg_save_area = 0x7fff63b5f420}}
fd = <optimized out>
list = 0x0
nlist = 1
cp = 0x7f19ba123ec0 ""
#3 0x00007f19b9ffa2e0 in __GI___libc_fatal (
message=message@entry=0x7f19ba1790d8 "libgcc_s.so.1 must be installed for pthread_cancel to work\n")
at ../sysdeps/posix/libc_fatal.c:164
No locals.
#4 0x00007f19ba175ad9 in pthread_cancel_init () at ../sysdeps/nptl/unwind-forcedunwind.c:65
resume = <optimized out>
personality = <optimized out>
forcedunwind = <optimized out>
getcfa = <optimized out>
handle = 0x0
#5 0x00007f19ba1720a4 in __pthread_cancel (th=139748453226240) at pthread_cancel.c:38
pd = 0x7f19b8e99700
result = <optimized out>
oldval = <optimized out>
newval = <optimized out>
#6 0x000055c942a866e5 in bioKillThreads () at bio.c:313
err = <optimized out>
j = 0
#7 0x000055c942a6d9ef in killThreads () at debug.c:1875
No locals.
#8 doFastMemoryTest () at debug.c:1884
No locals.
--Type <RET> for more, q to quit, c to continue without paging-- c
#9 0x000055c942a70b7f in sigsegvHandler (sig=6, info=0x7fff63b5f670, secret=0x7fff63b5f540) at debug.c:1982
uc = 0x7fff63b5f540
eip = 0x7f19b9fb000b <__GI_raise+203>
#10 <signal handler called>
No locals.
#11 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
set = {__val = {0, 139748469180824, 0, 140734866261311, 11, 128, 2, 80, 139748465213248, 94322895730709, 94322897245392, 139748469180328, 94321776787458, 139745350909952, 94321776787457, 139748472313904}}
pid = <optimized out>
tid = <optimized out>
ret = <optimized out>
#12 0x00007f19b9f8f859 in __GI_abort () at abort.c:79
save_stage = 1
act = {__sigaction_handler = {sa_handler = 0x7fff63b604ef, sa_sigaction = 0x7fff63b604ef}, sa_mask = {__val = {4199936044253174460, 18446744073709545184, 94322896266112, 139743462238400, 140734866262240, 4199936044099033788, 4288720840637012668, 0, 94322919711168, 0, 0, 94322896243096, 47, 140734866262240, 139748471467637, 0}}, sa_flags = -330405376, sa_restorer = 0x7fff00000001}
sigs = {__val = {32, 0 <repeats 15 times>}}
#13 0x00007f19b9ff9fcc in __libc_message (action=do_abort, fmt=0x7f19ba123ebe "%s", fmt=0x7f19ba123ebe "%s", action=do_abort) at ../sysdeps/posix/libc_fatal.c:155
ap = {{gp_offset = 24, fp_offset = 32537, overflow_arg_area = 0x7fff63b60530, reg_save_area = 0x7fff63b604d0}}
fd = <optimized out>
list = 0x0
nlist = 1
cp = 0x7f19ba123ec0 ""
#14 0x00007f19b9ffa2e0 in __GI___libc_fatal (message=message@entry=0x7f19ba1790d8 "libgcc_s.so.1 must be installed for pthread_cancel to work\n") at ../sysdeps/posix/libc_fatal.c:164
No locals.
#15 0x00007f19ba175ad9 in pthread_cancel_init () at ../sysdeps/nptl/unwind-forcedunwind.c:65
resume = <optimized out>
personality = <optimized out>
forcedunwind = <optimized out>
getcfa = <optimized out>
handle = 0x0
#16 0x00007f19ba1720a4 in __pthread_cancel (th=139748453226240) at pthread_cancel.c:38
pd = 0x7f19b8e99700
result = <optimized out>
oldval = <optimized out>
newval = <optimized out>
#17 0x000055c942a866e5 in bioKillThreads () at bio.c:313
err = <optimized out>
j = 0
#18 0x000055c942a6d9ef in killThreads () at debug.c:1875
No locals.
#19 doFastMemoryTest () at debug.c:1884
No locals.
#20 0x000055c942a6dec3 in _serverPanic (file=0x55c942b83707 "server.c", line=6580, msg=<optimized out>) at debug.c:1107
ap = {{gp_offset = 32, fp_offset = 48, overflow_arg_area = 0x7fff63b607c0, reg_save_area = 0x7fff63b606f0}}
fmtmsg = "Redis aborting for OUT OF MEMORY. Allocating 528443 bytes!\000\000\000\000\000\000<\000\000\000\000\000\000\000\327\006\266c\377\177\000\000o\006\266c\377\177\000\000o\006\266c\377\177\000\000\250\aݹ\031\177\000\000\000jN\354<\000\000\000o\006\266c\377\177\000\000\000\020\n\000\000\000\000\000`\205;S\031\177\000\000\017\251\241B\311U\000\001\240\262\300B\311U\000\000\000jN췐i\277\033\070\270B\311U\000\000\000\000\000\000\000\000\000\000\300\t\240\271\031\177\000\000\250\aݹ\031\177\000\000\000\000\n\000\000\000\000\000@\000\000\000\000\000\000\000"...
#21 0x000055c942a0961f in redisOutOfMemoryHandler (allocation_size=528443) at server.c:6580
No locals.
#22 0x000055c942a1a9ae in zmalloc_usable (size=size@entry=528443, usable=usable@entry=0x7fff63b607f0) at zmalloc.c:143
ptr = 0x0
#23 0x000055c942a27bec in _addReplyProtoToList (c=0x7f1988555a00, s=0x7f188f6d28c0 "\\Inventory\\Entity\\InventoryItem", len=528427) at networking.c:369
usable_size = 0
size = 528427
ln = <optimized out>
tail = <optimized out>
#24 0x000055c942a27d3f in addReply (c=0x7f1988555a00, obj=0x7f195cce4380) at networking.c:416
buf = "\000\200\211\271\031\177\000\000WX\243B\311U\000\000\200\372\006!\031\177\000\000\017\000\000\000\000\000\000"
len = <optimized out>
#25 0x000055c942a29bb4 in addReplyBulk (c=0x7f1988555a00, obj=0x7f195cce4380) at networking.c:993
No locals.
#26 0x000055c942a49203 in getGenericCommand (c=0x7f1988555a00) at t_string.c:324
o = 0x7f195cce4380
#27 0x000055c942a0d90f in call (c=0x7f1988555a00, flags=15) at server.c:3319
dirty = <optimized out>
client_old_flags = 0
real_cmd = 0x55c942c2fcb0 <redisCommandTable+114576>
prev_core_propagates = <optimized out>
call_timer = 1670842699209995
monotonic_start = 0
duration = <optimized out>
zmalloc_used = <optimized out>
#28 0x000055c942a0ee1d in processCommand (c=c@entry=0x7f1988555a00) at server.c:3953
err = 0x7f18d720c6c5 "*2\r\n$3\r\nGET\r\n$32\r\ninventory_76561199381052974_csgo\r\n"
cmd_flags = 16386
is_read_command = <optimized out>
is_write_command = <optimized out>
is_denyoom_command = <optimized out>
is_denystale_command = <optimized out>
is_denyloading_command = <optimized out>
is_may_replicate_command = <optimized out>
is_deny_async_loading_command = 0
obey_client = <optimized out>
acl_errpos = 1119400080
acl_retval = 0
deny_write_type = <optimized out>
#29 0x000055c942a26630 in processCommandAndResetClient (c=c@entry=0x7f1988555a00) at networking.c:2444
deadclient = 0
old_client = 0x0
#30 0x000055c942a293f8 in processInputBuffer (c=0x7f1988555a00) at networking.c:2548
No locals.
#31 0x000055c942a2cbd8 in readQueryFromClient (conn=<optimized out>) at networking.c:2684
c = 0x7f1988555a00
nread = 52
big_arg = <optimized out>
qblen = <optimized out>
readlen = 20474
#32 0x000055c942ad630c in callHandler (handler=<optimized out>, conn=0x7f199a3d8900) at connhelpers.h:79
No locals.
#33 connSocketEventHandler (el=<optimized out>, fd=<optimized out>, clientData=0x7f199a3d8900, mask=<optimized out>) at connection.c:310
conn = 0x7f199a3d8900
invert = 0
call_write = <optimized out>
call_read = 1
#34 0x000055c942a048ca in aeProcessEvents (eventLoop=eventLoop@entry=0x7f19b982b0f0, flags=flags@entry=27) at ae.c:436
fd = <optimized out>
mask = 1
invert = <optimized out>
fe = <optimized out>
fired = 0
j = 0
tv = {tv_sec = 0, tv_usec = 8869}
tvp = <optimized out>
usUntilTimer = <optimized out>
processed = 0
numevents = 1
#35 0x000055c942a04c8d in aeMain (eventLoop=0x7f19b982b0f0) at ae.c:496
No locals.
#36 0x000055c942a007ba in main (argc=<optimized out>, argv=0x7fff63b60c48) at server.c:7075
tv = {tv_sec = 1670723146, tv_usec = 502223}
j = <optimized out>
config_from_stdin = <optimized out>
hashseed = "\021\215\020\235>\242u\325\373\203 \302\016\347\232~"
exec_name = <optimized out>
background = <optimized out>
(gdb)
Additional information
- OS distribution and version - ubuntu 20.04
- Steps to reproduce (if any)
Comment From: shipitsin-ilia
in gdb backtrace I see fmtmsg = "Redis aborting for OUT OF MEMORY., but I do not see that in redis log, shouldn't it be there ?
btw, I noticed that 7.0.6 was released today, I'll try it
Comment From: oranagra
yes, the OOM panic should have been in the log. did you get a crash log? or did running it in gdb prevented that? i wonder how much memory redis consumed and and how much memory was on the OS.
the one thing that's new in 7.0.6 in that regard, is an improved warning about bad overcommit setting, see https://github.com/redis/redis/pull/11357 this can cause OOM panic even if there is enough memory see https://github.com/jemalloc/jemalloc/issues/1328
Comment From: shipitsin-ilia
I have core dump, redis binary. Thus I was able to extract "bt full" (see above). I did not get anything like "out of memory" in redis logs.
I have full "redis-server.log", I can share it privately
Comment From: shipitsin-ilia
redis was running on itself (supervised by systemd as usual), no gdb.
Comment From: shipitsin-ilia
as for warning, I'm afraid nobody will look at logs as soon as redis starts and ready to serve requests. if it is possible to refuse to start in that case, it might get an attention
Comment From: oranagra
please check and report what value you have with /proc/sys/vm/overcommit_memory and which version of jemalloc you're using (INFO MEMORY).
you're saying that you don't see something similar to these prints in the log:
Out Of Memory allocating %zu bytes!
------------------------------------------------
!!! Software Failure. Press left mouse button to continue
Guru Meditation: Redis aborting for OUT OF MEMORY. Allocating %zu bytes!
------ STACK TRACE ------
you're welcome to send the log over to oran@redis.com
Comment From: shipitsin-ilia
$ cat /proc/sys/vm/overcommit_memory
2
$
indeed, there are such messages, I did not notice them earlier
# grep 'Out Of Memory allocating' redis-server.log
123965:M 11 Dec 2022 01:45:37.419 # Out Of Memory allocating 2046331 bytes!
28673:M 12 Dec 2022 10:58:19.210 # Out Of Memory allocating 528443 bytes!
#
as for jemalloc, I cannot check now, it was redis-7.0.5, we updated to 7.0.6, it is not possible to install 7.0.5, it is gone from PPA.
log sent
Comment From: oranagra
your overcommit_memory setting is bad (as indicated by the above jemalloc issue i shared).
it causes VM fragmentation which then leads to being unable to allocate memory (despite having free memory).
you may be able to tweak that using vm.max_map_count as suggested in #10234, but the right solution is to change overcommit_memory to 1 (as advised by redis on startup)
Comment From: shipitsin-ilia
thanks. we'll review overcommit settings.