any ideas on how to resolve this?
As it dumped core, could be interesting to "gdb it" and get a stackframe.
The server has ECC memory and there are no messages in the syslog about ECC failures. The SEGV occurred on two consecutive days when systemd was reloading so for the moment I find it unlikely that memory may be the root cause but looking at a core dump would certainly help. Where is it?
find / -name '*core*'The bt full of gdb seems to indicate that systemd chokes on a unit file. I bet it is clamav-freshclam. I am going to reboot now.
(gdb) bt full
#0 0x00007fec4596675b in kill () at ../sysdeps/unix/syscall-template.S:120
No locals.
#1 0x000055d907cfab1c in crash (sig=11) at ../src/core/main.c:257
sa = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {0 <repeats 16 times>}},
sa_flags = 0, sa_restorer = 0x0}
pid = <optimized out>
__func__ = "crash"
__PRETTY_FUNCTION__ = "crash"
#2 <signal handler called>
No locals.
#3 0x000055d907d74d73 in unit_active_state (u=<optimized out>) at ../src/core/unit.c:830
__PRETTY_FUNCTION__ = "unit_active_state"
#4 0x000055d907d84d86 in unit_may_gc (u=0x55d908ddfa00) at ../src/core/unit.c:392
state = <optimized out>
r = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__func__ = <optimized out>
#5 0x000055d907d739fe in unit_may_gc (u=0x55d908ddfa00) at ../src/core/unit.c:389
state = <optimized out>
r = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__func__ = <optimized out>
_found = <optimized out>
__assert_in_set = <optimized out>
_error = <optimized out>
_u = <optimized out>
_l = <optimized out>
#6 unit_add_to_gc_queue (u=0x55d908ddfa00) at ../src/core/unit.c:468
__PRETTY_FUNCTION__ = "unit_add_to_gc_queue"
#7 0x000055d907d85ab1 in unit_clear_dependencies (u=0x55d908da3000) at ../src/core/unit.c:555
other_deps = 0x0
other = 0x55d908ddfa00
deps = 0x55d908dc4878
__PRETTY_FUNCTION__ = <optimized out>
#8 unit_free (u=0x55d908da3000) at ../src/core/unit.c:698
slice = 0x55d908cb92a0
t = 0x0
__PRETTY_FUNCTION__ = <optimized out>
#9 0x000055d907d45d05 in unit_free (u=<optimized out>) at ../src/core/unit.c:654
slice = <optimized out>
t = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__unique_prefix_i15 = <optimized out>
j = <optimized out>
j = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
#10 manager_clear_jobs_and_units (m=m@entry=0x55d908cb78f0) at ../src/core/manager.c:1421
u = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
#11 0x000055d907d4b01a in manager_clear_jobs_and_units (m=0x55d908cb78f0) at ../src/core/manager.c:1418
u = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
#12 manager_reload (m=0x55d908cb78f0) at ../src/core/manager.c:3826
reloading = 0x0
fds = 0x7fec45112ba8
f = 0x55d908f6c940
r = <optimized out>
__PRETTY_FUNCTION__ = "manager_reload"
__func__ = "manager_reload"
#13 0x000055d907cf9a7b in invoke_main_loop (ret_error_message=0x7ffd4b3057f0, ret_switch_root_init=<synthetic pointer>,
ret_switch_root_dir=<synthetic pointer>, ret_fds=0x7ffd4b3057e0, ret_shutdown_verb=<synthetic pointer>,
ret_retval=<synthetic pointer>, ret_reexecute=<synthetic pointer>, saved_rlimit_memlock=0x7ffd4b305820,
saved_rlimit_nofile=0x7ffd4b305830, m=0x55d908cb78f0) at ../src/core/main.c:1934
saved_log_target = <optimized out>
saved_log_level = <optimized out>
r = <optimized out>
r = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__func__ = <optimized out>
_error = <optimized out>
_level = <optimized out>
_e = <optimized out>
saved_log_target = <optimized out>
saved_log_level = <optimized out>
_level = <optimized out>
_e = <optimized out>
_level = <optimized out>
_e = <optimized out>
_level = <optimized out>
_e = <optimized out>
_ptr_ = <optimized out>
_ptr_ = <optimized out>
_level = <optimized out>
_e = <optimized out>
table = <optimized out>
_level = <optimized out>
_e = <optimized out>
#14 main (argc=1, argv=0x7ffd4b305ab8) at ../src/core/main.c:2910
initrd_timestamp = {realtime = 0, monotonic = 0}
userspace_timestamp = {realtime = 1726568591653978, monotonic = 37908672}
kernel_timestamp = {realtime = 1726568553745306, monotonic = 0}
security_start_timestamp = {realtime = 1726568591655330, monotonic = 37910024}
security_finish_timestamp = {realtime = 1726568591655914, monotonic = 37910609}
saved_rlimit_nofile = {rlim_cur = 1024, rlim_max = 4096}
saved_rlimit_memlock = {rlim_cur = 65536, rlim_max = 65536}
skip_setup = <optimized out>
loaded_policy = false
queue_default_job = <optimized out>
first_boot = <optimized out>
reexecute = false
switch_root_dir = 0x0
switch_root_init = 0x0
before_startup = <optimized out>
after_startup = <optimized out>
timespan = "\000\000\000\000\000\000\000\000\000\303t\255\211[\367\r\000\000\000\000\000\000\000\000\310\375\377\377\377\377\377\377\002", '\000' <repeats 15 times>, "\002\000\000\000\000\000\000\000\006\000\000\000\000\000\000\200"
shutdown_verb = 0x0
error_message = 0x0
r = <optimized out>
retval = 1
m = 0x55d908cb78f0
fds = 0x0
finish = <optimized out>
systemd = "systemd"
__func__ = "main"
__PRETTY_FUNCTION__ = "main"
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
_head = <optimized out>
_item = <optimized out>
#10 manager_clear_jobs_and_units (m=m@entry=0x55d908cb78f0) at ../src/core/manager.c:1421
u = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
#11 0x000055d907d4b01a in manager_clear_jobs_and_units (m=0x55d908cb78f0) at ../src/core/manager.c:1418
u = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
#12 manager_reload (m=0x55d908cb78f0) at ../src/core/manager.c:3826
reloading = 0x0
fds = 0x7fec45112ba8
f = 0x55d908f6c940
r = <optimized out>
__PRETTY_FUNCTION__ = "manager_reload"
__func__ = "manager_reload"
#13 0x000055d907cf9a7b in invoke_main_loop (ret_error_message=0x7ffd4b3057f0, ret_switch_root_init=<synthetic pointer>,
ret_switch_root_dir=<synthetic pointer>, ret_fds=0x7ffd4b3057e0, ret_shutdown_verb=<synthetic pointer>,
ret_retval=<synthetic pointer>, ret_reexecute=<synthetic pointer>, saved_rlimit_memlock=0x7ffd4b305820,
saved_rlimit_nofile=0x7ffd4b305830, m=0x55d908cb78f0) at ../src/core/main.c:1934
saved_log_target = <optimized out>
saved_log_level = <optimized out>
r = <optimized out>
r = <optimized out>
__PRETTY_FUNCTION__ = <optimized out>
__func__ = <optimized out>
_error = <optimized out>
_level = <optimized out>
_e = <optimized out>
saved_log_target = <optimized out>
saved_log_level = <optimized out>
_level = <optimized out>
_e = <optimized out>
_level = <optimized out>
_e = <optimized out>
_level = <optimized out>
_e = <optimized out>
_ptr_ = <optimized out>
_ptr_ = <optimized out>
_level = <optimized out>
_e = <optimized out>
table = <optimized out>
_level = <optimized out>
_e = <optimized out>
#14 main (argc=1, argv=0x7ffd4b305ab8) at ../src/core/main.c:2910
initrd_timestamp = {realtime = 0, monotonic = 0}
userspace_timestamp = {realtime = 1726568591653978, monotonic = 37908672}
kernel_timestamp = {realtime = 1726568553745306, monotonic = 0}
security_start_timestamp = {realtime = 1726568591655330, monotonic = 37910024}
security_finish_timestamp = {realtime = 1726568591655914, monotonic = 37910609}
saved_rlimit_nofile = {rlim_cur = 1024, rlim_max = 4096}
saved_rlimit_memlock = {rlim_cur = 65536, rlim_max = 65536}
skip_setup = <optimized out>
loaded_policy = false
queue_default_job = <optimized out>
first_boot = <optimized out>
reexecute = false
switch_root_dir = 0x0
switch_root_init = 0x0
before_startup = <optimized out>
after_startup = <optimized out>
timespan = "\000\000\000\000\000\000\000\000\000\303t\255\211[\367\r\000\000\000\000\000\000\000\000\310\375\377\377\377\377\377\377\002", '\000' <repeats 15 times>, "\002\000\000\000\000\000\000\000\006\000\000\000\000\000\000\200"
shutdown_verb = 0x0
error_message = 0x0
r = <optimized out>
retval = 1
m = 0x55d908cb78f0
fds = 0x0
finish = <optimized out>
systemd = "systemd"
__func__ = "main"
__PRETTY_FUNCTION__ = "main"