Apr 13 17:20:03 controlhost kernel: slurmctld_srvcn[23331]: segfault at 0 ip 000000367aa8967e sp 00002b4599d0bb18 error 4 in libc-2.12.so[367aa00000+18a000] core.7425 #0 0x000000367aa8967e in memcpy () from /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install slurm-15.08.4-1.el6.x86_64 (gdb) bt #0 0x000000367aa8967e in memcpy () from /lib64/libc.so.6 #1 0x0000000000509ce2 in packmem (valp=0x0, size_val=31, buffer=0x2b45b037f350) at pack.c:623 #2 0x0000000000496589 in pack_part (part_ptr=0x2a284d0, buffer=0x2b45b037f350, protocol_version=7424) at partition_mgr.c:1205 #3 0x0000000000495daa in pack_all_part (buffer_ptr=0x2b4599d1bcc8, buffer_size=0x2b4599d1bcdc, show_flags=1, uid=296, protocol_version=7424) at partition_mgr.c:1136 #4 0x00000000004a3052 in _slurm_rpc_dump_partitions (msg=0x2b45b000e700) at proc_req.c:1712 #5 0x000000000049e6ce in slurmctld_req (msg=0x2b45b000e700, arg=0x2b459c005fc0) at proc_req.c:324 #6 0x000000000043e1a3 in _service_connection (arg=0x2b459c005fc0) at controller.c:1096 #7 0x000000367b2079d1 in start_thread () from /lib64/libpthread.so.0 #8 0x000000367aae88fd in clone () from /lib64/libc.so.6 (gdb) info thread 41 Thread 0x2b4598706700 (LWP 23286) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 40 Thread 0x2b4599110700 (LWP 23297) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 39 Thread 0x2b4598908700 (LWP 23309) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 38 Thread 0x2b4598605700 (LWP 23267) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 37 Thread 0x2b459b332700 (LWP 23322) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 36 Thread 0x2b45ac403700 (LWP 23269) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 35 Thread 0x2b4599a19700 (LWP 23311) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 34 Thread 0x2b459b433700 (LWP 23324) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 33 Thread 0x2b4598201700 (LWP 7688) 0x000000367aae1373 in select () from /lib64/libc.so.6 32 Thread 0x2b459a120700 (LWP 23314) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 31 Thread 0x2b4599c1b700 (LWP 23312) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 30 Thread 0x2b4598b0a700 (LWP 23285) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 29 Thread 0x2b459b02f700 (LWP 23319) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 28 Thread 0x2b4598807700 (LWP 23308) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 27 Thread 0x2b459ac2b700 (LWP 23284) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 26 Thread 0x2b459b534700 (LWP 23325) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 25 Thread 0x2b459b231700 (LWP 23321) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 24 Thread 0x2b4598403700 (LWP 7690) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 23 Thread 0x2b459a221700 (LWP 23298) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 22 Thread 0x2b45ae322700 (LWP 23332) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 21 Thread 0x2b4599817700 (LWP 23293) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 20 Thread 0x2b4598d0c700 (LWP 23270) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 19 Thread 0x2b4598e0d700 (LWP 23282) 0x000000367aaaca3d in nanosleep () from /lib64/libc.so.6 18 Thread 0x2b459900f700 (LWP 23283) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 17 Thread 0x2b458d964700 (LWP 7686) 0x000000367b20822d in pthread_join () from /lib64/libpthread.so.0 16 Thread 0x2b459b635700 (LWP 23326) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 15 Thread 0x2b459af2e700 (LWP 23318) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 14 Thread 0x2b4598100700 (LWP 7687) 0x000000367b20b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 13 Thread 0x2b4598c0b700 (LWP 23281) 0x000000367b20822d in pthread_join () from /lib64/libpthread.so.0 12 Thread 0x2b458c100700 (LWP 7431) 0x000000367b20b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 11 Thread 0x2b4598f0e700 (LWP 23271) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 10 Thread 0x2b458d863700 (LWP 7685) 0x000000367aaaca3d in nanosleep () from /lib64/libc.so.6 9 Thread 0x2b459a423700 (LWP 23306) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 8 Thread 0x2b4599211700 (LWP 23305) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 7 Thread 0x2b4587e41700 (LWP 7428) 0x000000367aaaca3d in nanosleep () from /lib64/libc.so.6 6 Thread 0x2b459a322700 (LWP 23330) 0x000000367b20f82e in __libc_sigaction () from /lib64/libpthread.so.0 5 Thread 0x2b4598302700 (LWP 7689) 0x000000367b20f4b5 in sigwait () from /lib64/libpthread.so.0 4 Thread 0x2b458ce8f700 (LWP 7684) 0x000000367b20b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 3 Thread 0x2b4587f42700 (LWP 7429) 0x000000367b20822d in pthread_join () from /lib64/libpthread.so.0 2 Thread 0x2b4585cc86e0 (LWP 7425) 0x000000000050517e in bit_test (b=0x2d2a700, bit=69) at bitstring.c:236 * 1 Thread 0x2b4599d1c700 (LWP 23331) 0x000000367aa8967e in memcpy () from /lib64/libc.so.6 Apr 16 06:18:02 controlhost kernel: slurmctld_srvcn[11009]: segfault at 0 ip 000000367aa8967e sp 00002b5f248f7b18 error 4 in libc-2.12.so[367aa00000+18a000] core.23652 Program terminated with signal 11, Segmentation fault. #0 0x000000367aa8967e in memcpy () from /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install slurm-15.08.4-1.el6.x86_64 (gdb) bt #0 0x000000367aa8967e in memcpy () from /lib64/libc.so.6 #1 0x0000000000509ce2 in packmem (valp=0x0, size_val=27, buffer=0x2b6074140400) at pack.c:623 #2 0x0000000000496589 in pack_part (part_ptr=0x19f9270, buffer=0x2b6074140400, protocol_version=7424) at partition_mgr.c:1205 #3 0x0000000000495daa in pack_all_part (buffer_ptr=0x2b5f24907cc8, buffer_size=0x2b5f24907cdc, show_flags=1, uid=1531101, protocol_version=7424) at partition_mgr.c:1136 #4 0x00000000004a3052 in _slurm_rpc_dump_partitions (msg=0x2b6074645e70) at proc_req.c:1712 #5 0x000000000049e6ce in slurmctld_req (msg=0x2b6074645e70, arg=0x2b5f2800c710) at proc_req.c:324 #6 0x000000000043e1a3 in _service_connection (arg=0x2b5f2800c710) at controller.c:1096 #7 0x000000367b2079d1 in start_thread () from /lib64/libpthread.so.0 #8 0x000000367aae88fd in clone () from /lib64/libc.so.6 (gdb) info thread 27 Thread 0x2b5f3b02f700 (LWP 10978) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 26 Thread 0x2b5f3b130700 (LWP 11010) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 25 Thread 0x2b5f26726700 (LWP 11006) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 24 Thread 0x2b5f3a221700 (LWP 11005) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 23 Thread 0x2b5f24201700 (LWP 23698) 0x000000367aae1373 in select () from /lib64/libc.so.6 22 Thread 0x2b5f3a524700 (LWP 10984) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 21 Thread 0x2b5f25211700 (LWP 11007) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 20 Thread 0x2b5f39e1d700 (LWP 11001) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 19 Thread 0x2b5f38a09700 (LWP 10985) 0x000000367aadf0d3 in poll () from /lib64/libc.so.6 18 Thread 0x2b5f24f0e700 (LWP 11011) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 17 Thread 0x2b5f3a827700 (LWP 11012) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 16 Thread 0x2b5f38d0c700 (LWP 11002) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 15 Thread 0x2b5f3ac2b700 (LWP 11003) 0x000000367b20e264 in __lll_lock_wait () from /lib64/libpthread.so.0 14 Thread 0x2b5f27b3a700 (LWP 11004) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 13 Thread 0x2b5f190e4700 (LWP 23695) 0x000000367aaaca3d in nanosleep () from /lib64/libc.so.6 12 Thread 0x2b5f13935700 (LWP 23657) 0x000000367b20b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 11 Thread 0x2b5f24403700 (LWP 23700) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 10 Thread 0x2b5f26c2b700 (LWP 11008) bit_test (b=0x2381540, bit=155) at bitstring.c:236 9 Thread 0x2b5f18710700 (LWP 23694) 0x000000367b20b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 8 Thread 0x2b5f25514700 (LWP 11000) 0x000000367b20b5bc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 7 Thread 0x2b5f24302700 (LWP 23699) 0x000000367b20f4b5 in sigwait () from /lib64/libpthread.so.0 6 Thread 0x2b5f13834700 (LWP 23655) 0x000000367b20822d in pthread_join () from /lib64/libpthread.so.0 5 Thread 0x2b5f24100700 (LWP 23697) 0x000000367b20b98e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 4 Thread 0x2b5f191e5700 (LWP 23696) 0x000000367b20822d in pthread_join () from /lib64/libpthread.so.0 3 Thread 0x2b5f13733700 (LWP 23654) 0x000000367aaaca3d in nanosleep () from /lib64/libc.so.6 2 Thread 0x2b5f115ba6e0 (LWP 23652) 0x000000367aa75fe2 in malloc_consolidate () from /lib64/libc.so.6 * 1 Thread 0x2b5f24908700 (LWP 11009) 0x000000367aa8967e in memcpy () from /lib64/libc.so.6