On CentOS 6.3 and 6.4 kernel panics after apparently random amount of time ranging from 24 hours to two weeks.<br />
<br />
Crash dump for 2.6.32-358.0.1.el6.x86_64<br />
<br />
KERNEL: /usr/lib/debug/lib/modules/2.6.32-358.0.1.el6.x86_64/vmlinux<br />
DUMPFILE: vmcore [PARTIAL DUMP]<br />
CPUS: 4<br />
DATE: Wed Mar 13 18:11:41 2013<br />
UPTIME: 3 days, 23:55:36<br />
LOAD AVERAGE: 0.00, 0.00, 0.00<br />
TASKS: 287<br />
NODENAME: FS-01<br />
RELEASE: 2.6.32-358.0.1.el6.x86_64<br />
VERSION: <a href="http://bugs.centos.org/view.php?id=1">0000001</a> SMP Wed Feb 27 06:06:45 UTC 2013<br />
MACHINE: x86_64 (2660 Mhz)<br />
MEMORY: 4 GB<br />
PANIC: "Kernel panic - not syncing: Watchdog detected hard LOCKUP on cpu 3"<br />
PID: 29503<br />
COMMAND: "freeswitch"<br />
TASK: ffff880136782040 [THREAD_INFO: ffff880013008000]<br />
CPU: 3<br />
STATE: TASK_INTERRUPTIBLE (PANIC)<br />
<br />
crash> bt<br />
PID: 29503 TASK: ffff880136782040 CPU: 3 COMMAND: "freeswitch"<br />
#0 [ffff880028387b30] machine_kexec at ffffffff81035b7b<br />
<a href="http://bugs.centos.org/view.php?id=1">0000001</a> [ffff880028387b90] crash_kexec at ffffffff810c0dd2<br />
<a href="http://bugs.centos.org/view.php?id=2">0000002</a> [ffff880028387c60] panic at ffffffff8150d0af<br />
#3 [ffff880028387ce0] watchdog_overflow_callback at ffffffff810e11dd<br />
#4 [ffff880028387d00] __perf_event_overflow at ffffffff81116fe0<br />
#5 [ffff880028387da0] perf_event_overflow at ffffffff81117604<br />
#6 [ffff880028387db0] intel_pmu_handle_irq at ffffffff810207e2<br />
<a href="http://bugs.centos.org/view.php?id=7">0000007</a> [ffff880028387e90] perf_event_nmi_handler at ffffffff815117f9<br />
<a href="http://bugs.centos.org/view.php?id=8">0000008</a> [ffff880028387ea0] notifier_call_chain at ffffffff815132b5<br />
<a href="http://bugs.centos.org/view.php?id=9">0000009</a> [ffff880028387ee0] atomic_notifier_call_chain at ffffffff8151331a<br />
<a href="http://bugs.centos.org/view.php?id=10">0000010</a> [ffff880028387ef0] notify_die at ffffffff8109cc1e<br />
<a href="http://bugs.centos.org/view.php?id=11">0000011</a> [ffff880028387f20] do_nmi at ffffffff81510f7b<br />
#12 [ffff880028387f50] nmi at ffffffff81510840<br />
[exception RIP: _spin_lock+30]<br />
RIP: ffffffff815100ae RSP: ffff880013009ab8 RFLAGS: 00000093<br />
RAX: 000000000000105a RBX: ffff880028396700 RCX: 000000000000003c<br />
RDX: 0000000000001058 RSI: ffff880028296700 RDI: ffff880028296700<br />
RBP: ffff880013009ab8 R8: ffff88013dc65200 R9: 0000000000000000<br />
R10: 0000000000000000 R11: 00000000000e7ef0 R12: ffff880028296700<br />
R13: 0000000000000003 R14: 0000000000000001 R15: ffff880028296700<br />
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018<br />
--- <NMI exception stack> ---<br />
<a href="http://bugs.centos.org/view.php?id=13">0000013</a> [ffff880013009ab8] _spin_lock at ffffffff815100ae<br />
<a href="http://bugs.centos.org/view.php?id=14">0000014</a> [ffff880013009ac0] double_lock_balance at ffffffff81055bda<br />
<a href="http://bugs.centos.org/view.php?id=15">0000015</a> [ffff880013009ae0] pull_rt_task at ffffffff81064124<br />
#16 [ffff880013009b30] pre_schedule_rt at ffffffff810642d5<br />
#17 [ffff880013009b40] schedule at ffffffff8150d4f2<br />
#18 [ffff880013009c00] futex_wait_queue_me at ffffffff810a9af9<br />
#19 [ffff880013009c40] futex_wait at ffffffff810aac08<br />
#20 [ffff880013009dc0] do_futex at ffffffff810ac371<br />
#21 [ffff880013009ef0] sys_futex at ffffffff810ace2b<br />
#22 [ffff880013009f80] system_call_fastpath at ffffffff8100b072<br />
RIP: 00007f66f9b8743c RSP: 00007f66ae5a33b0 RFLAGS: 00000282<br />
RAX: 00000000000000ca RBX: ffffffff8100b072 RCX: 00000000fffffff8<br />
RDX: 00000000747f507a RSI: 0000000000000080 RDI: 0000000001d79254<br />
RBP: 00007f66bc48cea8 R8: 0000000000000000 R9: 000000003a3fa82e<br />
R10: 0000000000000000 R11: 0000000000000246 R12: 00007f66fa606df0<br />
R13: 0000000000000320 R14: 00007f66fa606de0 R15: 00007f66fa606de0<br />
ORIG_RAX: 00000000000000ca CS: 0033 SS: 002b<br />
<br />
Crash dump for 2.6.32-279.22.1.el6.x86_64<br />
<br />
KERNEL: /usr/lib/debug/lib/modules/2.6.32-279.22.1.el6.x86_64/vmlinux<br />
DUMPFILE: vmcore [PARTIAL DUMP]<br />
CPUS: 4<br />
DATE: Fri Mar 8 17:35:52 2013<br />
UPTIME: 1 days, 08:05:58<br />
LOAD AVERAGE: 0.01, 0.05, 0.02<br />
TASKS: 305<br />
NODENAME: FS-01<br />
RELEASE: 2.6.32-279.22.1.el6.x86_64<br />
VERSION: <a href="http://bugs.centos.org/view.php?id=1">0000001</a> SMP Wed Feb 6 03:10:46 UTC 2013<br />
MACHINE: x86_64 (2659 Mhz)<br />
MEMORY: 4 GB<br />
PANIC: "Kernel panic - not syncing: Watchdog detected hard LOCKUP on cpu 3"<br />
PID: 23875<br />
COMMAND: "freeswitch"<br />
TASK: ffff8800373ad540 [THREAD_INFO: ffff88003720e000]<br />
CPU: 3<br />
STATE: TASK_INTERRUPTIBLE (PANIC)<br />
<br />
crash> bt<br />
PID: 23875 TASK: ffff8800373ad540 CPU: 3 COMMAND: "freeswitch"<br />
#0 [ffff880028387b00] machine_kexec at ffffffff81031f7b<br />
<a href="http://bugs.centos.org/view.php?id=1">0000001</a> [ffff880028387b60] crash_kexec at ffffffff810b8e72<br />
<a href="http://bugs.centos.org/view.php?id=2">0000002</a> [ffff880028387c30] panic at ffffffff814e990a<br />
#3 [ffff880028387cb0] watchdog_overflow_callback at ffffffff810d881d<br />
#4 [ffff880028387cd0] __perf_event_overflow at ffffffff8110a63d<br />
#5 [ffff880028387d70] perf_event_overflow at ffffffff8110abf4<br />
#6 [ffff880028387d80] intel_pmu_handle_irq at ffffffff8101e9e6<br />
<a href="http://bugs.centos.org/view.php?id=7">0000007</a> [ffff880028387e90] perf_event_nmi_handler at ffffffff814edfb9<br />
<a href="http://bugs.centos.org/view.php?id=8">0000008</a> [ffff880028387ea0] notifier_call_chain at ffffffff814efb05<br />
<a href="http://bugs.centos.org/view.php?id=9">0000009</a> [ffff880028387ee0] atomic_notifier_call_chain at ffffffff814efb6a<br />
<a href="http://bugs.centos.org/view.php?id=10">0000010</a> [ffff880028387ef0] notify_die at ffffffff81096bfe<br />
<a href="http://bugs.centos.org/view.php?id=11">0000011</a> [ffff880028387f20] do_nmi at ffffffff814ed783<br />
#12 [ffff880028387f50] nmi at ffffffff814ed090<br />
[exception RIP: _spin_lock+30]<br />
RIP: ffffffff814ec8fe RSP: ffff88003720fab8 RFLAGS: 00000093<br />
RAX: 000000000000711b RBX: ffff880028396680 RCX: 000000000000003c<br />
RDX: 0000000000007119 RSI: ffff880028216680 RDI: ffff880028216680<br />
RBP: ffff88003720fab8 R8: ffff88013cc64a00 R9: 0000000000000000<br />
R10: 0000000000000000 R11: dead000000200200 R12: ffff880028216680<br />
R13: 0000000000000003 R14: 0000000000000000 R15: ffff880028216680<br />
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018<br />
--- <NMI exception stack> ---<br />
<a href="http://bugs.centos.org/view.php?id=13">0000013</a> [ffff88003720fab8] _spin_lock at ffffffff814ec8fe<br />
<a href="http://bugs.centos.org/view.php?id=14">0000014</a> [ffff88003720fac0] double_lock_balance at ffffffff8105234a<br />
<a href="http://bugs.centos.org/view.php?id=15">0000015</a> [ffff88003720fae0] pull_rt_task at ffffffff81060844<br />
#16 [ffff88003720fb30] pre_schedule_rt at ffffffff810609f5<br />
#17 [ffff88003720fb40] schedule at ffffffff814e9d42<br />
#18 [ffff88003720fc00] futex_wait_queue_me at ffffffff810a24e9<br />
#19 [ffff88003720fc40] futex_wait at ffffffff810a35f8<br />
#20 [ffff88003720fdc0] do_futex at ffffffff810a4d61<br />
#21 [ffff88003720fef0] sys_futex at ffffffff810a581b<br />
#22 [ffff88003720ff80] system_call_fastpath at ffffffff8100b072<br />
RIP: 00007ff9f58f943c RSP: 00007ff99a633fa0 RFLAGS: 00000246<br />
RAX: 00000000000000ca RBX: ffffffff8100b072 RCX: 0000000000000000<br />
RDX: 000000003aa53fa8 RSI: 0000000000000080 RDI: 0000000001341354<br />
RBP: 00007ff98ca19548 R8: 0000000000000000 R9: 000000001d529fd2<br />
R10: 0000000000000000 R11: 0000000000000246 R12: 00007ff9f6378df0<br />
R13: 0000000000000320 R14: 00007ff9f6378de0 R15: 00007ff9f6378de0<br />
ORIG_RAX: 00000000000000ca CS: 0033 SS: 002b
↧