on XFS writes, the server would get a kernel panic by random. So far, it's been happening on 2 weeks interval. Below is the log.


-----

[922371.445221] BUG: unable to handle kernel paging request at 0000000389b14ad8
[922371.445730] IP: [<ffffffff81557980>] schedule+0x250/0x451
[922371.446093] PGD 17b7c6067 PUD 0 
[922371.446436] Thread overran stack, or stack corrupted
[922371.446680] Oops: 0000 [#1] SMP 
[922371.447021] last sysfs file: /sys/devices/system/cpu/cpu11/cache/index2/shared_cpu_map
[922371.447386] CPU 0 
[922371.447585] Modules linked in: btrfs zlib_deflate crc32c libcrc32c ufs qnx4 hfsplus hfs minix ntfs vfat msdos fat jfs reiserfs netconsole configfs xfs exportfs fbcon tileblit font bitblit softcursor dell_wmi dcdbas psmouse vga16fb joydev serio_raw vgastate power_meter bnx2 lp parport usbhid hid usb_storage mpt2sas scsi_transport_sas
[922371.452534] Pid: 803, comm: flush-8:0 Not tainted 2.6.32-32-server #62-Ubuntu PowerEdge R710
[922371.452913] RIP: 0010:[<ffffffff81557980>]  [<ffffffff81557980>] schedule+0x250/0x451
[922371.453372] RSP: 0018:ffff88022149a280  EFLAGS: 00010087
[922371.453616] RAX: 0000000081055cc3 RBX: ffff880009015f00 RCX: 0000000000000001
[922371.453958] RDX: ffff880222e8ae00 RSI: ffffffff817d5e00 RDI: ffff880222e8ae00
[922371.454299] RBP: ffff88022149a320 R08: 0000000000000000 R09: 0000000000000100
[922371.480427] R10: fffea2c9014dd580 R11: 0000000000000001 R12: 0000000000000000
[922371.506921] R13: ffffffff81570f40 R14: 00000001057fa251 R15: 00000000ffffffff
[922371.533337] FS:  0000000000000000(0000) GS:ffff880009000000(0000) knlGS:0000000000000000
[922371.560002] CS:  0010 DS: 0018 ES: 0018 CR0: 000000008005003b
[922371.573587] CR2: 0000000389b14ad8 CR3: 00000001ad407000 CR4: 00000000000006f0
[922371.601358] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[922371.629838] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
[922371.659001] Process flush-8:0 (pid: 803, threadinfo ffff88022149a000, task ffff880222e8ae00)
[922371.688450] Stack:
[922371.702807]  0000000000015f00 0000000000015f00 ffff880222e8b1d0 ffff88022149bfd8
[922371.717663] <0> 0000000000015f00 ffff880222e8ae00 0000000000015f00 ffff88022149bfd8
[922371.746297] <0> 0000000000015f00 ffff880222e8b1d0 0000000000015f00 0000000000015f00
[922371.788745] Call Trace:
[922371.802681]  [<ffffffff8155837d>] schedule_timeout+0x22d/0x300
[922371.816525]  [<ffffffff810f7a96>] ? find_lock_page+0x26/0x80
[922371.830133]  [<ffffffff810f803f>] ? find_or_create_page+0x3f/0xb0
[922371.843599]  [<ffffffff815592ae>] __down+0x7e/0xc0
[922371.856770]  [<ffffffff8108b021>] down+0x41/0x50
[922371.869659]  [<ffffffffa01621f3>] xfs_buf_lock+0x23/0x60 [xfs]
[922371.882403]  [<ffffffffa0162375>] _xfs_buf_find+0x145/0x240 [xfs]
[922371.894892]  [<ffffffffa01624d0>] xfs_buf_get_flags+0x60/0x170 [xfs]
[922371.907127]  [<ffffffffa01625f8>] xfs_buf_read_flags+0x18/0xa0 [xfs]
[922371.919262]  [<ffffffffa0157529>] xfs_trans_read_buf+0x1c9/0x300 [xfs]
[922371.931032]  [<ffffffff810f6527>] ? unlock_page+0x27/0x30
[922371.942743]  [<ffffffffa0126e8e>] xfs_btree_read_buf_block+0x5e/0xc0 [xfs]
[922371.954441]  [<ffffffffa0127584>] xfs_btree_lookup_get_block+0x84/0xf0 [xfs]
[922371.965886]  [<ffffffffa0127c27>] xfs_btree_lookup+0xd7/0x4a0 [xfs]
[922371.976976]  [<ffffffffa015d82a>] ? kmem_zone_zalloc+0x3a/0x50 [xfs]
[922371.987853]  [<ffffffffa0113dac>] ? xfs_allocbt_init_cursor+0x4c/0xc0 [xfs]
[922371.998550]  [<ffffffffa0110d9c>] xfs_alloc_lookup_ge+0x1c/0x20 [xfs]
[922372.009119]  [<ffffffffa01127fb>] xfs_alloc_ag_vextent_near+0x5b/0x9a0 [xfs]
[922372.019540]  [<ffffffffa0113215>] xfs_alloc_ag_vextent+0xd5/0x130 [xfs]
[922372.029747]  [<ffffffffa01139d8>] xfs_alloc_vextent+0x1f8/0x490 [xfs]
[922372.039761]  [<ffffffffa0121856>] xfs_bmap_btalloc+0x176/0x9f0 [xfs]
[922372.049512]  [<ffffffffa0122fb1>] xfs_bmap_alloc+0x21/0x40 [xfs]
[922372.059372]  [<ffffffffa0123b6f>] xfs_bmapi+0xb9f/0x1290 [xfs]
[922372.069136]  [<ffffffffa014b274>] ? xfs_log_reserve+0xd4/0xe0 [xfs]
[922372.078831]  [<ffffffffa0145055>] xfs_iomap_write_allocate+0x1c5/0x3c0 [xfs]
[922372.088471]  [<ffffffff8105f0fb>] ? enqueue_task_fair+0x5b/0xa0
[922372.098157]  [<ffffffffa0145dab>] xfs_iomap+0x2ab/0x2e0 [xfs]
[922372.107705]  [<ffffffffa015e45d>] xfs_map_blocks+0x2d/0x40 [xfs]
[922372.117076]  [<ffffffffa015f86a>] xfs_page_state_convert+0x3da/0x720 [xfs]
[922372.126686]  [<ffffffff812baa3d>] ? radix_tree_delete+0x14d/0x2d0
[922372.136318]  [<ffffffffa015fd0a>] xfs_vm_writepage+0x7a/0x130 [xfs]
[922372.146051]  [<ffffffff8110f91e>] ? __dec_zone_page_state+0x2e/0x30
[922372.155947]  [<ffffffff81103d33>] pageout+0x123/0x280
[922372.165811]  [<ffffffff811042f3>] shrink_page_list+0x263/0x600
[922372.175760]  [<ffffffff8110499e>] shrink_inactive_list+0x30e/0x810
[922372.185710]  [<ffffffffa0125a12>] ? xfs_btree_is_lastrec+0x52/0x70 [xfs]
[922372.195867]  [<ffffffffa012ad04>] ? xfs_btree_insrec+0x2e4/0x5a0 [xfs]
[922372.206030]  [<ffffffff81104f31>] shrink_list+0x91/0xf0
[922372.216254]  [<ffffffff81105127>] shrink_zone+0x197/0x240
[922372.226443]  [<ffffffff81105242>] shrink_zones+0x72/0x100
[922372.236518]  [<ffffffff8110535e>] do_try_to_free_pages+0x8e/0x370
[922372.246663]  [<ffffffff8110573f>] try_to_free_pages+0x6f/0x80
[922372.256836]  [<ffffffff811031e0>] ? isolate_pages_global+0x0/0x50
[922372.267075]  [<ffffffff810fc5c8>] __alloc_pages_slowpath+0x2d8/0x590
[922372.277365]  [<ffffffff810fc9f1>] __alloc_pages_nodemask+0x171/0x180
[922372.287717]  [<ffffffff8112faf7>] alloc_pages_current+0x87/0xd0
[922372.298003]  [<ffffffff81135a78>] new_slab+0x248/0x310
[922372.308211]  [<ffffffff81138369>] __slab_alloc+0x169/0x2d0
[922372.318452]  [<ffffffffa015d7aa>] ? kmem_zone_alloc+0x9a/0xe0 [xfs]
[922372.328775]  [<ffffffff8113888b>] kmem_cache_alloc+0xfb/0x130
[922372.339272]  [<ffffffffa015d7aa>] kmem_zone_alloc+0x9a/0xe0 [xfs]
[922372.349832]  [<ffffffffa015d80e>] kmem_zone_zalloc+0x1e/0x50 [xfs]
[922372.360398]  [<ffffffffa0155ae1>] _xfs_trans_alloc+0x31/0x80 [xfs]
[922372.370962]  [<ffffffffa0155caf>] xfs_trans_alloc+0x9f/0xb0 [xfs]
[922372.381454]  [<ffffffffa013de37>] ? xfs_iunlock+0x57/0xc0 [xfs]
[922372.391657]  [<ffffffffa01450be>] xfs_iomap_write_allocate+0x22e/0x3c0 [xfs]
[922372.401953]  [<ffffffff812a1ca1>] ? generic_make_request+0x1b1/0x4f0
[922372.412556]  [<ffffffffa0156749>] ? xfs_trans_unlocked_item+0x39/0x60 [xfs]
[922372.422663]  [<ffffffffa0145dab>] xfs_iomap+0x2ab/0x2e0 [xfs]
[922372.432774]  [<ffffffffa015e45d>] xfs_map_blocks+0x2d/0x40 [xfs]
[922372.442828]  [<ffffffffa015f86a>] xfs_page_state_convert+0x3da/0x720 [xfs]
[922372.452816]  [<ffffffff812ba7a5>] ? radix_tree_gang_lookup_tag_slot+0x95/0xe0
[922372.472803]  [<ffffffffa015fd0a>] xfs_vm_writepage+0x7a/0x130 [xfs]
[922372.482987]  [<ffffffff8110f91e>] ? __dec_zone_page_state+0x2e/0x30
[922372.493246]  [<ffffffff810fe7e7>] __writepage+0x17/0x40
[922372.503463]  [<ffffffff810ff967>] write_cache_pages+0x1d7/0x3e0
[922372.513669]  [<ffffffff810fe7d0>] ? __writepage+0x0/0x40
[922372.523933]  [<ffffffff810ffb94>] generic_writepages+0x24/0x30
[922372.534263]  [<ffffffffa015eafd>] xfs_vm_writepages+0x5d/0x80 [xfs]
[922372.544450]  [<ffffffff810ffbc1>] do_writepages+0x21/0x40
[922372.554631]  [<ffffffff81168db6>] writeback_single_inode+0xf6/0x3d0
[922372.564811]  [<ffffffff811694e5>] writeback_sb_inodes+0x195/0x280
[922372.574909]  [<ffffffff81169d00>] writeback_inodes_wb+0xa0/0x1b0
[922372.584944]  [<ffffffff8116a04b>] wb_writeback+0x23b/0x2a0
[922372.594940]  [<ffffffff81077bec>] ? lock_timer_base+0x3c/0x70
[922372.604969]  [<ffffffff8116a22c>] wb_do_writeback+0x17c/0x190
[922372.615045]  [<ffffffff81077d00>] ? process_timeout+0x0/0x10
[922372.624969]  [<ffffffff8116a293>] bdi_writeback_task+0x53/0xf0
[922372.634680]  [<ffffffff81111636>] bdi_start_fn+0x86/0x100
[922372.644283]  [<ffffffff811115b0>] ? bdi_start_fn+0x0/0x100
[922372.653939]  [<ffffffff81085d16>] kthread+0x96/0xa0
[922372.663479]  [<ffffffff810141ea>] child_rip+0xa/0x20
[922372.672948]  [<ffffffff81085c80>] ? kthread+0x0/0xa0
[922372.682455]  [<ffffffff810141e0>] ? child_rip+0x0/0x20
[922372.691906] Code: b0 ff 48 85 c0 48 89 45 c0 0f 84 bd 03 00 00 48 8b 75 c0 48 39 75 c8 0f 84 29 04 00 00 48 8b 7d c8 48 8b 47 08 48 89 fa 8b 40 18 <48> 8b 0c c5 c0 64 86 81 48 8b 45 b8 48 01 c8 48 3b b8 30 08 00 
[922372.727701] RIP  [<ffffffff81557980>] schedule+0x250/0x451
[922372.738354]  RSP <ffff88022149a280>
[922372.748804] CR2: 0000000389b14ad8
[922372.766718] ---[ end trace f7ab1322c0f96190 ]---
[922372.777191] Kernel panic - not syncing: Fatal exception in interrupt
[922372.787731] Pid: 803, comm: flush-8:0 Tainted: G      D    2.6.32-32-server #62-Ubuntu
[922372.808863] Call Trace:
[922372.819535]  [<ffffffff815571bb>] panic+0x78/0x139
[922372.830161]  [<ffffffff8155b13a>] oops_end+0xea/0xf0
[922372.840738]  [<ffffffff81040bc3>] no_context+0xf3/0x190
[922372.851362]  [<ffffffff81040d85>] __bad_area_nosemaphore+0x125/0x1e0
[922372.861886]  [<ffffffff81040e53>] bad_area_nosemaphore+0x13/0x20
[922372.872436]  [<ffffffff8155cce4>] do_page_fault+0x2e4/0x3b0
[922372.883040]  [<ffffffff8155a475>] page_fault+0x25/0x30
[922372.893571]  [<ffffffff81557980>] ? schedule+0x250/0x451
[922372.904119]  [<ffffffff8155837d>] schedule_timeout+0x22d/0x300
[922372.914835]  [<ffffffff810f7a96>] ? find_lock_page+0x26/0x80
[922372.925313]  [<ffffffff810f803f>] ? find_or_create_page+0x3f/0xb0
[922372.935835]  [<ffffffff815592ae>] __down+0x7e/0xc0
[922372.946371]  [<ffffffff8108b021>] down+0x41/0x50
[922372.956850]  [<ffffffffa01621f3>] xfs_buf_lock+0x23/0x60 [xfs]
[922372.967386]  [<ffffffffa0162375>] _xfs_buf_find+0x145/0x240 [xfs]
[922372.977937]  [<ffffffffa01624d0>] xfs_buf_get_flags+0x60/0x170 [xfs]
[922372.988531]  [<ffffffffa01625f8>] xfs_buf_read_flags+0x18/0xa0 [xfs]
[922372.999134]  [<ffffffffa0157529>] xfs_trans_read_buf+0x1c9/0x300 [xfs]
[922373.009767]  [<ffffffff810f6527>] ? unlock_page+0x27/0x30
[922373.020208]  [<ffffffffa0126e8e>] xfs_btree_read_buf_block+0x5e/0xc0 [xfs]
[922373.030589]  [<ffffffffa0127584>] xfs_btree_lookup_get_block+0x84/0xf0 [xfs]
[922373.040085]  [<ffffffffa0127c27>] xfs_btree_lookup+0xd7/0x4a0 [xfs]
[922373.049660]  [<ffffffffa015d82a>] ? kmem_zone_zalloc+0x3a/0x50 [xfs]
[922373.059183]  [<ffffffffa0113dac>] ? xfs_allocbt_init_cursor+0x4c/0xc0 [xfs]
[922373.068697]  [<ffffffffa0110d9c>] xfs_alloc_lookup_ge+0x1c/0x20 [xfs]
[922373.078259]  [<ffffffffa01127fb>] xfs_alloc_ag_vextent_near+0x5b/0x9a0 [xfs]
[922373.087803]  [<ffffffffa0113215>] xfs_alloc_ag_vextent+0xd5/0x130 [xfs]
[922373.097121]  [<ffffffffa01139d8>] xfs_alloc_vextent+0x1f8/0x490 [xfs]
[922373.106182]  [<ffffffffa0121856>] xfs_bmap_btalloc+0x176/0x9f0 [xfs]
[922373.115282]  [<ffffffffa0122fb1>] xfs_bmap_alloc+0x21/0x40 [xfs]
[922373.124391]  [<ffffffffa0123b6f>] xfs_bmapi+0xb9f/0x1290 [xfs]
[922373.133505]  [<ffffffffa014b274>] ? xfs_log_reserve+0xd4/0xe0 [xfs]
[922373.142618]  [<ffffffffa0145055>] xfs_iomap_write_allocate+0x1c5/0x3c0 [xfs]
[922373.151848]  [<ffffffff8105f0fb>] ? enqueue_task_fair+0x5b/0xa0
[922373.161202]  [<ffffffffa0145dab>] xfs_iomap+0x2ab/0x2e0 [xfs]
[922373.170554]  [<ffffffffa015e45d>] xfs_map_blocks+0x2d/0x40 [xfs]
[922373.179958]  [<ffffffffa015f86a>] xfs_page_state_convert+0x3da/0x720 [xfs]
[922373.189467]  [<ffffffff812baa3d>] ? radix_tree_delete+0x14d/0x2d0
[922373.199037]  [<ffffffffa015fd0a>] xfs_vm_writepage+0x7a/0x130 [xfs]
[922373.208670]  [<ffffffff8110f91e>] ? __dec_zone_page_state+0x2e/0x30
[922373.218421]  [<ffffffff81103d33>] pageout+0x123/0x280
[922373.228138]  [<ffffffff811042f3>] shrink_page_list+0x263/0x600
[922373.237940]  [<ffffffff8110499e>] shrink_inactive_list+0x30e/0x810
[922373.247655]  [<ffffffffa0125a12>] ? xfs_btree_is_lastrec+0x52/0x70 [xfs]
[922373.257575]  [<ffffffffa012ad04>] ? xfs_btree_insrec+0x2e4/0x5a0 [xfs]
[922373.267464]  [<ffffffff81104f31>] shrink_list+0x91/0xf0
[922373.277411]  [<ffffffff81105127>] shrink_zone+0x197/0x240
[922373.287281]  [<ffffffff81105242>] shrink_zones+0x72/0x100
[922373.296947]  [<ffffffff8110535e>] do_try_to_free_pages+0x8e/0x370
[922373.306640]  [<ffffffff8110573f>] try_to_free_pages+0x6f/0x80
[922373.316401]  [<ffffffff811031e0>] ? isolate_pages_global+0x0/0x50
[922373.326145]  [<ffffffff810fc5c8>] __alloc_pages_slowpath+0x2d8/0x590
[922373.335987]  [<ffffffff810fc9f1>] __alloc_pages_nodemask+0x171/0x180
[922373.345797]  [<ffffffff8112faf7>] alloc_pages_current+0x87/0xd0
[922373.355582]  [<ffffffff81135a78>] new_slab+0x248/0x310
[922373.365347]  [<ffffffff81138369>] __slab_alloc+0x169/0x2d0
[922373.375169]  [<ffffffffa015d7aa>] ? kmem_zone_alloc+0x9a/0xe0 [xfs]
[922373.385125]  [<ffffffff8113888b>] kmem_cache_alloc+0xfb/0x130
[922373.395231]  [<ffffffffa015d7aa>] kmem_zone_alloc+0x9a/0xe0 [xfs]
[922373.405448]  [<ffffffffa015d80e>] kmem_zone_zalloc+0x1e/0x50 [xfs]
[922373.415886]  [<ffffffffa0155ae1>] _xfs_trans_alloc+0x31/0x80 [xfs]
[922373.426225]  [<ffffffffa0155caf>] xfs_trans_alloc+0x9f/0xb0 [xfs]
[922373.436494]  [<ffffffffa013de37>] ? xfs_iunlock+0x57/0xc0 [xfs]
[922373.446565]  [<ffffffffa01450be>] xfs_iomap_write_allocate+0x22e/0x3c0 [xfs]
[922373.456689]  [<ffffffff812a1ca1>] ? generic_make_request+0x1b1/0x4f0
[922373.466926]  [<ffffffffa0156749>] ? xfs_trans_unlocked_item+0x39/0x60 [xfs]
[922373.477118]  [<ffffffffa0145dab>] xfs_iomap+0x2ab/0x2e0 [xfs]
[922373.487229]  [<ffffffffa015e45d>] xfs_map_blocks+0x2d/0x40 [xfs]
[922373.497285]  [<ffffffffa015f86a>] xfs_page_state_convert+0x3da/0x720 [xfs]
[922373.507313]  [<ffffffff812ba7a5>] ? radix_tree_gang_lookup_tag_slot+0x95/0xe0
[922373.527265]  [<ffffffffa015fd0a>] xfs_vm_writepage+0x7a/0x130 [xfs]
[922373.537493]  [<ffffffff8110f91e>] ? __dec_zone_page_state+0x2e/0x30
[922373.547801]  [<ffffffff810fe7e7>] __writepage+0x17/0x40
[922373.558097]  [<ffffffff810ff967>] write_cache_pages+0x1d7/0x3e0
[922373.568302]  [<ffffffff810fe7d0>] ? __writepage+0x0/0x40
[922373.578526]  [<ffffffff810ffb94>] generic_writepages+0x24/0x30
[922373.588826]  [<ffffffffa015eafd>] xfs_vm_writepages+0x5d/0x80 [xfs]
[922373.599069]  [<ffffffff810ffbc1>] do_writepages+0x21/0x40
[922373.609208]  [<ffffffff81168db6>] writeback_single_inode+0xf6/0x3d0
[922373.619344]  [<ffffffff811694e5>] writeback_sb_inodes+0x195/0x280
[922373.629366]  [<ffffffff81169d00>] writeback_inodes_wb+0xa0/0x1b0
[922373.639352]  [<ffffffff8116a04b>] wb_writeback+0x23b/0x2a0
[922373.649303]  [<ffffffff81077bec>] ? lock_timer_base+0x3c/0x70
[922373.659257]  [<ffffffff8116a22c>] wb_do_writeback+0x17c/0x190
[922373.669258]  [<ffffffff81077d00>] ? process_timeout+0x0/0x10
[922373.679144]  [<ffffffff8116a293>] bdi_writeback_task+0x53/0xf0
[922373.688858]  [<ffffffff81111636>] bdi_start_fn+0x86/0x100
[922373.698423]  [<ffffffff811115b0>] ? bdi_start_fn+0x0/0x100
[922373.707995]  [<ffffffff81085d16>] kthread+0x96/0xa0
[922373.717413]  [<ffffffff810141ea>] child_rip+0xa/0x20
[922373.726804]  [<ffffffff81085c80>] ? kthread+0x0/0xa0
[922373.736192]  [<ffffffff810141e0>] ? child_rip+0x0/0x20
[922373.745712] Rebooting in 30 seconds..[   31.090808] bnx2: eth1 NIC Copper Link is Up, 1000 Mbps full duplex