Sorry, got another crash right now, in run_filter+0x6c/0xc0. [93544.101832] ------------[ cut here ]------------ [93544.101920] Kernel BUG at d00000000dc50000 [verbose debug info unavailable] [93544.101935] Oops: Exception in kernel mode, sig: 5 [#1] [93544.101954] SMP NR_CPUS=2048 NUMA PowerNV [93544.102006] Modules linked in: nfnetlink_queue nfnetlink_log nfnetlink bluetooth aha3xx(OE) vhost_net vhost macvtap macvlan drbd_transport_rdma(OE) drbd_transport_tcp(OE) dm_zero drbd(OE) xt_CHECKSUM iptabl e_mangle ipt_REJECT nf_reject_ipv4 xt_tcpudp ebtable_filter ebtables ip6table_filter ip6_tables xt_addrtype xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_na t_ipv4 iptable_filter ip_tables x_tables nf_nat nf_conntrack aufs nfsv3 nfs fscache binfmt_misc bridge stp llc powernv_rng uio_pdrv_genirq uio kvm_hv kvm rdma_cm nfsd iw_cm mlx5_ib auth_rpcgss nfs_acl lockd gr ace sunrpc ib_ipoib ib_uverbs ib_umad ib_cm ib_sa ib_mad ib_core ib_addr autofs4 raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq raid1 raid0 multipath linear ses enclosur e bnx2x mlx5_core vxlan ip6_udp_tunnel ipr udp_tunnel nvme mdio libcrc32c [last unloaded: aha3xx] [93544.103269] CPU: 64 PID: 0 Comm: swapper/64 Tainted: G OE 4.4.0-13-generic #29-Ubuntu [93544.103356] task: c000000ff09a0e90 ti: c000000ff0a1c000 task.ti: c000000ff0a1c000 [93544.103429] NIP: d00000000dc50000 LR: c000000000ab21bc CTR: d00000000dc50000 [93544.103503] REGS: c000000ff0a1f0d0 TRAP: 0700 Tainted: G OE (4.4.0-13-generic) [93544.103588] MSR: 9000000100029033 CR: 22002848 XER: 00000000 [93544.103774] CFAR: c000000000ab21b8 SOFTE: 1 GPR00: c000000000ab7ff4 c000000ff0a1f350 c0000000015a3b00 c000000f85451400 GPR04: d00000000dc30028 000000000000001c c000000f99380000 c000001e3e2e962c GPR08: c000000001515f00 d00000000dc50000 0000000000000000 000000000000001d GPR12: d00000000dc50000 c000000007b06000 c000000ff980a938 0000000000000100 GPR16: c000000ff980a738 c000000ff980a538 c000000ff0a1f5c0 c0000000015e4cf8 GPR20: 0000000000000000 c0000000015e4ce8 c000000f991b0800 0000000000000000 GPR24: 0000000000000000 000000000000002a c000001e3e2e9602 c000000f99380000 GPR28: 000000000000001c c000001e4ad19000 c000000f85451400 000000000000001c [93544.104713] NIP [d00000000dc50000] 0xd00000000dc50000 [93544.104807] LR [c000000000ab21bc] run_filter+0x6c/0xc0 [93544.104899] Call Trace: [93544.104948] [c000000ff0a1f350] [c000000ff0a1f390] 0xc000000ff0a1f390 (unreliable) [93544.105110] [c000000ff0a1f390] [c000000000ab7ff4] tpacket_rcv+0x144/0xa50 [93544.105247] [c000000ff0a1f480] [c00000000096e4d0] dev_hard_start_xmit+0x2c0/0x530 [93544.105407] [c000000ff0a1f560] [c00000000096eff8] __dev_queue_xmit+0x668/0x770 [93544.105568] [c000000ff0a1f600] [c000000000a0b160] arp_xmit+0x30/0xd0 [93544.105704] [c000000ff0a1f670] [c000000000a0b294] arp_send_dst.part.0+0x94/0xb0 [93544.105864] [c000000ff0a1f6e0] [c000000000a0be24] arp_solicit+0x114/0x2b0 [93544.106001] [c000000ff0a1f7b0] [c00000000097a5b4] neigh_probe+0x84/0xd0 [93544.106139] [c000000ff0a1f7e0] [c00000000098043c] neigh_timer_handler+0xbc/0x310 [93544.106299] [c000000ff0a1f820] [c000000000149ebc] call_timer_fn+0x5c/0x1c0 [93544.106435] [c000000ff0a1f8b0] [c00000000014a37c] run_timer_softirq+0x31c/0x3f0 [93544.106596] [c000000ff0a1f980] [c0000000000beaf8] __do_softirq+0x188/0x3e0 [93544.106733] [c000000ff0a1fa70] [c0000000000befc8] irq_exit+0xc8/0x100 [93544.106872] [c000000ff0a1fa90] [c00000000001f954] timer_interrupt+0xa4/0xe0 [93544.107009] [c000000ff0a1fac0] [c000000000002714] decrementer_common+0x114/0x180 [93544.107170] --- interrupt: 901 at arch_local_irq_restore+0x74/0x90 [93544.107170] LR = arch_local_irq_restore+0x74/0x90 [93544.107375] [c000000ff0a1fdb0] [c000000ff981d678] 0xc000000ff981d678 (unreliable) [93544.107538] [c000000ff0a1fdd0] [c000000000900da8] cpuidle_enter_state+0x1a8/0x410 [93544.107697] [c000000ff0a1fe30] [c000000000119898] call_cpuidle+0x78/0xd0 [93544.107834] [c000000ff0a1fe70] [c000000000119c6c] cpu_startup_entry+0x37c/0x490 [93544.107995] [c000000ff0a1ff30] [c00000000004565c] start_secondary+0x33c/0x360 [93544.108131] [c000000ff0a1ff90] [c000000000008b6c] start_secondary_prolog+0x10/0x14 [93544.108289] Instruction dump: [93544.108360] XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX [93544.108588] XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX [93544.108821] ---[ end trace 8419e626fa8d7aaf ]--- [93544.111837] [93544.210799] ------------[ cut here ]------------ On the second machine (that ran the same command) the backtrace is a bit different: [94483.530269] Netfilter messages via NETLINK v0.30. [94484.521665] Oops: Exception in kernel mode, sig: 4 [#1] [94484.521754] SMP NR_CPUS=2048 NUMA PowerNV [94484.521766] Modules linked in: nfnetlink_queue nfnetlink_log nfnetlink bluetooth drbd_transport_rdma(OE) drbd_transport_tcp(OE) dm_zero drbd(OE) lru_cache xt_CHECKSUM iptable_mangle ipt_MASQUERADE nf_nat_masquerade_ipv4 iptable_nat nf_nat_ipv4 nf_nat nf_conntrack_ipv4 nf_defrag_ipv4 xt_conntrack nf_conntrack ipt_REJECT nf_reject_ipv4 xt_tcpudp ebtable_filter ebtables ip6table_filter ip6_tables iptable_filter ip_tables x_tables nfsv3 nfs_acl nfs lockd grace fscache binfmt_misc bridge stp llc uio_pdrv_genirq uio powernv_rng kvm_hv kvm rdma_cm iw_cm mlx5_ib ib_ipoib ib_uverbs ib_umad sunrpc ib_cm ib_sa ib_mad ib_core ib_addr autofs4 ses enclosure bnx2x mlx5_core vxlan ip6_udp_tunnel udp_tunnel ipr mdio nvme libcrc32c [last unloaded: drbd] [94484.522727] CPU: 56 PID: 0 Comm: swapper/56 Tainted: G OE 4.4.0-13-generic #29-Ubuntu [94484.522813] task: c000000ff0996610 ti: c000001fffdd0000 task.ti: c000000ff097c000 [94484.522887] NIP: d000000018250000 LR: c000000000ab21bc CTR: d000000018250000 [94484.522960] REGS: c000001fffdd35e0 TRAP: 0e40 Tainted: G OE (4.4.0-13-generic) [94484.523045] MSR: 9000000000089033 CR: 24002888 XER: 00000000 [94484.523231] CFAR: c000000000ab21b8 SOFTE: 1 GPR00: c000000000ab7ff4 c000001fffdd3860 c0000000015a3b00 c000000fe03d5b00 GPR04: d000000018230028 000000000000002e c000000fdb618000 000000000000ffff GPR08: c000000001515f00 d000000018250000 0000000000000000 d0000000119f8828 GPR12: d000000018250000 c000000007b01400 c000000fd7d2b100 c000000fd84b0000 GPR16: c000000fdb618900 000000000000fac4 c000001fffdd39f0 0000000000000001 GPR20: d000000013db6f50 000000000000dd86 0000000000000000 ffffffff80000000 GPR24: 0000000000000005 000000000000002e c000000fa1be3510 c000000fdb618000 GPR28: 000000000000002e c000000fe09f7800 c000000fe03d5b00 000000000000002e [94484.524170] NIP [d000000018250000] 0xd000000018250000 [94484.524225] LR [c000000000ab21bc] run_filter+0x6c/0xc0 [94484.524274] Call Trace: [94484.524323] [c000001fffdd38a0] [c000000000ab7ff4] tpacket_rcv+0x144/0xa50 [94484.524461] [c000001fffdd3990] [c000000000966a9c] __netif_receive_skb_core+0x4ac/0xdc0 [94484.524619] [c000001fffdd3a70] [c00000000096aa74] netif_receive_skb_internal+0x34/0xd0 [94484.524779] [c000001fffdd3ab0] [c00000000096bc5c] napi_gro_receive+0x11c/0x230 [94484.524951] [c000001fffdd3af0] [d0000000119b7b1c] bnx2x_rx_int+0xb4c/0x1710 [bnx2x] [94484.525122] [c000001fffdd3c90] [d0000000119b8fc0] bnx2x_poll+0x310/0x440 [bnx2x] [94484.525280] [c000001fffdd3d40] [c00000000096b20c] net_rx_action+0x2dc/0x4a0 [94484.525420] [c000001fffdd3e50] [c0000000000beaf8] __do_softirq+0x188/0x3e0 [94484.525556] [c000001fffdd3f40] [c0000000000befc8] irq_exit+0xc8/0x100 [94484.525693] [c000001fffdd3f60] [c00000000001132c] __do_irq+0x8c/0x1d0 [94484.525832] [c000001fffdd3f90] [c000000000024800] call_do_irq+0x14/0x24 [94484.525968] [c000000ff097fa70] [c000000000011508] do_IRQ+0x98/0x140 [94484.526107] [c000000ff097fac0] [c000000000002594] hardware_interrupt_common+0x114/0x180 [94484.526268] --- interrupt: 501 at arch_local_irq_restore+0x5c/0x90 [94484.526268] LR = arch_local_irq_restore+0x40/0x90 [94484.526474] [c000000ff097fdb0] [c000000ff961d678] 0xc000000ff961d678 (unreliable) [94484.526636] [c000000ff097fdd0] [c000000000900da8] cpuidle_enter_state+0x1a8/0x410 [94484.526796] [c000000ff097fe30] [c000000000119898] call_cpuidle+0x78/0xd0 [94484.526932] [c000000ff097fe70] [c000000000119c6c] cpu_startup_entry+0x37c/0x490 [94484.527093] [c000000ff097ff30] [c00000000004565c] start_secondary+0x33c/0x360 [94484.527230] [c000000ff097ff90] [c000000000008b6c] start_secondary_prolog+0x10/0x14 [94484.527387] Instruction dump: [94484.527459] XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX [94484.527685] XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX [94484.527924] ---[ end trace f95ca9899b7ec379 ]--- [94484.530466] [94484.530497] Sending IPI to other CPUs That's with these versions: [ 0.000000] Linux version 4.4.0-13-generic (buildd@bos01-ppc64el-003) (gcc version 5.3.1 20160225 (Ubuntu/IBM 5.3.1-10ubuntu2) ) #29-Ubuntu SMP Fri Mar 11 19:30:44 UTC 2016 (Ubuntu 4.4.0-13.29-generic 4.4.5) [ 1.341655] bnx2x: QLogic 5771x/578xx 10/20-Gigabit Ethernet Driver bnx2x 1.712.30-0 (2014/02/10)