Saw similar issue on R1.06 82 ubuntu and was able to take gcore of vizd. Analytics of gcore below: ubuntu-build02@~$ gdb /github-build/R1.06/82/ubuntu-12-04/havana/store/sandbox/build/debug/analytics/vizd ~/gcore.vizd.23750 GNU gdb (Ubuntu/Linaro 7.4-2012.04-0ubuntu2.1) 7.4-2012.04 Copyright (C) 2012 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /auto/github-build/R1.06/82/ubuntu-12-04/havana/store/sandbox/build/debug/analytics/vizd...done. [New LWP 23783] [New LWP 23784] [New LWP 23785] [New LWP 23786] [New LWP 23788] [New LWP 23787] [New LWP 23790] [New LWP 23789] [New LWP 23750] warning: .dynamic section for "/lib/x86_64-linux-gnu/libpthread.so.0" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/lib/x86_64-linux-gnu/librt.so.1" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/lib/x86_64-linux-gnu/libm.so.6" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/lib/x86_64-linux-gnu/libc.so.6" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/lib/x86_64-linux-gnu/libdl.so.2" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/lib64/ld-linux-x86-64.so.2" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/lib/x86_64-linux-gnu/libutil.so.1" is not at the expected address (wrong library or version mismatch?) warning: .dynamic section for "/lib/x86_64-linux-gnu/libnss_files.so.2" is not at the expected address (wrong library or version mismatch?) warning: Could not load shared library symbols for 8 libraries, e.g. /usr/lib64/contrail/libthrift-0.8.0.so. Use the "info sharedlibrary" command to see the complete listing. Do you need "set solib-search-path" or "set sysroot"? warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available. warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available. warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available. warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fff945fe000 Core was generated by `vizd'. #0 0x00007fc6fe730f2c in sem_unlink (name=0x7fc6d82868d0 "\320A\202") at sem_unlink.c:52 52 sem_unlink.c: No such file or directory. (gdb) bt #0 0x00007fc6fe730f2c in sem_unlink (name=0x7fc6d82868d0 "\320A\202") at sem_unlink.c:52 #1 0x00000000004304d6 in tbb::mutex::scoped_lock::acquire (this=0x7fc6fc31f300, mutex=...) at build/include/tbb/mutex.h:105 #2 0x0000000000430491 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fc6fc31f300, mutex=...) at build/include/tbb/mutex.h:91 #3 0x00000000006a595b in CdbIf::Db_Uninit (this=0x7fc6d82868d0, task_id="analytics::DbHandler", task_instance=22) at controller/src/gendb/cdb_if.cc:694 #4 0x00000000004f81d0 in DbHandler::UnInit (this=0x7fc6d8286d30, instance=22) at controller/src/analytics/db_handler.cc:142 #5 0x00000000004caa1c in SandeshGenerator::StartDbifReinit (this=0x7fc6d8290040) at controller/src/analytics/generator.cc:124 #6 0x00000000004ce396 in boost::_mfi::mf0::operator() (this=0x7fc6d8286940, p=0x7fc6d8290040) at build/include/boost/bind/mem_fn_template.hpp:49 #7 0x00000000004cdeda in boost::_bi::list1 >::operator(), boost::_bi::list0> (this=0x7fc6d8286950, f=..., a=...) at build/include/boost/bind/bind.hpp:253 #8 0x00000000004cdb9f in boost::_bi::bind_t, boost::_bi::list1 > >::operator() (this=0x7fc6d8286940) at build/include/boost/bind/bind_template.hpp:20 #9 0x00000000004cd7ad in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke (function_obj_ptr=...) at build/include/boost/function/function_template.hpp:153 #10 0x00000000004d356c in boost::function0::operator() (this=0x7fc6d8286938) at build/include/boost/function/function_template.hpp:763 #11 0x00000000006b7cfe in CdbIf::Db_BatchAddColumn (this=0x7fc6d82868d0, done=false) at controller/src/gendb/cdb_if.cc:1297 #12 0x00000000006e2fd3 in boost::_mfi::mf1::operator() (this=0x7fc6e82ec430, p=0x7fc6d82868d0, a1=false) at build/include/boost/bind/mem_fn_template.hpp:165 #13 0x00000000006e0393 in boost::_bi::list2, boost::arg<1> >::operator(), boost::_bi::list1 > (this=0x7fc6e82ec440, f=..., a=...) at build/include/boost/bind/bind.hpp:313 #14 0x00000000006dd086 in boost::_bi::bind_t, boost::_bi::list2, boost::arg<1> > >::operator() (this=0x7fc6e82ec430, a1=@0x7fc6fc31fb74: false) at build/include/boost/bind/bind_template.hpp:32 #15 0x00000000006d8ea4 in boost::detail::function::void_function_obj_invoker1, boost::_bi::list2, boost::arg<1> > >, void, bool>::invoke (function_obj_ptr=..., a0=false) at build/include/boost/function/function_template.hpp:153 #16 0x0000000000547f7d in boost::function1::operator() (this=0x7fc6e82ec428, a0=false) at build/include/boost/function/function_template.hpp:763 #17 0x00000000006e99e5 in WorkQueue::OnExit (this=0x7fc6e82ec390, done=false) at controller/src/base/queue_task.h:292 #18 0x00000000006e92ca in WorkQueue::RunnerDone (this=0x7fc6e82ec390) at controller/src/base/queue_task.h:374 #19 0x00000000006e82c0 in QueueTaskRunner >::RunQueue (this=0x7fc6e84d8530) at controller/src/base/queue_task.h:57 #20 0x00000000006e790c in QueueTaskRunner >::Run (this=0x7fc6e84d8530) at controller/src/base/queue_task.h:36 #21 0x0000000000761a65 in TaskImpl::execute (this=0x7fc6e8548240) at controller/src/base/task.cc:224 #22 0x00007fc6febe2ece in ?? () #23 0x0000000000000000 in ?? () (gdb) f 2 #2 0x0000000000430491 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fc6fc31f300, mutex=...) at build/include/tbb/mutex.h:91 91 build/include/tbb/mutex.h: No such file or directory. (gdb) p mutex $1 = (tbb::mutex &) @0x7fc6d8286978: {static is_rw_mutex = false, static is_recursive_mutex = false, static is_fair_mutex = false, impl = {__data = {__lock = 2, __count = 0, __owner = 23786, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\352\\\000\000\001", '\000' , __align = 2}} (gdb) info threads Id Target Id Frame 9 LWP 23750 0x00007fc6fe7316dd in _jmpbuf_sp (regs=0x0) at ../sysdeps/x86_64/jmpbuf-unwind.h:39 8 LWP 23789 0x00007fc6fe730f2c in sem_unlink (name=0x6b81ce "UH\211\345SH\203\354\070H\211}\310H\211u\300H\213E\310H\215\220\250") at sem_unlink.c:52 7 LWP 23790 0x00007fc6fe730f2c in sem_unlink (name=0x7fc6ec2d7040 "\320A\202") at sem_unlink.c:52 6 LWP 23787 0x00007fc6fe7317eb in __pthread_disable_asynccancel () at ../nptl/sysdeps/unix/sysv/linux/x86_64/cancellation.S:111 5 LWP 23788 0x00007fc6fe730f2c in sem_unlink (name=0x6b81ce "UH\211\345SH\203\354\070H\211}\310H\211u\300H\213E\310H\215\220\250") at sem_unlink.c:52 4 LWP 23786 0x00007fc6fe730f2c in sem_unlink (name=0x6b81ce "UH\211\345SH\203\354\070H\211}\310H\211u\300H\213E\310H\215\220\250") at sem_unlink.c:52 3 LWP 23785 0x00007fc6fe730f2c in sem_unlink (name=0x6b81ce "UH\211\345SH\203\354\070H\211}\310H\211u\300H\213E\310H\215\220\250") at sem_unlink.c:52 2 LWP 23784 0x00007fc6fe730f2c in sem_unlink (name=0x7fc6d82ebed0 "\320A\202") at sem_unlink.c:52 * 1 LWP 23783 0x00007fc6fe730f2c in sem_unlink (name=0x7fc6d82868d0 "\320A\202") at sem_unlink.c:52 (gdb) t 3 [Switching to thread 3 (LWP 23785)] #0 0x00007fc6fe730f2c in sem_unlink (name=0x6b81ce "UH\211\345SH\203\354\070H\211}\310H\211u\300H\213E\310H\215\220\250") at sem_unlink.c:52 52 sem_unlink.c: No such file or directory. (gdb) bt #0 0x00007fc6fe730f2c in sem_unlink (name=0x6b81ce "UH\211\345SH\203\354\070H\211}\310H\211u\300H\213E\310H\215\220\250") at sem_unlink.c:52 #1 0x00000000004304d6 in tbb::mutex::scoped_lock::acquire (this=0x7fc6fbb1c830, mutex=...) at build/include/tbb/mutex.h:105 #2 0x0000000000430491 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fc6fbb1c830, mutex=...) at build/include/tbb/mutex.h:91 #3 0x00000000006d5ec6 in WorkQueue::MayBeStartRunner (this=0x7fc6f0310280) at controller/src/base/queue_task.h:239 #4 0x00000000006d1185 in WorkQueue::EnqueueInternal (this=0x7fc6f0310280, entry=...) at controller/src/base/queue_task.h:341 #5 0x00000000006cb84d in WorkQueue::Enqueue (this=0x7fc6f0310280, entry=...) at controller/src/base/queue_task.h:215 #6 0x00000000006b8289 in CdbIf::Db_AddColumn (this=0x7fc6d82ebed0, cl=...) at controller/src/gendb/cdb_if.cc:1309 #7 0x00000000004fefb8 in PopulateFlowIndexTables (fvalues=..., T2=@0x7fc6fbb1cee4: 168082780, T1=@0x7fc6fbb1cee8: 1721028, partition_no=0 '\000', dbif=0x7fc6d82ebed0) at controller/src/analytics/db_handler.cc:826 #8 0x00000000004ff538 in DbHandler::FlowTableInsert (this=0x7fc6d82ebd80, parent=..., header=...) at controller/src/analytics/db_handler.cc:943 #9 0x0000000000492dfe in Ruleeng::handle_flow_object (this=0x18ea590, parent=..., db=0x7fc6d82ebd80, header=...) at controller/src/analytics/ruleeng.cc:411 #10 0x0000000000492f29 in Ruleeng::rule_execute (this=0x18ea590, vmsgp=0x7fc6fbb1d2d0, uveproc=true, db=0x7fc6d82ebd80) at controller/src/analytics/ruleeng.cc:437 #11 0x000000000048eb5d in boost::_mfi::mf3::operator() (this=0x7fc6fbb1d1c8, p=0x18ea590, a1=0x7fc6fbb1d2d0, a2=true, a3=0x7fc6d82ebd80) at build/include/boost/bind/mem_fn_template.hpp:393 #12 0x000000000048e3d2 in boost::_bi::list4, boost::arg<1>, boost::arg<2>, boost::arg<3> >::operator(), boost::_bi::list3 > ( this=0x7fc6fbb1d1d8, f=..., a=...) at build/include/boost/bind/bind.hpp:447 #13 0x000000000048df75 in boost::_bi::bind_t, boost::_bi::list4, boost::arg<1>, boost::arg<2>, boost::arg<3> > >::operator() ( this=0x7fc6fbb1d1c8, a1=@0x7fc6fbb1d140: 0x7fc6fbb1d2d0, a2=@0x7fc6fbb1d13c: true, a3=@0x7fc6fbb1d130: 0x7fc6d82ebd80) at build/include/boost/bind/bind_template.hpp:116 #14 0x000000000048dbf8 in boost::detail::function::function_obj_invoker3, boost::_bi::list4, boost::arg<1>, boost::arg<2>, boost::arg<3> > >, bool, VizMsg const*, bool, DbHandler*>::invoke (function_obj_ptr=..., a0=0x7fc6fbb1d2d0, a1=true, a2=0x7fc6d82ebd80) at build/include/boost/function/function_template.hpp:132 #15 0x00000000004cca7a in boost::function3::operator() (this=0x7fc6fbb1d1c0, a0=0x7fc6fbb1d2d0, a1=true, a2=0x7fc6d82ebd80) at build/include/boost/function/function_template.hpp:763 #16 0x00000000004cb817 in SandeshGenerator::ProcessRules (this=0x7fc6d82eb840, vmsg=0x7fc6fbb1d2d0, rsc=true) at controller/src/analytics/generator.cc:231 #17 0x00000000004ca2bc in Generator::ReceiveSandeshMsg (this=0x7fc6d82eb840, vmsg=0x7fc6fbb1d2d0, rsc=true) at controller/src/analytics/generator.cc:75 #18 0x000000000049a510 in Collector::ReceiveSandeshMsg (this=0x18ea7f0, session=0x19cf3d0, msg=0x7fc6e431df90, rsc=true) at controller/src/analytics/collector.cc:173 #19 0x000000000063307a in SandeshServerConnection::ProcessSandeshMessage (this=0x1f16b10, msg=0x7fc6e431df90, resource=true) at tools/sandesh/library/cpp/sandesh_connection.cc:195 #20 0x0000000000622368 in ssm::Established::react (this=0x7fc6d8280f90, event=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:371 #21 0x0000000000630c8e in boost::statechart::custom_reaction::react (stt=..., evt=..., eventType=@0x7fc6fbb1d5a8: 0xc0ff20) at build/include/boost/statechart/custom_reaction.hpp:42 #22 0x000000000063098c in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (stt=..., evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:816 #23 0x00000000006305f9 in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction > > (this=0x7fc6d8280f90, evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:851 #24 0x0000000000630222 in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl, boost::statechart::custom_reaction, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (stt=..., evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:820 #25 0x000000000062fc87 in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react, boost::statechart::custom_reaction, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction > > (this=0x7fc6d8280f90, evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:851 #26 0x000000000062f3dd in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl >, boost::statechart::custom_reaction, boost::statechart::custom_reaction, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (stt=..., evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:820 #27 0x000000000062e849 in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react >, boost::statechart::custom_reaction, boost::statechart::custom_reaction, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na> > (this=0x7fc6d8280f90, evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:851 #28 0x000000000062d8ef in boost::statechart::simple_state, (boost::statechart::history_mode)0>::react_impl (this=0x7fc6d8280f90, evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:489 #29 0x00000000005f9c53 in boost::statechart::detail::send_function, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>::operator() (this=0x7fc6fbb1d780) at build/include/boost/statechart/state_machine.hpp:87 #30 0x0000000000628561 in boost::statechart::null_exception_translator::operator(), boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine, boost::statechart::null_exception_translator>::exception_event_handler> (this=0x1bbcd98, action=...) at build/include/boost/statechart/null_exception_translator.hpp:33 #31 0x0000000000626c6d in boost::statechart::state_machine, boost::statechart::null_exception_translator>::send_event (this=0x1bbcd40, evt=...) at build/include/boost/statechart/state_machine.hpp:897 #32 0x00000000006249f7 in boost::statechart::state_machine, boost::statechart::null_exception_translator>::process_event (this=0x1bbcd40, evt=...) at build/include/boost/statechart/state_machine.hpp:279 #33 0x000000000061f905 in SandeshStateMachine::DequeueEvent (this=0x1bbcd40, ec=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:718 ---Type to continue, or q to quit---q Quit (gdb) f 2 #2 0x0000000000430491 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fc6fbb1c830, mutex=...) at build/include/tbb/mutex.h:91 91 build/include/tbb/mutex.h: No such file or directory. (gdb) p mutex $2 = (tbb::mutex &) @0x7fc6f03102a0: {static is_rw_mutex = false, static is_recursive_mutex = false, static is_fair_mutex = false, impl = {__data = {__lock = 2, __count = 0, __owner = 23784, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\350\\\000\000\001", '\000' , __align = 2}} (gdb) t 2 [Switching to thread 2 (LWP 23784)] #0 0x00007fc6fe730f2c in sem_unlink (name=0x7fc6d82ebed0 "\320A\202") at sem_unlink.c:52 52 sem_unlink.c: No such file or directory. (gdb) bt #0 0x00007fc6fe730f2c in sem_unlink (name=0x7fc6d82ebed0 "\320A\202") at sem_unlink.c:52 #1 0x00000000004304d6 in tbb::mutex::scoped_lock::acquire (this=0x7fc6fbf1e300, mutex=...) at build/include/tbb/mutex.h:105 #2 0x0000000000430491 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fc6fbf1e300, mutex=...) at build/include/tbb/mutex.h:91 #3 0x00000000006a595b in CdbIf::Db_Uninit (this=0x7fc6d82ebed0, task_id="analytics::DbHandler", task_instance=181) at controller/src/gendb/cdb_if.cc:694 #4 0x00000000004f81d0 in DbHandler::UnInit (this=0x7fc6d82ebd80, instance=181) at controller/src/analytics/db_handler.cc:142 #5 0x00000000004caa1c in SandeshGenerator::StartDbifReinit (this=0x7fc6d82eb840) at controller/src/analytics/generator.cc:124 #6 0x00000000004ce396 in boost::_mfi::mf0::operator() (this=0x7fc6d82ebf40, p=0x7fc6d82eb840) at build/include/boost/bind/mem_fn_template.hpp:49 #7 0x00000000004cdeda in boost::_bi::list1 >::operator(), boost::_bi::list0> (this=0x7fc6d82ebf50, f=..., a=...) at build/include/boost/bind/bind.hpp:253 #8 0x00000000004cdb9f in boost::_bi::bind_t, boost::_bi::list1 > >::operator() (this=0x7fc6d82ebf40) at build/include/boost/bind/bind_template.hpp:20 #9 0x00000000004cd7ad in boost::detail::function::void_function_obj_invoker0, boost::_bi::list1 > >, void>::invoke (function_obj_ptr=...) at build/include/boost/function/function_template.hpp:153 #10 0x00000000004d356c in boost::function0::operator() (this=0x7fc6d82ebf38) at build/include/boost/function/function_template.hpp:763 #11 0x00000000006b7cfe in CdbIf::Db_BatchAddColumn (this=0x7fc6d82ebed0, done=false) at controller/src/gendb/cdb_if.cc:1297 #12 0x00000000006e2fd3 in boost::_mfi::mf1::operator() (this=0x7fc6f0310320, p=0x7fc6d82ebed0, a1=false) at build/include/boost/bind/mem_fn_template.hpp:165 #13 0x00000000006e0393 in boost::_bi::list2, boost::arg<1> >::operator(), boost::_bi::list1 > (this=0x7fc6f0310330, f=..., a=...) at build/include/boost/bind/bind.hpp:313 #14 0x00000000006dd086 in boost::_bi::bind_t, boost::_bi::list2, boost::arg<1> > >::operator() (this=0x7fc6f0310320, a1=@0x7fc6fbf1eb74: false) at build/include/boost/bind/bind_template.hpp:32 #15 0x00000000006d8ea4 in boost::detail::function::void_function_obj_invoker1, boost::_bi::list2, boost::arg<1> > >, void, bool>::invoke (function_obj_ptr=..., a0=false) at build/include/boost/function/function_template.hpp:153 #16 0x0000000000547f7d in boost::function1::operator() (this=0x7fc6f0310318, a0=false) at build/include/boost/function/function_template.hpp:763 #17 0x00000000006e99e5 in WorkQueue::OnExit (this=0x7fc6f0310280, done=false) at controller/src/base/queue_task.h:292 #18 0x00000000006e92ca in WorkQueue::RunnerDone (this=0x7fc6f0310280) at controller/src/base/queue_task.h:374 #19 0x00000000006e82c0 in QueueTaskRunner >::RunQueue (this=0x7fc6ec3bf3d0) at controller/src/base/queue_task.h:57 #20 0x00000000006e790c in QueueTaskRunner >::Run (this=0x7fc6ec3bf3d0) at controller/src/base/queue_task.h:36 #21 0x0000000000761a65 in TaskImpl::execute (this=0x7fc6ec05f8c0) at controller/src/base/task.cc:224 #22 0x00007fc6febe2ece in ?? () #23 0x0000000000000000 in ?? () (gdb) f 2 #2 0x0000000000430491 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fc6fbf1e300, mutex=...) at build/include/tbb/mutex.h:91 91 build/include/tbb/mutex.h: No such file or directory. (gdb) p mutex $3 = (tbb::mutex &) @0x7fc6d82ebf78: {static is_rw_mutex = false, static is_recursive_mutex = false, static is_fair_mutex = false, impl = {__data = {__lock = 2, __count = 0, __owner = 23785, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\351\\\000\000\001", '\000' , __align = 2}} (gdb) t 3 [Switching to thread 3 (LWP 23785)] #0 0x00007fc6fe730f2c in sem_unlink (name=0x6b81ce "UH\211\345SH\203\354\070H\211}\310H\211u\300H\213E\310H\215\220\250") at sem_unlink.c:52 52 sem_unlink.c: No such file or directory. (gdb) bt #0 0x00007fc6fe730f2c in sem_unlink (name=0x6b81ce "UH\211\345SH\203\354\070H\211}\310H\211u\300H\213E\310H\215\220\250") at sem_unlink.c:52 #1 0x00000000004304d6 in tbb::mutex::scoped_lock::acquire (this=0x7fc6fbb1c830, mutex=...) at build/include/tbb/mutex.h:105 #2 0x0000000000430491 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fc6fbb1c830, mutex=...) at build/include/tbb/mutex.h:91 #3 0x00000000006d5ec6 in WorkQueue::MayBeStartRunner (this=0x7fc6f0310280) at controller/src/base/queue_task.h:239 #4 0x00000000006d1185 in WorkQueue::EnqueueInternal (this=0x7fc6f0310280, entry=...) at controller/src/base/queue_task.h:341 #5 0x00000000006cb84d in WorkQueue::Enqueue (this=0x7fc6f0310280, entry=...) at controller/src/base/queue_task.h:215 #6 0x00000000006b8289 in CdbIf::Db_AddColumn (this=0x7fc6d82ebed0, cl=...) at controller/src/gendb/cdb_if.cc:1309 #7 0x00000000004fefb8 in PopulateFlowIndexTables (fvalues=..., T2=@0x7fc6fbb1cee4: 168082780, T1=@0x7fc6fbb1cee8: 1721028, partition_no=0 '\000', dbif=0x7fc6d82ebed0) at controller/src/analytics/db_handler.cc:826 #8 0x00000000004ff538 in DbHandler::FlowTableInsert (this=0x7fc6d82ebd80, parent=..., header=...) at controller/src/analytics/db_handler.cc:943 #9 0x0000000000492dfe in Ruleeng::handle_flow_object (this=0x18ea590, parent=..., db=0x7fc6d82ebd80, header=...) at controller/src/analytics/ruleeng.cc:411 #10 0x0000000000492f29 in Ruleeng::rule_execute (this=0x18ea590, vmsgp=0x7fc6fbb1d2d0, uveproc=true, db=0x7fc6d82ebd80) at controller/src/analytics/ruleeng.cc:437 #11 0x000000000048eb5d in boost::_mfi::mf3::operator() (this=0x7fc6fbb1d1c8, p=0x18ea590, a1=0x7fc6fbb1d2d0, a2=true, a3=0x7fc6d82ebd80) at build/include/boost/bind/mem_fn_template.hpp:393 #12 0x000000000048e3d2 in boost::_bi::list4, boost::arg<1>, boost::arg<2>, boost::arg<3> >::operator(), boost::_bi::list3 > ( this=0x7fc6fbb1d1d8, f=..., a=...) at build/include/boost/bind/bind.hpp:447 #13 0x000000000048df75 in boost::_bi::bind_t, boost::_bi::list4, boost::arg<1>, boost::arg<2>, boost::arg<3> > >::operator() ( this=0x7fc6fbb1d1c8, a1=@0x7fc6fbb1d140: 0x7fc6fbb1d2d0, a2=@0x7fc6fbb1d13c: true, a3=@0x7fc6fbb1d130: 0x7fc6d82ebd80) at build/include/boost/bind/bind_template.hpp:116 #14 0x000000000048dbf8 in boost::detail::function::function_obj_invoker3, boost::_bi::list4, boost::arg<1>, boost::arg<2>, boost::arg<3> > >, bool, VizMsg const*, bool, DbHandler*>::invoke (function_obj_ptr=..., a0=0x7fc6fbb1d2d0, a1=true, a2=0x7fc6d82ebd80) at build/include/boost/function/function_template.hpp:132 #15 0x00000000004cca7a in boost::function3::operator() (this=0x7fc6fbb1d1c0, a0=0x7fc6fbb1d2d0, a1=true, a2=0x7fc6d82ebd80) at build/include/boost/function/function_template.hpp:763 #16 0x00000000004cb817 in SandeshGenerator::ProcessRules (this=0x7fc6d82eb840, vmsg=0x7fc6fbb1d2d0, rsc=true) at controller/src/analytics/generator.cc:231 #17 0x00000000004ca2bc in Generator::ReceiveSandeshMsg (this=0x7fc6d82eb840, vmsg=0x7fc6fbb1d2d0, rsc=true) at controller/src/analytics/generator.cc:75 #18 0x000000000049a510 in Collector::ReceiveSandeshMsg (this=0x18ea7f0, session=0x19cf3d0, msg=0x7fc6e431df90, rsc=true) at controller/src/analytics/collector.cc:173 #19 0x000000000063307a in SandeshServerConnection::ProcessSandeshMessage (this=0x1f16b10, msg=0x7fc6e431df90, resource=true) at tools/sandesh/library/cpp/sandesh_connection.cc:195 #20 0x0000000000622368 in ssm::Established::react (this=0x7fc6d8280f90, event=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:371 #21 0x0000000000630c8e in boost::statechart::custom_reaction::react (stt=..., evt=..., eventType=@0x7fc6fbb1d5a8: 0xc0ff20) at build/include/boost/statechart/custom_reaction.hpp:42 #22 0x000000000063098c in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (stt=..., evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:816 #23 0x00000000006305f9 in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction > > (this=0x7fc6d8280f90, evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:851 #24 0x0000000000630222 in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl, boost::statechart::custom_reaction, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction >, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (stt=..., evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:820 #25 0x000000000062fc87 in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react, boost::statechart::custom_reaction, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction > > (this=0x7fc6d8280f90, evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:851 #26 0x000000000062f3dd in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl >, boost::statechart::custom_reaction, boost::statechart::custom_reaction, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, boost::statechart::simple_state, (boost::statechart::history_mode)0> > (stt=..., evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:820 #27 0x000000000062e849 in boost::statechart::simple_state, (boost::statechart::history_mode)0>::local_react >, boost::statechart::custom_reaction, boost::statechart::custom_reaction, boost::statechart::in_state_reaction >, boost::statechart::custom_reaction, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na> > (this=0x7fc6d8280f90, evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:851 #28 0x000000000062d8ef in boost::statechart::simple_state, (boost::statechart::history_mode)0>::react_impl (this=0x7fc6d8280f90, evt=..., eventType=0xc0ff20) at build/include/boost/statechart/simple_state.hpp:489 #29 0x00000000005f9c53 in boost::statechart::detail::send_function, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>::operator() (this=0x7fc6fbb1d780) at build/include/boost/statechart/state_machine.hpp:87 #30 0x0000000000628561 in boost::statechart::null_exception_translator::operator(), boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>, boost::statechart::state_machine, boost::statechart::null_exception_translator>::exception_event_handler> (this=0x1bbcd98, action=...) at build/include/boost/statechart/null_exception_translator.hpp:33 #31 0x0000000000626c6d in boost::statechart::state_machine, boost::statechart::null_exception_translator>::send_event (this=0x1bbcd40, evt=...) at build/include/boost/statechart/state_machine.hpp:897 #32 0x00000000006249f7 in boost::statechart::state_machine, boost::statechart::null_exception_translator>::process_event (this=0x1bbcd40, evt=...) at build/include/boost/statechart/state_machine.hpp:279 #33 0x000000000061f905 in SandeshStateMachine::DequeueEvent (this=0x1bbcd40, ec=...) at tools/sandesh/library/cpp/sandesh_state_machine.cc:718 ---Type to continue, or q to quit---q Quit (gdb) f 2 #2 0x0000000000430491 in tbb::mutex::scoped_lock::scoped_lock (this=0x7fc6fbb1c830, mutex=...) at build/include/tbb/mutex.h:91 91 build/include/tbb/mutex.h: No such file or directory. (gdb) p mutex $4 = (tbb::mutex &) @0x7fc6f03102a0: {static is_rw_mutex = false, static is_recursive_mutex = false, static is_fair_mutex = false, impl = {__data = {__lock = 2, __count = 0, __owner = 23784, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\350\\\000\000\001", '\000' , __align = 2}} (gdb) It's deadlock in acquiring mutexes involving cdbif, cdbif work queue and generator as above.