Thread4 executing the HttpSession::SendSession() is trying to acquire the HttpSession::mutex_ again - once before calling HttpSession::Send() and in TcpSession::Send() failure, it calls OnSessionEvent() via CloseInternal() which is where it tries again and deadlocks on itself.

root@a6s23:~# gdb /var/tmp/vizd.20 /var/tmp/core.10127
GNU gdb (Ubuntu/Linaro 7.4-2012.04-0ubuntu2.1) 7.4-2012.04
Copyright (C) 2012 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
For bug reporting instructions, please see:
<http://bugs.launchpad.net/gdb-linaro/>...
Reading symbols from /var/tmp/vizd.20...done.

warning: core file may not match specified executable file.
[New LWP 10947]
[New LWP 10948]
[New LWP 10949]
[New LWP 10950]
[New LWP 10127]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fff14ffe000
Core was generated by `contrail-collector'.
#0  0x00007fefeb9a5ee9 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) bt
#0  0x00007fefeb9a5ee9 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007fefed276d32 in ?? () from /usr/lib/libtbb_debug.so.2
#2  0x00007fefed27708d in ?? () from /usr/lib/libtbb_debug.so.2
#3  0x00007fefed289d9f in rml::internal::thread_monitor::commit_wait(rml::internal::thread_monitor::cookie&) () from /usr/lib/libtbb_debug.so.2
#4  0x00007fefed28943d in ?? () from /usr/lib/libtbb_debug.so.2
#5  0x00007fefed289270 in ?? () from /usr/lib/libtbb_debug.so.2
#6  0x00007fefecde0e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fefeb9a9ccd in personality () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()
(gdb) info threads
  Id   Target Id         Frame
  5    LWP 10127         0x00007fefecde789c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
  4    LWP 10950         0x00007fefecde789c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
  3    LWP 10949         0x00007fefeb9a5ee9 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
  2    LWP 10948         0x00007fefeb9a5ee9 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
* 1    LWP 10947         0x00007fefeb9a5ee9 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) thread apply all bt

Thread 5 (LWP 10127):
#0  0x00007fefecde789c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007fefecde3065 in _L_lock_858 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007fefecde2eba in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#3  0x00000000005cfa1e in lock (this=0xa4c560) at build/include/tbb/mutex.h:164
#4  acquire (mutex=..., this=0x7fff14e9e8e0) at build/include/tbb/mutex.h:105
#5  scoped_lock (mutex=..., this=0x7fff14e9e8e0) at build/include/tbb/mutex.h:91
#6  HttpSession::AcceptSession (this=0x1e5e250) at controller/src/http/http_session.cc:245
#7  0x00000000005ce0c7 in HttpServer::AcceptSession (this=<optimized out>, session=<optimized out>) at controller/src/http/http_server.cc:38
#8  0x000000000052abf8 in TcpServer::AcceptHandlerInternal (this=0x1db9190, server=..., error=...) at controller/src/io/tcp_server.cc:316
#9  0x00000000005312f3 in operator() (a1=..., p=<optimized out>, this=0x7fff14e9f0c0, a2=...) at build/include/boost/bind/mem_fn_template.hpp:280
#10 operator()<boost::_mfi::mf2<void, TcpServer, boost::intrusive_ptr<TcpServer>, const boost::system::error_code&>, boost::_bi::list1<const boost::system::error_code&> > (a=<synthetic pointer>, f=..., this=0x7fff14e9f0d0) at build/include/boost/bind/bind.hpp:392
#11 operator()<boost::system::error_code> (a1=..., this=0x7fff14e9f0c0) at build/include/boost/bind/bind_template.hpp:47
#12 operator() (this=0x7fff14e9f0c0) at build/include/boost/asio/detail/bind_handler.hpp:46
#13 asio_handler_invoke<boost::asio::detail::binder1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, TcpServer, boost::intrusive_ptr<TcpServer>, boost::system::error_code const&>, boost::_bi::list3<boost::_bi::value<TcpServer*>, boost::_bi::value<boost::intrusive_ptr<TcpServer> >, boost::arg<1> (*)()> >, boost::system::error_code> > (function=...) at build/include/boost/asio/handler_invoke_hook.hpp:64
#14 boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, TcpServer, boost::intrusive_ptr<TcpServer>, boost::system::error_code const&>, boost::_bi::list3<boost::_bi::value<TcpServer*>, boost::_bi::value<boost::intrusive_ptr<TcpServer> >, boost::arg<1> (*)()> >, boost::system::error_code>, boost::_bi::bind_t<void, boost::_mfi::mf2<void, TcpServer, boost::intrusive_ptr<TcpServer>, boost::system::error_code const&>, boost::_bi::list3<boost::_bi::value<TcpServer*>, boost::_bi::value<boost::intrusive_ptr<TcpServer> >, boost::arg<1> (*)()> > > (function=..., context=...) at build/include/boost/asio/detail/handler_invoke_helpers.hpp:39
#15 0x00000000005313c4 in boost::asio::detail::reactive_socket_accept_op<boost::asio::basic_socket<boost::asio::ip::tcp, boost::asio::stream_socket_service<boost::asio::ip::tcp> >, boost::asio::ip::tcp, boost::_bi::bind_t<void, boost::_mfi::mf2<void, TcpServer, boost::intrusive_ptr<TcpServer>, boost::system::error_code const&>, boost::_bi::list3<boost::_bi::value<TcpServer*>, boost::_bi::value<boost::intrusive_ptr<TcpServer> >, boost::arg<1> (*)()> > >::do_complete (owner=0x1d9e380, base=0x1da95a0)
    at build/include/boost/asio/detail/reactive_socket_accept_op.hpp:123
#16 0x00000000004b8091 in complete (owner=..., this=0x1db93d0, bytes_transferred=1, ec=...) at build/include/boost/asio/detail/task_io_service_operation.hpp:37
#17 do_run_one (ec=..., private_op_queue=..., this_thread=..., lock=..., this=<optimized out>) at build/include/boost/asio/detail/impl/task_io_service.ipp:366
#18 boost::asio::detail::task_io_service::run (this=0x1d9e380, ec=...) at build/include/boost/asio/detail/impl/task_io_service.ipp:146
#19 0x00000000005239b4 in run (ec=..., this=<optimized out>) at build/include/boost/asio/impl/io_service.ipp:68
#20 EventManager::Run (this=0x1d8aa40) at controller/src/io/event_manager.cc:32
#21 0x000000000041c849 in main (argc=<optimized out>, argv=<optimized out>) at controller/src/analytics/main.cc:365

Thread 4 (LWP 10950):
#0  0x00007fefecde789c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
#1  0x00007fefecde3065 in _L_lock_858 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007fefecde2eba in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0
#3  0x00000000005d06a2 in lock (this=0xa4c560) at build/include/tbb/mutex.h:164
#4  acquire (mutex=..., this=0x7fefe9e4d2b0) at build/include/tbb/mutex.h:105
#5  scoped_lock (mutex=..., this=0x7fefe9e4d2b0) at build/include/tbb/mutex.h:91
#6  HttpSession::OnSessionEvent (this=0x1dcee30, session=<optimized out>, event=TcpSession::CLOSE) at controller/src/http/http_session.cc:265
#7  0x00000000005389bd in operator() (a1=TcpSession::CLOSE, a0=0x1dcee30, this=0x1dceed8) at build/include/boost/function/function_template.hpp:763
#8  TcpSession::CloseInternal (this=0x1dcee30, callObserver=true) at controller/src/io/tcp_session.cc:225
#9  0x000000000053bbff in TcpSession::Send (this=<optimized out>, data=<optimized out>, size=4, sent=<optimized out>) at controller/src/io/tcp_session.cc:282
#10 0x0000000000572376 in HttpSession::SendSession (s=..., data=0x7fefe9e4d980 "61\r\n", size=4, sent=0x0) at controller/src/http/http_session.h:32
#11 0x0000000000572d26 in HttpSendXML (more=true, len=2494,
    buf=0x7fefe02c8ea0 "<NodeStatusUVE type=\"sandesh\"><data type=\"struct\" identifier=\"1\"><NodeStatus><name type=\"string\" identifier=\"1\" key=\"ObjectCollectorInfo\">nodeg21</name><process_status type=\"list\" identifier=\"4\" aggty"..., context=...,
    name=<optimized out>) at tools/sandesh/library/cpp/sandesh_http.cc:160
#12 SandeshHttp::Response (snh=0x7fefe0301200, context=...) at tools/sandesh/library/cpp/sandesh_http.cc:342
#13 0x000000000056b712 in SandeshUVE::Dispatch (this=0x7fefe0301200, sconn=<optimized out>) at tools/sandesh/library/cpp/sandesh.cc:566
#14 0x00000000004616f8 in Send (more=true, ctx=..., seqno=25, data=..., seq=<optimized out>) at build/production/analytics/analytics_types.cpp:2239
#15 SandeshUVEPerTypeMapImpl<NodeStatusUVE, NodeStatus>::SyncUVE (this=0xa49aa0, seqno=0, ctx=..., more=true) at build/include/sandesh/sandesh_uve.h:151
#16 0x00000000005ad148 in SandeshUVECacheReq::HandleRequest (this=0x7fefe0294040) at tools/sandesh/library/cpp/sandesh_uve.cc:62
#17 0x000000000056575a in Sandesh::ProcessRecv (rsnh=0x7fefe0294040) at tools/sandesh/library/cpp/sandesh.cc:393
#18 0x00000000005716ff in operator() (a0=0x7fefe0294040, this=0x7fefe9e4dc40) at build/include/boost/function/function_template.hpp:763
#19 QueueTaskRunner<SandeshRequest*, WorkQueue<SandeshRequest*> >::RunQueue (this=0x7fefe02362e0) at controller/src/base/queue_task.h:53
#20 0x00000000006b8700 in TaskImpl::execute (this=0x7fefe00c7fc0) at controller/src/base/task.cc:224
#21 0x00007fefed298ece in ?? () from /usr/lib/libtbb_debug.so.2
#22 0x00007fefed28fe0b in ?? () from /usr/lib/libtbb_debug.so.2
#23 0x00007fefed28e6f2 in ?? () from /usr/lib/libtbb_debug.so.2
#24 0x00007fefed2893ce in ?? () from /usr/lib/libtbb_debug.so.2
#25 0x00007fefed289270 in ?? () from /usr/lib/libtbb_debug.so.2
#26 0x00007fefecde0e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#27 0x00007fefeb9a9ccd in personality () from /lib/x86_64-linux-gnu/libc.so.6
#28 0x0000000000000000 in ?? ()
---Type <return> to continue, or q <return> to quit---

Thread 3 (LWP 10949):
#0  0x00007fefeb9a5ee9 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007fefed276d32 in ?? () from /usr/lib/libtbb_debug.so.2
#2  0x00007fefed27708d in ?? () from /usr/lib/libtbb_debug.so.2
#3  0x00007fefed289d9f in rml::internal::thread_monitor::commit_wait(rml::internal::thread_monitor::cookie&) () from /usr/lib/libtbb_debug.so.2
#4  0x00007fefed28943d in ?? () from /usr/lib/libtbb_debug.so.2
#5  0x00007fefed289270 in ?? () from /usr/lib/libtbb_debug.so.2
#6  0x00007fefecde0e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fefeb9a9ccd in personality () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 2 (LWP 10948):
#0  0x00007fefeb9a5ee9 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007fefed276d32 in ?? () from /usr/lib/libtbb_debug.so.2
#2  0x00007fefed27708d in ?? () from /usr/lib/libtbb_debug.so.2
#3  0x00007fefed289d9f in rml::internal::thread_monitor::commit_wait(rml::internal::thread_monitor::cookie&) () from /usr/lib/libtbb_debug.so.2
#4  0x00007fefed28943d in ?? () from /usr/lib/libtbb_debug.so.2
#5  0x00007fefed289270 in ?? () from /usr/lib/libtbb_debug.so.2
#6  0x00007fefecde0e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fefeb9a9ccd in personality () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()

Thread 1 (LWP 10947):
#0  0x00007fefeb9a5ee9 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1  0x00007fefed276d32 in ?? () from /usr/lib/libtbb_debug.so.2
#2  0x00007fefed27708d in ?? () from /usr/lib/libtbb_debug.so.2
#3  0x00007fefed289d9f in rml::internal::thread_monitor::commit_wait(rml::internal::thread_monitor::cookie&) () from /usr/lib/libtbb_debug.so.2
#4  0x00007fefed28943d in ?? () from /usr/lib/libtbb_debug.so.2
#5  0x00007fefed289270 in ?? () from /usr/lib/libtbb_debug.so.2
#6  0x00007fefecde0e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#7  0x00007fefeb9a9ccd in personality () from /lib/x86_64-linux-gnu/libc.so.6
#8  0x0000000000000000 in ?? ()
(gdb) t 5
[Switching to thread 5 (LWP 10127)]
#0  0x00007fefecde789c in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0
(gdb) f 3
#3  0x00000000005cfa1e in lock (this=0xa4c560) at build/include/tbb/mutex.h:164
164	build/include/tbb/mutex.h: No such file or directory.
(gdb) p *this
$1 = {static is_rw_mutex = false, static is_recursive_mutex = false, static is_fair_mutex = false, impl = {__data = {__lock = 2, __count = 0, __owner = 10950, __nusers = 1, __kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}},
    __size = "\002\000\000\000\000\000\000\000\306*\000\000\001", '\000' <repeats 26 times>, __align = 2}}
(gdb)