looks like we are doing a discovery heartbeat and seeing crash in curl library..
(gdb) #0 native_handle (impl=<error reading variable: Cannot access memory at address 0x10>, this=<optimized out>) at controller/src/http/client/http_curl.cc:480 #1 native_handle (this=<error reading variable: Cannot access memory at address 0x10>) at build/include/boost/asio/basic_socket.hpp:367 #2 opensocket (data=0x7f77a0004b10, purpose=<optimized out>, address=<optimized out>) at controller/src/http/client/http_curl.cc:302 #3 0x00000000005dd622 in Curl_socket (conn=0x7f77a0006288, ai=0x7f779400e978, addr=0x7f77a7621540, sockfd=0x7f77a7621530) at /ecbuilds/PipeLine/sb/third_party/curl-7.24.0/lib/connect.c:1189 #4 0x00000000005dcb3a in singleipconnect (conn=0x7f77a0006288, ai=0x7f779400e978, timeout_ms=0, sockp=0x7f77a7621694, connected=0x7f77a762171f) at /ecbuilds/PipeLine/sb/third_party/curl-7.24.0/lib/connect.c:851 #5 0x00000000005dd217 in Curl_connecthost (conn=0x7f77a0006288, remotehost=0x7f779400cc18, sockconn=0x7f77a0006430, addr=0x7f77a76216d8, connected=0x7f77a762171f) at /ecbuilds/PipeLine/sb/third_party/curl-7.24.0/lib/connect.c:1033 #6 0x00000000005f8752 in ConnectPlease (data=0x7f77a0008478, conn=0x7f77a0006288, connected=0x7f77a762171f) at /ecbuilds/PipeLine/sb/third_party/curl-7.24.0/lib/url.c:3229 #7 0x00000000005fb3fd in Curl_setup_conn (conn=0x7f77a0006288, protocol_done=0x7f77a76217fa) at /ecbuilds/PipeLine/sb/third_party/curl-7.24.0/lib/url.c:5146 #8 0x00000000005fb5c4 in Curl_connect (data=0x7f77a0008478, in_connect=0x7f77a0003d20, asyncp=0x7f77a76217f9, protocol_done=0x7f77a76217fa) at /ecbuilds/PipeLine/sb/third_party/curl-7.24.0/lib/url.c:5228 #9 0x00000000005df63a in multi_runsingle (multi=0x1041348, now=..., easy=0x7f77a0003d08) at /ecbuilds/PipeLine/sb/third_party/curl-7.24.0/lib/multi.c:1045 #10 0x00000000005e0b53 in curl_multi_perform (multi_handle=0x1041348, running_handles=0x7f77a76218b0) at /ecbuilds/PipeLine/sb/third_party/curl-7.24.0/lib/multi.c:1730 #11 0x00000000005cc3d5 in send_perform (g=0x103ff90, conn=<optimized out>) at controller/src/http/client/http_curl.cc:331 #12 0x00000000005c88be in HttpConnection::HttpProcessInternal(std::string, std::string, bool, bool, std::vector<std::string, std::allocator<std::string> >, boost::function<void (std::string&, boost::system::error_code&)>, http_method) (this=0x7f77a0004b10, body=..., path=..., header=<optimized out>, timeout=<optimized out>, hdr_options=..., cb=..., method=HTTP_POST) at controller/src/http/client/http_client.cc:217 #13 0x00000000005cb0bf in operator() (a7=HTTP_POST, a6=<error reading variable: access outside bounds of object referenced via synthetic pointer>, a5=<error reading variable: access outside bounds of object referenced via synthetic pointer>, a4=true, a3=false, a2=..., a1=..., p=0x7f77a0004b10, this=0x7f77a0008250) at build/include/boost/bind/mem_fn_template.hpp:845 #14 boost::_bi::list8<boost::_bi::value<HttpConnection*>, boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<bool>, boost::_bi::value<bool>, boost::_bi::value<std::vector<std::string, std::allocator<std::string> > >, boost::_bi::value<boost::function<void (std::string&, boost::system::error_code&)> >, boost::_bi::value<http_method> >::operator()<boost::_mfi::mf7<void, HttpConnection, std::string, std::string, bool, bool, std::vector<std::string, std::allocator<std::string> >, boost::function<void (std::string&, boost::system::error_code&)>, http_method>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf7<void, HttpConnection, std::string, std::string, bool, bool, std::vector<std::string, std::allocator<std::string> >, boost::function<void (std::string&, boost::system::error_code&)>, http_method>&, boost::_bi::list0&, int) (this=<optimized out>, f=..., a=...) at build/include/boost/bind/bind.hpp:747 #15 0x00000000005cb26b in operator() (this=<optimized out>) at build/include/boost/bind/bind_template.hpp:20 #16 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf7<void, HttpConnection, std::string, std::string, bool, bool, std::vector<std::string, std::allocator<std::string> >, boost::function<void (std::string&, boost::system::error_code&)>, http_method>, boost::_bi::list8<boost::_bi::value<HttpConnection*>, boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<bool>, boost::_bi::value<bool>, boost::_bi::value<std::vector<std::string, std::allocator<std::string> > >, boost::_bi::value<boost::function<void (std::string&, boost::system::error_code&)> >, boost::_bi::value<http_method> > >, void>::invoke(boost::detail::function::function_buffer&) (function_obj_ptr=...) at build/include/boost/function/function_template.hpp:153 #17 0x00000000005c6767 in operator() (this=<optimized out>) at build/include/boost/function/function_template.hpp:763 #18 HttpClient::DequeueEvent(boost::function<void ()>) (this=<optimized out>, cb=...) at controller/src/http/client/http_client.cc:376 #19 0x00000000005c9ee3 in operator() (a1=..., p=0x10409d0, this=0x7f77a7621bf8) at build/include/boost/bind/mem_fn_template.hpp:165 #20 operator()<bool, boost::_mfi::mf1<bool, HttpClient, boost::function<void()> >, boost::_bi::list1<boost::function<void()>&> > (a=<synthetic pointer>, f=..., this=0x7f77a7621c08) at build/include/boost/bind/bind.hpp:303 #21 operator()<boost::function<void()> > (a1=..., this=0x7f77a7621bf8) at build/include/boost/bind/bind_template.hpp:32 #22 boost::detail::function::function_obj_invoker1<boost::_bi::bind_t<bool, boost::_mfi::mf1<bool, HttpClient, boost::function<void ()> >, boost::_bi::list2<boost::_bi::value<HttpClient*>, boost::arg<1> > >, bool, boost::function<void ()> >::invoke(boost::detail::function::function_buffer&, boost::function<void ()>) (function_obj_ptr=..., a0=...) at build/include/boost/function/function_template.hpp:132 #23 0x0000000000510b29 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>, this=0x7f77a7621bf0) at build/include/boost/function/function_template.hpp:763 #24 QueueTaskRunner<boost::function<void ()>, WorkQueue<boost::function<void ()> > >::RunQueue() (this=0x7f77a00014f0) at controller/src/base/queue_task.h:53 #25 0x00000000006b85a0 in TaskImpl::execute (this=0x7f77a0005ac0) at controller/src/base/task.cc:224 #26 0x00007f77aa26aece in ?? () from /usr/lib/libtbb_debug.so.2 #27 0x00007f77aa261e0b in ?? () from /usr/lib/libtbb_debug.so.2 #28 0x00007f77aa2606f2 in ?? () from /usr/lib/libtbb_debug.so.2 #29 0x00007f77aa25b3ce in ?? () from /usr/lib/libtbb_debug.so.2 #30 0x00007f77aa25b270 in ?? () from /usr/lib/libtbb_debug.so.2 #31 0x00007f77a9db2e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #32 0x00007f77a897bccd in personality () from /lib/x86_64-linux-gnu/libc.so.6 #33 0x0000000000000000 in ?? () (gdb) frame 12 #12 0x00000000005c88be in HttpConnection::HttpProcessInternal(std::string, std::string, bool, bool, std::vector<std::string, std::allocator<std::string> >, boost::function<void (std::string&, boost::system::error_code&)>, http_method) (this=0x7f77a0004b10, body=..., path=..., header=<optimized out>, timeout=<optimized out>, hdr_options=..., cb=..., method=HTTP_POST) at controller/src/http/client/http_client.cc:217 217 controller/src/http/client/http_client.cc: No such file or directory. (gdb) info locals gi = 0x103ff90 curl_handle = <optimized out> __PRETTY_FUNCTION__ = "void HttpConnection::HttpProcessInternal(std::string, std::string, bool, bool, std::vector<std::basic_string<char> >, HttpConnection::HttpCb, http_method)" url = {static npos = <optimized out>, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f77a0007568 "http://10.204.216.31:5998/heartbeat"}} (gdb)
looks like we are doing a discovery heartbeat and seeing crash in curl library..
(gdb) src/http/ client/ http_curl. cc:480 boost/asio/ basic_socket. hpp:367 04b10, purpose=<optimized out>, address=<optimized out>) at controller/ src/http/ client/ http_curl. cc:302 06288, ai=0x7f779400e978, addr=0x7f77a762 1540, sockfd= 0x7f77a7621530) at /ecbuilds/ PipeLine/ sb/third_ party/curl- 7.24.0/ lib/connect. c:1189 06288, ai=0x7f779400e978, timeout_ms=0, sockp=0x7f77a76 21694, connected= 0x7f77a762171f) at /ecbuilds/ PipeLine/ sb/third_ party/curl- 7.24.0/ lib/connect. c:851 06288, remotehost= 0x7f779400cc18, sockconn= 0x7f77a0006430, addr=0x7f77a762 16d8, connected= 0x7f77a762171f) at /ecbuilds/ PipeLine/ sb/third_ party/curl- 7.24.0/ lib/connect. c:1033 08478, conn=0x7f77a000 6288, connected= 0x7f77a762171f) at /ecbuilds/ PipeLine/ sb/third_ party/curl- 7.24.0/ lib/url. c:3229 06288, protocol_ done=0x7f77a762 17fa) at /ecbuilds/ PipeLine/ sb/third_ party/curl- 7.24.0/ lib/url. c:5146 08478, in_connect= 0x7f77a0003d20, asyncp= 0x7f77a76217f9, protocol_ done=0x7f77a762 17fa) at /ecbuilds/ PipeLine/ sb/third_ party/curl- 7.24.0/ lib/url. c:5228 3d08) at /ecbuilds/ PipeLine/ sb/third_ party/curl- 7.24.0/ lib/multi. c:1045 handle= 0x1041348, running_ handles= 0x7f77a76218b0) at /ecbuilds/ PipeLine/ sb/third_ party/curl- 7.24.0/ lib/multi. c:1730 src/http/ client/ http_curl. cc:331 :HttpProcessInt ernal(std: :string, std::string, bool, bool, std::vector< std::string, std::allocator< std::string> >, boost:: function< void (std::string&, boost:: system: :error_ code&)> , http_method) (this=0x7f77a00 04b10, body=..., src/http/ client/ http_client. cc:217 8250) at build/include/ boost/bind/ mem_fn_ template. hpp:845 _bi::list8< boost:: _bi::value< HttpConnection* >, boost:: _bi::value< std::string> , boost:: _bi::value< std::string> , boost:: _bi::value< bool>, boost:: _bi::value< bool>, boost:: _bi::value< std::vector< std::string, std::allocator< std::string> > >, boost:: _bi::value< boost:: function< void (std::string&, boost:: system: :error_ code&)> >, boost:: _bi::value< http_method> >::operator( )<boost: :_mfi:: mf7<void, HttpConnection, std::string, std::string, bool, bool, std::vector< std::string, std::allocator< std::string> >, boost:: function< void (std::string&, boost:: system: :error_ code&)> , http_method>, boost:: _bi::list0> (boost: :_bi::type< void>, boost:: _mfi::mf7< void, HttpConnection, std::string, std::string, bool, bool, std::vector< std::string, std::allocator< std::string> >, boost:: function< void (std::string&, boost:: system: :error_ code&)> , http_method>&, boost::_bi::list0&, int) (this=<optimized out>, f=..., a=...) at build/include/ boost/bind/ bind.hpp: 747 boost/bind/ bind_template. hpp:20 detail: :function: :void_function_ obj_invoker0< boost:: _bi::bind_ t<void, boost:: _mfi::mf7< void, HttpConnection, std::string, std::string, bool, bool, std::vector< std::string, std::allocator< std::string> >, boost:: function< void (std::string&, boost:: system: :error_ code&)> , http_method>, boost:: _bi::list8< boost:: _bi::value< HttpConnection* >, boost:: _bi::value< std::string> , boost:: _bi::value< std::string> , boost:: _bi::value< bool>, boost:: _bi::value< bool>, boost:: _bi::value< std::vector< std::string, std::allocator< std::string> > >, boost:: _bi::value< boost:: function< void (std::string&, boost:: system: :error_ code&)> >, boost:: _bi::value< http_method> > >, void>:: invoke( boost:: detail: :function: :function_ buffer& ) (function_ obj_ptr= ...) at build/include/ boost/function/ function_ template. hpp:153 boost/function/ function_ template. hpp:763 :DequeueEvent( boost:: function< void ()>) (this=<optimized out>, cb=...) at controller/ src/http/ client/ http_client. cc:376 1bf8) at build/include/ boost/bind/ mem_fn_ template. hpp:165 _mfi::mf1< bool, HttpClient, boost:: function< void()> >, boost:: _bi::list1< boost:: function< void()> &> > (a=<synthetic pointer>, f=..., this=0x7f77a762 1c08) at build/include/ boost/bind/ bind.hpp: 303 )<boost: :function< void()> > (a1=..., this=0x7f77a762 1bf8) at build/include/ boost/bind/ bind_template. hpp:32 detail: :function: :function_ obj_invoker1< boost:: _bi::bind_ t<bool, boost:: _mfi::mf1< bool, HttpClient, boost:: function< void ()> >, boost:: _bi::list2< boost:: _bi::value< HttpClient* >, boost::arg<1> > >, bool, boost:: function< void ()> >::invoke( boost:: detail: :function: :function_ buffer& , boost:: function< void ()>) (function_ obj_ptr= ..., a0=...) at build/include/ boost/function/ function_ template. hpp:132 1bf0) at build/include/ boost/function/ function_ template. hpp:763 <boost: :function< void ()>, WorkQueue< boost:: function< void ()> > >::RunQueue() (this=0x7f77a00 014f0) at controller/ src/base/ queue_task. h:53 05ac0) at controller/ src/base/ task.cc: 224 libtbb_ debug.so. 2 libtbb_ debug.so. 2 libtbb_ debug.so. 2 libtbb_ debug.so. 2 libtbb_ debug.so. 2 64-linux- gnu/libpthread. so.0 64-linux- gnu/libc. so.6 :HttpProcessInt ernal(std: :string, std::string, bool, bool, std::vector< std::string, std::allocator< std::string> >, boost:: function< void (std::string&, boost:: system: :error_ code&)> , http_method) (this=0x7f77a00 04b10, body=..., src/http/ client/ http_client. cc:217 src/http/ client/ http_client. cc: No such file or directory. :HttpProcessInt ernal(std: :string, std::string, bool, bool, std::vector< std::basic_ string< char> >, HttpConnection: :HttpCb, http_method)" allocator< char>> = {<__gnu_ cxx::new_ allocator< char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f77a0007568 "http:// 10.204. 216.31: 5998/heartbeat"}}
#0 native_handle (impl=<error reading variable: Cannot access memory at address 0x10>, this=<optimized out>) at controller/
#1 native_handle (this=<error reading variable: Cannot access memory at address 0x10>) at build/include/
#2 opensocket (data=0x7f77a00
#3 0x00000000005dd622 in Curl_socket (conn=0x7f77a00
#4 0x00000000005dcb3a in singleipconnect (conn=0x7f77a00
#5 0x00000000005dd217 in Curl_connecthost (conn=0x7f77a00
#6 0x00000000005f8752 in ConnectPlease (data=0x7f77a00
#7 0x00000000005fb3fd in Curl_setup_conn (conn=0x7f77a00
#8 0x00000000005fb5c4 in Curl_connect (data=0x7f77a00
#9 0x00000000005df63a in multi_runsingle (multi=0x1041348, now=..., easy=0x7f77a000
#10 0x00000000005e0b53 in curl_multi_perform (multi_
#11 0x00000000005cc3d5 in send_perform (g=0x103ff90, conn=<optimized out>) at controller/
#12 0x00000000005c88be in HttpConnection:
path=..., header=<optimized out>, timeout=<optimized out>, hdr_options=..., cb=..., method=HTTP_POST) at controller/
#13 0x00000000005cb0bf in operator() (a7=HTTP_POST, a6=<error reading variable: access outside bounds of object referenced via synthetic pointer>, a5=<error reading variable: access outside bounds of object referenced via synthetic pointer>, a4=true, a3=false,
a2=..., a1=..., p=0x7f77a0004b10, this=0x7f77a000
#14 boost::
#15 0x00000000005cb26b in operator() (this=<optimized out>) at build/include/
#16 boost::
#17 0x00000000005c6767 in operator() (this=<optimized out>) at build/include/
#18 HttpClient:
#19 0x00000000005c9ee3 in operator() (a1=..., p=0x10409d0, this=0x7f77a762
#20 operator()<bool, boost::
#21 operator(
#22 boost::
#23 0x0000000000510b29 in operator() (a0=<error reading variable: access outside bounds of object referenced via synthetic pointer>, this=0x7f77a762
#24 QueueTaskRunner
#25 0x00000000006b85a0 in TaskImpl::execute (this=0x7f77a00
#26 0x00007f77aa26aece in ?? () from /usr/lib/
#27 0x00007f77aa261e0b in ?? () from /usr/lib/
#28 0x00007f77aa2606f2 in ?? () from /usr/lib/
#29 0x00007f77aa25b3ce in ?? () from /usr/lib/
#30 0x00007f77aa25b270 in ?? () from /usr/lib/
#31 0x00007f77a9db2e9a in start_thread () from /lib/x86_
#32 0x00007f77a897bccd in personality () from /lib/x86_
#33 0x0000000000000000 in ?? ()
(gdb) frame 12
#12 0x00000000005c88be in HttpConnection:
path=..., header=<optimized out>, timeout=<optimized out>, hdr_options=..., cb=..., method=HTTP_POST) at controller/
217 controller/
(gdb) info locals
gi = 0x103ff90
curl_handle = <optimized out>
__PRETTY_FUNCTION__ = "void HttpConnection:
url = {static npos = <optimized out>, _M_dataplus = {<std::
(gdb)