When there is connectivity issues with neutron and etcd the locking does not work.
This is a result of the commit 1a57bbcc24021a6c237cad6a26f5035f0fc8fa92
The stack trace is below:
2019-08-22 07:37:16.684 179 DEBUG vmware_nsx.common.locking [req-7f3d6482-612f-4f48-96ea-c97ee81bbefd 16526abbc46d4fda91208694b0f87943 8c0647f98f5249b0ab6ee6c20cf19145 - default default] Lock df38f0f9-38d9-4fe7-a8c5-d21cb664d3d2 taken with stack trace [('/usr/lib/python2.7/site-packages/eventlet/greenthread.py', 219, 'main', 'result = function(*args, **kwargs)'), ('/usr/lib/python2.7/site-packages/eventlet/wsgi.py', 789, 'process_request', 'proto.__init__(conn_state, self)'), ('/usr/lib/python2.7/site-packages/eventlet/wsgi.py', 343, '__init__', 'self.handle()'), ('/usr/lib/python2.7/site-packages/eventlet/wsgi.py', 376, 'handle', 'self.handle_one_request()'), ('/usr/lib/python2.7/site-packages/eventlet/wsgi.py', 450, 'handle_one_request', 'self.handle_one_response()'), ('/usr/lib/python2.7/site-packages/eventlet/wsgi.py', 547, 'handle_one_response', 'result = self.application(self.environ, start_response)'), ('/usr/lib/python2.7/site-packages/paste/urlmap.py', 216, '__call__', 'return app(environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 129, '__call__', 'resp = self.call_func(req, *args, **kw)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 193, 'call_func', 'return self.func(req, *args, **kwargs)'), ('/usr/lib/python2.7/site-packages/oslo_middleware/base.py', 130, '__call__', 'response = req.get_response(self.application)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1314, 'send', 'application, catch_exc_info=False)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1278, 'call_application', 'app_iter = application(self.environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 129, '__call__', 'resp = self.call_func(req, *args, **kw)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 193, 'call_func', 'return self.func(req, *args, **kwargs)'), ('/usr/lib/python2.7/site-packages/oslo_middleware/base.py', 130, '__call__', 'response = req.get_response(self.application)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1314, 'send', 'application, catch_exc_info=False)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1278, 'call_application', 'app_iter = application(self.environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 129, '__call__', 'resp = self.call_func(req, *args, **kw)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 193, 'call_func', 'return self.func(req, *args, **kwargs)'), ('/usr/lib/python2.7/site-packages/oslo_middleware/request_id.py', 58, '__call__', 'response = req.get_response(self.application)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1314, 'send', 'application, catch_exc_info=False)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1278, 'call_application', 'app_iter = application(self.environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 129, '__call__', 'resp = self.call_func(req, *args, **kw)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 193, 'call_func', 'return self.func(req, *args, **kwargs)'), ('/usr/lib/python2.7/site-packages/osprofiler/web.py', 112, '__call__', 'return request.get_response(self.application)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1314, 'send', 'application, catch_exc_info=False)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1278, 'call_application', 'app_iter = application(self.environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 129, '__call__', 'resp = self.call_func(req, *args, **kw)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 193, 'call_func', 'return self.func(req, *args, **kwargs)'), ('/usr/lib/python2.7/site-packages/oslo_middleware/catch_errors.py', 40, '__call__', 'response = req.get_response(self.application)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1314, 'send', 'application, catch_exc_info=False)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1278, 'call_application', 'app_iter = application(self.environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 129, '__call__', 'resp = self.call_func(req, *args, **kw)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 193, 'call_func', 'return self.func(req, *args, **kwargs)'), ('/usr/lib/python2.7/site-packages/keystonemiddleware/auth_token/__init__.py', 332, '__call__', 'response = req.get_response(self._app)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1314, 'send', 'application, catch_exc_info=False)'), ('/usr/lib/python2.7/site-packages/webob/request.py', 1278, 'call_application', 'app_iter = application(self.environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 143, '__call__', 'return resp(environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 143, '__call__', 'return resp(environ, start_response)'), ('/usr/lib/python2.7/site-packages/routes/middleware.py', 141, '__call__', 'response = self.app(environ, start_response)'), ('/usr/lib/python2.7/site-packages/webob/dec.py', 143, '__call__', 'return resp(environ, start_response)'), ('/usr/lib/python2.7/site-packages/pecan/middleware/recursive.py', 56, '__call__', 'return self.application(environ, start_response)'), ('/usr/lib/python2.7/site-packages/pecan/core.py', 840, '__call__', 'return super(Pecan, self).__call__(environ, start_response)'), ('/usr/lib/python2.7/site-packages/pecan/core.py', 683, '__call__', 'self.invoke_controller(controller, args, kwargs, state)'), ('/usr/lib/python2.7/site-packages/pecan/core.py', 574, 'invoke_controller', 'result = controller(*args, **kwargs)'), ('/usr/lib/python2.7/site-packages/neutron_lib/db/api.py', 135, 'wrapped', 'return f(*args, **kwargs)'), ('/usr/lib/python2.7/site-packages/oslo_db/api.py', 142, 'wrapper', 'return f(*args, **kwargs)'), ('/usr/lib/python2.7/site-packages/neutron_lib/db/api.py', 179, 'wrapped', 'return f(*dup_args, **dup_kwargs)'), ('/usr/lib/python2.7/site-packages/neutron/pecan_wsgi/controllers/utils.py', 76, 'wrapped', 'return f(*args, **kwargs)'), ('/usr/lib/python2.7/site-packages/neutron/pecan_wsgi/controllers/resource.py', 159, 'post', 'return self.create(resources)'), ('/usr/lib/python2.7/site-packages/neutron/pecan_wsgi/controllers/resource.py', 177, 'create', 'return {key: creator(*creator_args, **creator_kwargs)}'), ('/usr/lib/python2.7/site-packages/vmware_nsx/plugins/nsx_v/plugin.py', 2964, 'create_subnet', "with locking.LockManager.get_lock(subnet['subnet']['network_id']):"), ('/usr/lib/python2.7/site-packages/vmware_nsx/common/locking.py', 40, 'get_lock', 'traceback.extract_stack())')] get_lock /usr/lib/python2.7/site-packages/vmware_nsx/common/locking.py:40^[[00m
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation [req-7f3d6482-612f-4f48-96ea-c97ee81bbefd 16526abbc46d4fda91208694b0f87943 8c0647f98f5249b0ab6ee6c20cf19145 - default default] POST failed.: LockAcquireFailed: Acquiring lock df38f0f9-38d9-4fe7-a8c5-d21cb664d3d2 failed
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation Traceback (most recent call last):
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/pecan/core.py", line 683, in __call__
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation self.invoke_controller(controller, args, kwargs, state)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/pecan/core.py", line 574, in invoke_controller
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation result = controller(*args, **kwargs)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/neutron_lib/db/api.py", line 139, in wrapped
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation setattr(e, '_RETRY_EXCEEDED', True)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/neutron_lib/db/api.py", line 135, in wrapped
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/oslo_db/api.py", line 154, in wrapper
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation ectxt.value = e.inner_exc
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/oslo_db/api.py", line 142, in wrapper
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/neutron_lib/db/api.py", line 183, in wrapped
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation LOG.debug("Retry wrapper got retriable exception: %s", e)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation self.force_reraise()
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation six.reraise(self.type_, self.value, self.tb)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/neutron_lib/db/api.py", line 179, in wrapped
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation return f(*dup_args, **dup_kwargs)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/neutron/pecan_wsgi/controllers/utils.py", line 76, in wrapped
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation return f(*args, **kwargs)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/neutron/pecan_wsgi/controllers/resource.py", line 159, in post
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation return self.create(resources)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/neutron/pecan_wsgi/controllers/resource.py", line 177, in create
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation return {key: creator(*creator_args, **creator_kwargs)}
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/vmware_nsx/plugins/nsx_v/plugin.py", line 2964, in create_subnet
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation with locking.LockManager.get_lock(subnet['subnet']['network_id']):
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation File "/usr/lib/python2.7/site-packages/tooz/locking.py", line 55, in __enter__
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation raise coordination.LockAcquireFailed(msg)
2019-08-22 07:37:16.698 179 ERROR neutron.pecan_wsgi.hooks.translation LockAcquireFailed: Acquiring lock df38f0f9-38d9-4fe7-a8c5-d21cb664d3d2 failed
