Exception in neutron port-list with simultaneous VM delete

Exception in seen in nova while executing neutron port-list after VM delete.

127.0.0.1 - - [2015-03-18 09:24:35] "DELETE /virtual-machine/9516c312-eed0-4cb5-ac39-395912344798 HTTP/1.1" 200 115 0.007674
192.168.0.101 - - [2015-03-18 09:24:35] "POST /neutron/port HTTP/1.1" 200 115 0.066042
192.168.0.101 - - [2015-03-18 09:24:35] "GET /virtual-machine/9516c312-eed0-4cb5-ac39-395912344798 HTTP/1.1" 404 198 0.003772
192.168.0.101 - - [2015-03-18 09:24:35] "GET /virtual-machine/9516c312-eed0-4cb5-ac39-395912344798 HTTP/1.1" 404 198 0.002407
192.168.0.101 - - [2015-03-18 09:24:35] "GET /virtual-machine/9516c312-eed0-4cb5-ac39-395912344798 HTTP/1.1" 404 198 0.001352
127.0.0.1 - - [2015-03-18 09:24:35] "GET /logical-routers?count=False&obj_uuids=9516c312-eed0-4cb5-ac39-395912344798&detail=True HTTP/1.1" 404 171 0.001748
<class 'cfgm_common.exceptions.NoIdError'>
Python 2.7.3: /usr/bin/python
Wed Mar 18 09:24:35 2015

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_interface.py in plugin_get_ports(self=<vnc_openstack.neutron_plugin_interface.NeutronPluginInterface object>, context={u'is_admin': True, u'operation': u'READALL', u'roles': [u'admin'], u'tenant': u'a929311426664f5899862c311a13e4a6', u'tenant_id': u'a929311426664f5899862c311a13e4a6', u'type': u'port', u'user_id': u'1cdd0ebf37e84849b23909dec43ba7b1'}, port={u'fields': [], u'filters': {u'device_id': [u'9516c312-eed0-4cb5-ac39-395912344798']}})
  433 try:
  434 cfgdb = self._get_user_cfgdb(context)
  435 ports_info = cfgdb.port_list(context, filters)
  436 return json.dumps(ports_info)
  437 except Exception as e:
ports_info undefined
cfgdb = <vnc_openstack.neutron_plugin_db.DBInterface object>
cfgdb.port_list = <bound method DBInterface.port_list of <vnc_openstack.neutron_plugin_db.DBInterface object>>
context = {u'is_admin': True, u'operation': u'READALL', u'roles': [u'admin'], u'tenant': u'a929311426664f5899862c311a13e4a6', u'tenant_id': u'a929311426664f5899862c311a13e4a6', u'type': u'port', u'user_id': u'1cdd0ebf37e84849b23909dec43ba7b1'}
filters = {u'device_id': [u'9516c312-eed0-4cb5-ac39-395912344798']}

 /usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py in port_list(self=<vnc_openstack.neutron_plugin_db.DBInterface object>, context={u'is_admin': True, u'operation': u'READALL', u'roles': [u'admin'], u'tenant': u'a929311426664f5899862c311a13e4a6', u'tenant_id': u'a929311426664f5899862c311a13e4a6', u'type': u'port', u'user_id': u'1cdd0ebf37e84849b23909dec43ba7b1'}, filters={u'device_id': [u'9516c312-eed0-4cb5-ac39-395912344798']})
 3659 # port has a back_ref to LR, so need to read in LRs based on device id
 3660 device_ids = filters['device_id']
 3661 router_objs = self._logical_router_list(obj_uuids=device_ids)
 3662 more_ports = []
 3663 for router_obj in router_objs:
router_objs undefined
self = <vnc_openstack.neutron_plugin_db.DBInterface object>
self._logical_router_list = <bound method DBInterface._logical_router_list o..._openstack.neutron_plugin_db.DBInterface object>>
obj_uuids undefined
device_ids = [u'9516c312-eed0-4cb5-ac39-395912344798']

 /usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py in _logical_router_list(self=<vnc_openstack.neutron_plugin_db.DBInterface object>, parent_id=None, back_ref_id=None, obj_uuids=[u'9516c312-eed0-4cb5-ac39-395912344798'], fields=None)
  543 obj_uuids=obj_uuids,
  544 detail=True,
  545 fields=fields)
  546 return rtr_obj
  547 #end _logical_router_list
fields = None

 /usr/lib/python2.7/dist-packages/vnc_api/gen/vnc_api_client_gen.py in logical_routers_list(self=<vnc_api.vnc_api.VncApi object>, parent_id=None, parent_fq_name=None, back_ref_id=None, obj_uuids=[u'9516c312-eed0-4cb5-ac39-395912344798'], fields=None, detail=True, count=False)
 6824 content = self._request_server(rest.OP_GET,
 6825 vnc_api.gen.resource_client.LogicalRouter.create_uri,
 6826 data = query_params)
 6827
 6828 if not detail:
data undefined
query_params = {'count': False, 'detail': True, 'obj_uuids': u'9516c312-eed0-4cb5-ac39-395912344798'}

 /usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _request_server(self=<vnc_api.vnc_api.VncApi object>, op=2, url=u'/logical-routers', data={'count': False, 'detail': True, 'obj_uuids': u'9516c312-eed0-4cb5-ac39-395912344798'}, retry_on_error=True, retry_after_authn=False, retry_count=30)
  361 return self._request(op, url, data=data, retry_on_error=retry_on_error,
  362 retry_after_authn=retry_after_authn,
  363 retry_count=retry_count)
  364
  365 def _request(self, op, url, data=None, retry_on_error=True,
retry_count = 30

 /usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py in _request(self=<vnc_api.vnc_api.VncApi object>, op=2, url=u'/logical-routers', data={'count': False, 'detail': True, 'obj_uuids': u'9516c312-eed0-4cb5-ac39-395912344798'}, retry_on_error=True, retry_after_authn=False, retry_count=30)
  402 elif status == 404:
  403 raise NoIdError('Error: oper %s url %s body %s response %s'
  404 % (op, url, data, content))
  405 elif status == 403:
  406 raise PermissionDenied(content)
op = 2
url = u'/logical-routers'
data = {'count': False, 'detail': True, 'obj_uuids': u'9516c312-eed0-4cb5-ac39-395912344798'}
content = u'Unknown id: 9516c312-eed0-4cb5-ac39-395912344798'
<class 'cfgm_common.exceptions.NoIdError'>: Unknown id: Error: oper 2 url /logical-routers body {'count': False, 'obj_uuids': u'9516c312-eed0-4cb5-ac39-395912344798', 'detail': True} response Unknown id: 9516c312-eed0-4cb5-ac39-395912344798
    __class__ = <class 'cfgm_common.exceptions.NoIdError'>
    __delattr__ = <method-wrapper '__delattr__' of NoIdError object>
    __dict__ = {'_unknown_id': u"Error: oper 2 url /logical-routers body {'coun...Unknown id: 9516c312-eed0-4cb5-ac39-395912344798"}
    __doc__ = None
    __format__ = <built-in method __format__ of NoIdError object>
    __getattribute__ = <method-wrapper '__getattribute__' of NoIdError object>
    __getitem__ = <method-wrapper '__getitem__' of NoIdError object>
    __getslice__ = <method-wrapper '__getslice__' of NoIdError object>
    __hash__ = <method-wrapper '__hash__' of NoIdError object>
    __init__ = <bound method NoIdError.__init__ of NoIdError()>
    __module__ = 'cfgm_common.exceptions'
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of NoIdError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of NoIdError object>
    __repr__ = <method-wrapper '__repr__' of NoIdError object>
    __setattr__ = <method-wrapper '__setattr__' of NoIdError object>
    __setstate__ = <built-in method __setstate__ of NoIdError object>
    __sizeof__ = <built-in method __sizeof__ of NoIdError object>
    __str__ = <bound method NoIdError.__str__ of NoIdError()>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of NoIdError object>
    __weakref__ = None
    _unknown_id = u"Error: oper 2 url /logical-routers body {'coun...Unknown id: 9516c312-eed0-4cb5-ac39-395912344798"
    args = ()
    message = ''

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_interface.py", line 435, in plugin_get_ports
    ports_info = cfgdb.port_list(context, filters)
  File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py", line 3661, in port_list
    router_objs = self._logical_router_list(obj_uuids=device_ids)
  File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_db.py", line 545, in _logical_router_list
    fields=fields)
  File "/usr/lib/python2.7/dist-packages/vnc_api/gen/vnc_api_client_gen.py", line 6826, in logical_routers_list
    data = query_params)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 363, in _request_server
    retry_count=retry_count)
  File "/usr/lib/python2.7/dist-packages/vnc_api/vnc_api.py", line 404, in _request
    % (op, url, data, content))
NoIdError: Unknown id: Error: oper 2 url /logical-routers body {'count': False, 'obj_uuids': u'9516c312-eed0-4cb5-ac39-395912344798', 'detail': True} response Unknown id: 9516c312-eed0-4cb5-ac39-395912344798

Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 764, in _handle
    return route.call(**args)
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 1625, in wrapper
    rv = callback(*a, **ka)
  File "/usr/lib/python2.7/dist-packages/bottle.py", line 1575, in wrapper
    rv = callback(*a, **ka)
  File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_interface.py", line 473, in plugin_http_post_port
    return self.plugin_get_ports(context, port)
  File "/usr/lib/python2.7/dist-packages/vnc_openstack/neutron_plugin_interface.py", line 439, in plugin_get_ports
    raise e
NoIdError: Unknown id: Error: oper 2 url /logical-routers body {'count': False, 'obj_uuids': u'9516c312-eed0-4cb5-ac39-395912344798', 'detail': True} response Unknown id: 9516c312-eed0-4cb5-ac39-395912344798

neutron/server.log
===============

2015-03-18 09:24:35.908 ERROR [neutron.api.v2.resource] index failed
Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/neutron/api/v2/resource.py", line 84, in resource
    result = method(request=request, **args)
  File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 273, in index
    return self._items(request, True, parent_id)
  File "/usr/lib/python2.7/dist-packages/neutron/api/v2/base.py", line 227, in _items
    obj_list = obj_getter(request.context, **kwargs)
  File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin.py", line 528, in get_ports
    for p in self._list_resource('port', context, filters, fields)]
  File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin.py", line 324, in _list_resource
    fields=fields, obj_name=res_type)
  File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin.py", line 233, in _transform_response
    self._raise_contrail_error(info, obj_name)
  File "/usr/lib/python2.7/dist-packages/neutron_plugin_contrail/plugins/opencontrail/contrail_plugin.py", line 248, in _raise_contrail_error
    raise exc.NeutronException(**info)
NeutronException: An unknown exception occurred.

nova/nova-api.log
==============
2015-03-18 09:24:30.829 6525 INFO nova.osapi_compute.wsgi.server [req-ec8a107c-8058-418d-881a-c2da75c1e617 1cdd0ebf37e84849b23909dec43ba7b1 a929311426664f5899862c311a13e4a6] 192.168.0.101 "DELETE /v1.1/a929311426664f5899862c311a13e4a6/servers/9516c312-eed0-4cb5-ac39-395912344798 HTTP/1.1" status: 204 len: 179 time: 0.1509612

2015-03-18 09:24:35.911 6525 ERROR nova.api.openstack [req-afdbfca5-9c3c-4a82-b5ca-c5edc87c33a9 1cdd0ebf37e84849b23909dec43ba7b1 a929311426664f5899862c311a13e4a6] Caught error: 500-{u'NeutronError': {u'message': u'An unknown exception occurred.', u'type': u'NeutronException', u'detail': u''}}
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack Traceback (most recent call last):
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/__init__.py", line 119, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack return req.get_response(self.application)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1320, in send
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack application, catch_exc_info=False)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/request.py", line 1284, in call_application
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack app_iter = application(self.environ, start_response)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack return resp(environ, start_response)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/keystoneclient/middleware/auth_token.py", line 539, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack return self.app(env, start_response)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack return resp(environ, start_response)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack return resp(environ, start_response)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/routes/middleware.py", line 131, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack response = self.app(environ, start_response)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 144, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack return resp(environ, start_response)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 130, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack resp = self.call_func(req, *args, **self.kwargs)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/local/lib/python2.7/dist-packages/webob/dec.py", line 195, in call_func
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack return self.func(req, *args, **kwargs)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line 917, in __call__
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack content_type, body, accept)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line 1002, in _process_stack
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack request, action_args)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/wsgi.py", line 890, in post_process_extensions
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack **action_args)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/compute/contrib/security_groups.py", line 583, in detail
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack self._extend_servers(req, list(resp_obj.obj['servers']))
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/api/openstack/compute/contrib/security_groups.py", line 527, in _extend_servers
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack servers[0]['id']))
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/nova/network/security_group/neutron_driver.py", line 321, in get_instance_security_groups
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack ports = neutron.list_ports(**params)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 108, in with_params
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack ret = self.function(instance, *args, **kwargs)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 306, in list_ports
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack **_params)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 1343, in list
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack for r in self._pagination(collection, path, **params):
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 1356, in _pagination
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack res = self.get(path, params=params)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 1329, in get
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack headers=headers, params=params)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 1314, in retry_request
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack headers=headers, params=params)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 1257, in do_request
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack self._handle_fault_response(status_code, replybody)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 1227, in _handle_fault_response
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack exception_handler_v20(status_code, des_error_body)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack File "/usr/lib/python2.7/dist-packages/neutronclient/v2_0/client.py", line 93, in exception_handler_v20
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack message=msg)
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack NeutronClientException: 500-{u'NeutronError': {u'message': u'An unknown exception occurred.', u'type': u'NeutronException', u'detail': u''}}
2015-03-18 09:24:35.911 6525 TRACE nova.api.openstack
2015-03-18 09:24:35.917 6525 INFO nova.api.openstack [req-afdbfca5-9c3c-4a82-b5ca-c5edc87c33a9 1cdd0ebf37e84849b23909dec43ba7b1 a929311426664f5899862c311a13e4a6] http://192.168.0.101:8774/v1.1/a929311426664f5899862c311a13e4a6/servers/detail?all_tenants=True returned with HTTP 500
2015-03-18 09:24:35.919 6525 INFO nova.osapi_compute.wsgi.server [req-afdbfca5-9c3c-4a82-b5ca-c5edc87c33a9 1cdd0ebf37e84849b23909dec43ba7b1 a929311426664f5899862c311a13e4a6] 192.168.0.101 "GET /v1.1/a929311426664f5899862c311a13e4a6/servers/detail?all_tenants=True HTTP/1.1" status: 500 len: 335 time: 0.0688860