| 2022-03-18 12:23:47 |
Arx Cruz |
description |
There are some jobs failing on tempest test, with two different errors however related, the first one, all tempest tests that relies on keystone endpoint are failing with the following error:
https://logserver.rdoproject.org/46/38646/16/check/periodic-tripleo-ci-centos-9-ovb-1ctlr_2comp-featureset020-master/b9ff5d0/logs/undercloud/var/log/tempest/stestr_results.html.gz
ft1.1: setUpClass (cinder_tempest_plugin.scenario.test_snapshots.SnapshotDataIntegrityTests)testtools.testresult.real._StringException: Traceback (most recent call last):
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 168, in setUpClass
raise value.with_traceback(trace)
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 153, in setUpClass
cls.setup_credentials()
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 372, in setup_credentials
manager = cls.get_client_manager(
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 701, in get_client_manager
cred_provider = cls._get_credentials_provider()
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 676, in _get_credentials_provider
cls._creds_provider = credentials.get_credentials_provider(
File "/usr/lib/python3.9/site-packages/tempest/common/credentials_factory.py", line 141, in get_credentials_provider
**get_dynamic_provider_params(identity_version))
File "/usr/lib/python3.9/site-packages/tempest/common/credentials_factory.py", line 72, in get_dynamic_provider_params
admin_creds = admin_creds or get_configured_admin_credentials(
File "/usr/lib/python3.9/site-packages/tempest/common/credentials_factory.py", line 257, in get_configured_admin_credentials
credentials = get_credentials(fill_in=fill_in,
File "/usr/lib/python3.9/site-packages/tempest/common/credentials_factory.py", line 300, in get_credentials
return auth.get_credentials(auth_url,
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 653, in get_credentials
creds = auth_provider.fill_credentials()
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 122, in fill_credentials
auth_data = self.get_auth()
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 148, in get_auth
self.set_auth()
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 157, in set_auth
self.cache = self._get_auth()
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 312, in _get_auth
token, auth_data = auth_func(**auth_params)
File "/usr/lib/python3.9/site-packages/tempest/lib/services/identity/v3/token_client.py", line 190, in get_token
body = self.auth(**kwargs)
File "/usr/lib/python3.9/site-packages/tempest/lib/services/identity/v3/token_client.py", line 143, in auth
resp, body = self.post(self.auth_url, body=body)
File "/usr/lib/python3.9/site-packages/tempest/lib/common/rest_client.py", line 299, in post
return self.request('POST', url, extra_headers, headers, body, chunked)
File "/usr/lib/python3.9/site-packages/tempest/lib/services/identity/v3/token_client.py", line 172, in request
raise exceptions.IdentityError(
tempest.lib.exceptions.IdentityError: Got identity error
Details: Unexpected status code 500
Checking the keystone logs I found this:
https://logserver.rdoproject.org/46/38646/16/check/periodic-tripleo-ci-centos-9-ovb-1ctlr_2comp-featureset020-master/b9ff5d0/logs/overcloud-controller-0/var/log/containers/keystone/keystone.log.txt.gz
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context Traceback (most recent call last):
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/server/flask/request_processing/middleware/auth_context.py", line 103, in _inner
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return method(self, request)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/server/flask/request_processing/middleware/auth_context.py", line 353, in process_request
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context resp = super(AuthContextMiddleware, self).process_request(request)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystonemiddleware/auth_token/__init__.py", line 409, in process_request
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context data, user_auth_ref = self._do_fetch_token(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystonemiddleware/auth_token/__init__.py", line 445, in _do_fetch_token
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context data = self.fetch_token(token, **kwargs)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/server/flask/request_processing/middleware/auth_context.py", line 247, in fetch_token
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self.token = self.token_provider_api.validate_token(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/common/manager.py", line 115, in wrapped
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context __ret_val = __f(*args, **kwargs)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/token/provider.py", line 145, in validate_token
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context token = self._validate_token(token_id)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "<decorator-gen-26>", line 2, in _validate_token
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/cache/region.py", line 1577, in get_or_create_for_user_func
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self.get_or_create(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/cache/region.py", line 1042, in get_or_create
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context with Lock(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/lock.py", line 185, in __enter__
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self._enter()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/lock.py", line 94, in _enter
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context generated = self._enter_create(value, createdtime)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/lock.py", line 178, in _enter_create
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self.creator()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/cache/region.py", line 995, in gen_value
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context created_value = creator(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/token/provider.py", line 179, in _validate_token
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context token.mint(token_id, issued_at)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/models/token_model.py", line 583, in mint
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self._validate_project_scope()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/models/token_model.py", line 532, in _validate_project_scope
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context if self.project_scoped and not self.roles:
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/models/token_model.py", line 458, in roles
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context roles = self._get_project_roles()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/models/token_model.py", line 417, in _get_project_roles
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context PROVIDERS.assignment_api.get_roles_for_user_and_project(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/common/manager.py", line 115, in wrapped
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context __ret_val = __f(*args, **kwargs)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "<decorator-gen-3>", line 2, in get_roles_for_user_and_project
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/cache/region.py", line 1577, in get_or_create_for_user_func
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self.get_or_create(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/cache/region.py", line 1042, in get_or_create
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context with Lock(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/lock.py", line 185, in __enter__
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self._enter()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/lock.py", line 94, in _enter
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context generated = self._enter_create(value, createdtime)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/lock.py", line 178, in _enter_create
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self.creator()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/dogpile/cache/region.py", line 995, in gen_value
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context created_value = creator(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/assignment/core.py", line 128, in get_roles_for_user_and_project
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context assignment_list = self.list_role_assignments(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/common/manager.py", line 115, in wrapped
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context __ret_val = __f(*args, **kwargs)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/assignment/core.py", line 995, in list_role_assignments
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context role_assignments = self._list_effective_role_assignments(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/assignment/core.py", line 841, in _list_effective_role_assignments
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context direct_refs = list_role_assignments_for_actor(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/assignment/core.py", line 776, in list_role_assignments_for_actor
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context non_inherited_refs = self.driver.list_role_assignments(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/keystone/assignment/backends/sql.py", line 255, in list_role_assignments
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return [denormalize_role(ref) for ref in query.all()]
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/orm/query.py", line 2759, in all
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self._iter().all()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/orm/query.py", line 2894, in _iter
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context result = self.session.execute(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/orm/session.py", line 1686, in execute
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context conn = self._connection_for_bind(bind, close_with_result=True)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/orm/session.py", line 1532, in _connection_for_bind
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self._transaction._connection_for_bind(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/orm/session.py", line 747, in _connection_for_bind
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context conn = bind.connect()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 3204, in connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self._connection_cls(self, close_with_result=close_with_result)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 120, in __init__
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self.dispatch.engine_connect(self, _branch_from is not None)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/event/attr.py", line 343, in __call__
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context fn(*args, **kw)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/oslo_db/sqlalchemy/engines.py", line 83, in _connect_ping_listener
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context connection.scalar(select(1))
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 1171, in scalar
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self.execute(object_, *multiparams, **params).scalar()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 1289, in execute
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/sql/elements.py", line 325, in _execute_on_connection
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return connection._execute_clauseelement(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 1481, in _execute_clauseelement
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context ret = self._execute_context(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 1708, in _execute_context
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self._handle_dbapi_exception(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 2024, in _handle_dbapi_exception
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context util.raise_(newraise, with_traceback=exc_info[2], from_=e)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context raise exception
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 1700, in _execute_context
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context conn = self._revalidate_connection()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 566, in _revalidate_connection
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self._dbapi_connection = self.engine.raw_connection(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 3283, in raw_connection
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self._wrap_pool_connect(self.pool.connect, _connection)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 3257, in _wrap_pool_connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context util.raise_(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context raise exception
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/base.py", line 3250, in _wrap_pool_connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return fn()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/pool/base.py", line 310, in connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return _ConnectionFairy._checkout(self)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/pool/base.py", line 868, in _checkout
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context fairy = _ConnectionRecord.checkout(pool)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/pool/base.py", line 481, in checkout
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context rec._checkin_failed(err, _fairy_was_created=False)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context compat.raise_(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context raise exception
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/pool/base.py", line 478, in checkout
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context dbapi_connection = rec.get_connection()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/pool/base.py", line 607, in get_connection
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self.__connect()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/pool/base.py", line 666, in __connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context pool.logger.debug("Error on connect(): %s", e)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context compat.raise_(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context raise exception
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/pool/base.py", line 661, in __connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self.dbapi_connection = connection = pool._invoke_creator(self)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/create.py", line 590, in connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return dialect.connect(*cargs, **cparams)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib64/python3.9/site-packages/sqlalchemy/engine/default.py", line 597, in connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return self.dbapi.connect(*cargs, **cparams)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/pymysql/__init__.py", line 94, in Connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context return Connection(*args, **kwargs)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/pymysql/connections.py", line 327, in __init__
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self.connect()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/pymysql/connections.py", line 587, in connect
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context self._get_server_information()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/pymysql/connections.py", line 969, in _get_server_information
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context packet = self._read_packet()
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/pymysql/connections.py", line 646, in _read_packet
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context packet_header = self._read_bytes(4)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context File "/usr/lib/python3.9/site-packages/pymysql/connections.py", line 698, in _read_bytes
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context raise err.OperationalError(
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context oslo_db.exception.DBConnectionError: (pymysql.err.OperationalError) (2013, 'Lost connection to MySQL server during query')
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context [SQL: SELECT 1]
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context (Background on this error at: https://sqlalche.me/e/14/e3q8)
2022-03-17 23:24:08.820 110 ERROR keystone.server.flask.request_processing.middleware.auth_context
This SQL error match with other tempest errors related to db connection:
ft36.8: tempest.scenario.test_network_v6.TestGettingAddress.test_slaac_from_os[compute,id-2c92df61-29f0-4eaa-bee3-7c65bef62a43,network,slow]testtools.testresult.real._StringException: pythonlogging:'': {{{
2022-03-17 19:24:08,805 201279 INFO [tempest.lib.common.rest_client] Request (TestGettingAddress:setUp): 500 POST http://10.0.0.5:8774/v2.1/os-keypairs 0.015s
2022-03-17 19:24:08,805 201279 DEBUG [tempest.lib.common.rest_client] Request - Headers: {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-OpenStack-Nova-API-Version': '2.1', 'X-Auth-Token': '<omitted>'}
Body: {"keypair": {"name": "tempest-TestGettingAddress-859199174", "public_key": "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBNyTQ4wV6HTMsdwTQzAUJ7DecjJ4+6JpCA3s0q7Cw+IJD7hImMHsmj1iOIa+OheCiiAq/dtRnWhFSTKDvX8zAyrpcUv3QtAzZ0TlBEtOKOh6hWJZ7Ni7+0PrCd+q1TFiiQ=="}}
Response - Headers: {'date': 'Thu, 17 Mar 2022 23:24:08 GMT', 'server': 'Apache', 'content-length': '209', 'openstack-api-version': 'compute 2.1', 'x-openstack-nova-api-version': '2.1', 'vary': 'OpenStack-API-Version,X-OpenStack-Nova-API-Version', 'x-openstack-request-id': 'req-e85b9e92-e0a6-42cf-a49d-178e2d8b91c5', 'x-compute-request-id': 'req-e85b9e92-e0a6-42cf-a49d-178e2d8b91c5', 'content-type': 'application/json; charset=UTF-8', 'connection': 'close', 'status': '500', 'content-location': 'http://10.0.0.5:8774/v2.1/os-keypairs'}
Body: b'{"computeFault": {"code": 500, "message": "Unexpected API Error. Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible.\\n<class \'oslo_db.exception.DBConnectionError\'>"}}'
}}}
Traceback (most recent call last):
File "/usr/lib/python3.9/site-packages/tempest/scenario/test_network_v6.py", line 68, in setUp
self.keypair = self.create_keypair()
File "/usr/lib/python3.9/site-packages/tempest/scenario/manager.py", line 179, in create_keypair
body = client.create_keypair(**kwargs)
File "/usr/lib/python3.9/site-packages/tempest/lib/services/compute/keypairs_client.py", line 91, in create_keypair
resp, body = self.post("os-keypairs", body=post_body)
File "/usr/lib/python3.9/site-packages/tempest/lib/common/rest_client.py", line 299, in post
return self.request('POST', url, extra_headers, headers, body, chunked)
File "/usr/lib/python3.9/site-packages/tempest/lib/services/compute/base_compute_client.py", line 47, in request
resp, resp_body = super(BaseComputeClient, self).request(
File "/usr/lib/python3.9/site-packages/tempest/lib/common/rest_client.py", line 703, in request
self._error_checker(resp, resp_body)
File "/usr/lib/python3.9/site-packages/tempest/lib/common/rest_client.py", line 879, in _error_checker
raise exceptions.ServerFault(resp_body, resp=resp,
tempest.lib.exceptions.ServerFault: Got server fault
Details: Unexpected API Error. Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible.
<class 'oslo_db.exception.DBConnectionError'>
But i did not see anything on mysql logs.
Checking the nova-compute service, i notice this:
/python3.9/site-packages/oslo_service/periodic_task.py:210
2022-03-17 23:23:10.814 2 ERROR oslo.messaging._drivers.impl_rabbit [req-a656b24b-ddcf-450c-84a2-27583e592954 - - - - -] [c11d0399-f03d-4fcf-9865-09584ccc3dff] AMQP server on overcloud-controller-0.internalapi.localdomain:5672 is unreachable: [Errno 104] Connection reset by peer. Trying again in 1 seconds.: ConnectionResetError: [Errno 104] Connection reset by peer
2022-03-17 23:23:11.004 2 ERROR oslo.messaging._drivers.impl_rabbit [-] [0055e02b-a50c-4688-a649-1e63e3b56e6d] AMQP server on overcloud-controller-0.internalapi.localdomain:5672 is unreachable: <RecoverableConnectionError: unknown error>. Trying again in 1 seconds.: amqp.exceptions.RecoverableConnectionError: <RecoverableConnectionError: unknown error>
2022-03-17 23:23:11.006 2 ERROR oslo.messaging._drivers.impl_rabbit [-] [949b296b-d736-4e0a-99a8-5177270052d1] AMQP server on overcloud-controller-0.internalapi.localdomain:5672 is unreachable: <RecoverableConnectionError: unknown error>. Trying again in 1 seconds.: amqp.exceptions.RecoverableConnectionError: <RecoverableConnectionError: unknown error>
2022-03-17 23:23:11.818 2 ERROR oslo.messaging._drivers.impl_rabbit [req-a656b24b-ddcf-450c-84a2-27583e592954 - - - - -] [c11d0399-f03d-4fcf-9865-09584ccc3dff] AMQP server on overcloud-controller-0.internalapi.localdomain:5672 is unreachable: [Errno 111] ECONNREFUSED. Trying again in 2 seconds.: ConnectionRefusedError: [Errno 111] ECONNREFUSED
In both nova-compute-1 and nova-compute-0 nodes |
There are some jobs failing on tempest test, with two different errors however related, the first one, all tempest tests that relies on keystone endpoint are failing with the following error:
https://logserver.rdoproject.org/46/38646/16/check/periodic-tripleo-ci-centos-9-ovb-1ctlr_2comp-featureset020-master/b9ff5d0/logs/undercloud/var/log/tempest/stestr_results.html.gz
ft1.1: setUpClass (cinder_tempest_plugin.scenario.test_snapshots.SnapshotDataIntegrityTests)testtools.testresult.real._StringException: Traceback (most recent call last):
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 168, in setUpClass
raise value.with_traceback(trace)
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 153, in setUpClass
cls.setup_credentials()
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 372, in setup_credentials
manager = cls.get_client_manager(
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 701, in get_client_manager
cred_provider = cls._get_credentials_provider()
File "/usr/lib/python3.9/site-packages/tempest/test.py", line 676, in _get_credentials_provider
cls._creds_provider = credentials.get_credentials_provider(
File "/usr/lib/python3.9/site-packages/tempest/common/credentials_factory.py", line 141, in get_credentials_provider
**get_dynamic_provider_params(identity_version))
File "/usr/lib/python3.9/site-packages/tempest/common/credentials_factory.py", line 72, in get_dynamic_provider_params
admin_creds = admin_creds or get_configured_admin_credentials(
File "/usr/lib/python3.9/site-packages/tempest/common/credentials_factory.py", line 257, in get_configured_admin_credentials
credentials = get_credentials(fill_in=fill_in,
File "/usr/lib/python3.9/site-packages/tempest/common/credentials_factory.py", line 300, in get_credentials
return auth.get_credentials(auth_url,
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 653, in get_credentials
creds = auth_provider.fill_credentials()
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 122, in fill_credentials
auth_data = self.get_auth()
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 148, in get_auth
self.set_auth()
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 157, in set_auth
self.cache = self._get_auth()
File "/usr/lib/python3.9/site-packages/tempest/lib/auth.py", line 312, in _get_auth
token, auth_data = auth_func(**auth_params)
File "/usr/lib/python3.9/site-packages/tempest/lib/services/identity/v3/token_client.py", line 190, in get_token
body = self.auth(**kwargs)
File "/usr/lib/python3.9/site-packages/tempest/lib/services/identity/v3/token_client.py", line 143, in auth
resp, body = self.post(self.auth_url, body=body)
File "/usr/lib/python3.9/site-packages/tempest/lib/common/rest_client.py", line 299, in post
return self.request('POST', url, extra_headers, headers, body, chunked)
File "/usr/lib/python3.9/site-packages/tempest/lib/services/identity/v3/token_client.py", line 172, in request
raise exceptions.IdentityError(
tempest.lib.exceptions.IdentityError: Got identity error
Details: Unexpected status code 500 |
|
