The Kernel team experienced this problem yesterday, Feb 15th.
MAAS Version: MAAS version: 2.9.2 (9164-g.ac176b5c4-0ubuntu1~20.04.1)
Deb Pkg Installation
Here are the logs from the rackd controller at the time this occurred.
There were no obvious warnings/errors/tracebacks in the regiond.log
I restarted maas-rackd and maas-regiond services / updated the boot sources and the problem appears to have resolved itself. Yet, would be nice to know why this happened to begin with.
2022-02-15 15:31:44 provisioningserver.rpc.clusterservice: [critical] Failed to contact region. (While requesting RPC info at http://10.246.72.3:5240/MAAS).
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 460, in callback
self._startRunCallbacks(result)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1367, in _doUpdate
eventloops, maas_url = yield self._get_rpc_info(urls)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1631, in _get_rpc_info
raise config_exc
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1602, in _get_rpc_info
eventloops, maas_url = yield self._parallel_fetch_rpc_info(urls)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1576, in handle_responses
errors[0].raiseException()
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 467, in raiseException
raise self.value.with_traceback(self.tb)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1537, in _serial_fetch_rpc_info
raise last_exc
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1529, in _serial_fetch_rpc_info
response = yield self._fetch_rpc_info(url, orig_url)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1631, in _get_rpc_info
raise config_exc
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1602, in _get_rpc_info
eventloops, maas_url = yield self._parallel_fetch_rpc_info(urls)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1576, in handle_responses
errors[0].raiseException()
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 467, in raiseException
raise self.value.with_traceback(self.tb)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1537, in _serial_fetch_rpc_info
raise last_exc
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1529, in _serial_fetch_rpc_info
response = yield self._fetch_rpc_info(url, orig_url)
twisted.internet.error.ConnectingCancelledError: HostnameAddress(hostname=b'10.246.72.3', port=5240)
2022-02-15 15:33:39 provisioningserver.rpc.clusterservice: [critical] Failed to contact region. (While requesting RPC info at http://10.246.72.3:5240/MAAS).
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 460, in callback
self._startRunCallbacks(result)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
--- <exception caught here> ---
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1367, in _doUpdate
eventloops, maas_url = yield self._get_rpc_info(urls)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1631, in _get_rpc_info
raise config_exc
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1602, in _get_rpc_info
eventloops, maas_url = yield self._parallel_fetch_rpc_info(urls)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1576, in handle_responses
errors[0].raiseException()
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 467, in raiseException
raise self.value.with_traceback(self.tb)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1537, in _serial_fetch_rpc_info
raise last_exc
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1529, in _serial_fetch_rpc_info
response = yield self._fetch_rpc_info(url, orig_url)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1631, in _get_rpc_info
raise config_exc
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1602, in _get_rpc_info
eventloops, maas_url = yield self._parallel_fetch_rpc_info(urls)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1576, in handle_responses
errors[0].raiseException()
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 467, in raiseException
raise self.value.with_traceback(self.tb)
File "/usr/lib/python3/dist-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/usr/lib/python3/dist-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1537, in _serial_fetch_rpc_info
raise last_exc
File "/usr/lib/python3/dist-packages/provisioningserver/rpc/clusterservice.py", line 1529, in _serial_fetch_rpc_info
response = yield self._fetch_rpc_info(url, orig_url)
twisted.internet.error.ConnectingCancelledError: HostnameAddress(hostname=b'10.246.72.3', port=5240)
The Kernel team experienced this problem yesterday, Feb 15th. ac176b5c4- 0ubuntu1~ 20.04.1)
MAAS Version: MAAS version: 2.9.2 (9164-g.
Deb Pkg Installation
Here are the logs from the rackd controller at the time this occurred. errors/ tracebacks in the regiond.log
There were no obvious warnings/
I restarted maas-rackd and maas-regiond services / updated the boot sources and the problem appears to have resolved itself. Yet, would be nice to know why this happened to begin with.
2022-02-15 15:31:44 provisioningser ver.rpc. clusterservice: [critical] Failed to contact region. (While requesting RPC info at http:// 10.246. 72.3:5240/ MAAS). python3/ dist-packages/ twisted/ internet/ defer.py" , line 460, in callback _startRunCallba cks(result) python3/ dist-packages/ twisted/ internet/ defer.py" , line 568, in _startRunCallbacks _runCallbacks( ) python3/ dist-packages/ twisted/ internet/ defer.py" , line 654, in _runCallbacks current. result, *args, **kw) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1475, in gotResult lbacks( r, g, status) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1367, in _doUpdate rpc_info( urls) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1631, in _get_rpc_info python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1602, in _get_rpc_info fetch_rpc_ info(urls) python3/ dist-packages/ twisted/ internet/ defer.py" , line 654, in _runCallbacks current. result, *args, **kw) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1576, in handle_responses 0].raiseExcepti on() python3/ dist-packages/ twisted/ python/ failure. py", line 467, in raiseException with_traceback( self.tb) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1537, in _serial_ fetch_rpc_ info python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1529, in _serial_ fetch_rpc_ info rpc_info( url, orig_url) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1416, in _inlineCallbacks throwExceptionI ntoGenerator( g) python3/ dist-packages/ twisted/ python/ failure. py", line 491, in throwExceptionI ntoGenerator python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1631, in _get_rpc_info python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1602, in _get_rpc_info fetch_rpc_ info(urls) python3/ dist-packages/ twisted/ internet/ defer.py" , line 654, in _runCallbacks current. result, *args, **kw) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1576, in handle_responses 0].raiseExcepti on() python3/ dist-packages/ twisted/ python/ failure. py", line 467, in raiseException with_traceback( self.tb) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1416, in _inlineCallbacks throwExceptionI ntoGenerator( g) python3/ dist-packages/ twisted/ python/ failure. py", line 491, in throwExceptionI ntoGenerator python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1537, in _serial_ fetch_rpc_ info python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1529, in _serial_ fetch_rpc_ info rpc_info( url, orig_url) internet. error.Connectin gCancelledError : HostnameAddress (hostname= b'10.246. 72.3', port=5240)
Traceback (most recent call last):
File "/usr/lib/
self.
File "/usr/lib/
self.
File "/usr/lib/
current.result = callback(
File "/usr/lib/
_inlineCal
--- <exception caught here> ---
File "/usr/lib/
eventloops, maas_url = yield self._get_
File "/usr/lib/
raise config_exc
File "/usr/lib/
eventloops, maas_url = yield self._parallel_
File "/usr/lib/
current.result = callback(
File "/usr/lib/
errors[
File "/usr/lib/
raise self.value.
File "/usr/lib/
raise last_exc
File "/usr/lib/
response = yield self._fetch_
File "/usr/lib/
result = result.
File "/usr/lib/
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/
raise config_exc
File "/usr/lib/
eventloops, maas_url = yield self._parallel_
File "/usr/lib/
current.result = callback(
File "/usr/lib/
errors[
File "/usr/lib/
raise self.value.
File "/usr/lib/
result = result.
File "/usr/lib/
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/
raise last_exc
File "/usr/lib/
response = yield self._fetch_
twisted.
2022-02-15 15:33:39 provisioningser ver.rpc. clusterservice: [critical] Failed to contact region. (While requesting RPC info at http:// 10.246. 72.3:5240/ MAAS). python3/ dist-packages/ twisted/ internet/ defer.py" , line 460, in callback _startRunCallba cks(result) python3/ dist-packages/ twisted/ internet/ defer.py" , line 568, in _startRunCallbacks _runCallbacks( ) python3/ dist-packages/ twisted/ internet/ defer.py" , line 654, in _runCallbacks current. result, *args, **kw) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1475, in gotResult lbacks( r, g, status) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1367, in _doUpdate rpc_info( urls) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1631, in _get_rpc_info python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1602, in _get_rpc_info fetch_rpc_ info(urls) python3/ dist-packages/ twisted/ internet/ defer.py" , line 654, in _runCallbacks current. result, *args, **kw) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1576, in handle_responses 0].raiseExcepti on() python3/ dist-packages/ twisted/ python/ failure. py", line 467, in raiseException with_traceback( self.tb) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1537, in _serial_ fetch_rpc_ info python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1529, in _serial_ fetch_rpc_ info rpc_info( url, orig_url) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1416, in _inlineCallbacks throwExceptionI ntoGenerator( g) python3/ dist-packages/ twisted/ python/ failure. py", line 491, in throwExceptionI ntoGenerator python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1631, in _get_rpc_info python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1602, in _get_rpc_info fetch_rpc_ info(urls) python3/ dist-packages/ twisted/ internet/ defer.py" , line 654, in _runCallbacks current. result, *args, **kw) python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1576, in handle_responses 0].raiseExcepti on() python3/ dist-packages/ twisted/ python/ failure. py", line 467, in raiseException with_traceback( self.tb) python3/ dist-packages/ twisted/ internet/ defer.py" , line 1416, in _inlineCallbacks throwExceptionI ntoGenerator( g) python3/ dist-packages/ twisted/ python/ failure. py", line 491, in throwExceptionI ntoGenerator python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1537, in _serial_ fetch_rpc_ info python3/ dist-packages/ provisioningser ver/rpc/ clusterservice. py", line 1529, in _serial_ fetch_rpc_ info rpc_info( url, orig_url) internet. error.Connectin gCancelledError : HostnameAddress (hostname= b'10.246. 72.3', port=5240)
Traceback (most recent call last):
File "/usr/lib/
self.
File "/usr/lib/
self.
File "/usr/lib/
current.result = callback(
File "/usr/lib/
_inlineCal
--- <exception caught here> ---
File "/usr/lib/
eventloops, maas_url = yield self._get_
File "/usr/lib/
raise config_exc
File "/usr/lib/
eventloops, maas_url = yield self._parallel_
File "/usr/lib/
current.result = callback(
File "/usr/lib/
errors[
File "/usr/lib/
raise self.value.
File "/usr/lib/
raise last_exc
File "/usr/lib/
response = yield self._fetch_
File "/usr/lib/
result = result.
File "/usr/lib/
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/
raise config_exc
File "/usr/lib/
eventloops, maas_url = yield self._parallel_
File "/usr/lib/
current.result = callback(
File "/usr/lib/
errors[
File "/usr/lib/
raise self.value.
File "/usr/lib/
result = result.
File "/usr/lib/
return g.throw(self.type, self.value, self.tb)
File "/usr/lib/
raise last_exc
File "/usr/lib/
response = yield self._fetch_
twisted.