we first find this bug in ceilometer.collector, after create 50k instances, the ceilometer.collectos uses 5.7 G memory. So we think there is an memory leak. After some research on this issue we think the problem is in function _safe_log.
def _safe_log(log_func, msg, msg_data):
"""Sanitizes the msg_data field before logging."""
SANITIZE = ['_context_auth_token', 'auth_token', 'new_pass']
def _fix_passwords(d):
"""Sanitizes the password fields in the dictionary."""
for k in d.iterkeys():
if k.lower().find('password') != -1:
d[k] = '<SANITIZED>'
elif k.lower() in SANITIZE:
d[k] = '<SANITIZED>'
elif isinstance(d[k], dict):
_fix_passwords(d[k])
return d
return log_func(msg, _fix_passwords(copy.deepcopy(msg_data)))
we find that in gc.garbage, there are many things like those:
['_context_auth_token', 'auth_token', 'new_pass'],
(<cell at 0x363e1a0: list object at 0x363cdd0>,
<cell at 0x363e130: function object at 0x361a7d0>),
<function _fix_passwords at 0x361a7d0>,
<cell at 0x363e2f0: list object at 0x361b368>,
<cell at 0x363e4e8: function object at 0x361a8c0>,
['_context_auth_token', 'auth_token', 'new_pass'],
(<cell at 0x363e2f0: list object at 0x361b368>,
<cell at 0x363e4e8: function object at 0x361a8c0>),
<function _fix_passwords at 0x361a8c0>,
<cell at 0x363e520: list object at 0x3640440>,
<cell at 0x363e4b0: function object at 0x361a9b0>,
['_context_auth_token', 'auth_token', 'new_pass'],
(<cell at 0x363e520: list object at 0x3640440>,
<cell at 0x363e4b0: function object at 0x361a9b0>),
<function _fix_passwords at 0x361a9b0>,
<cell at 0x363e670: list object at 0x361b4d0>,
<cell at 0x363e868: function object at 0x361aa28>,
['_context_auth_token', 'auth_token', 'new_pass'],
(<cell at 0x363e670: list object at 0x361b4d0>,
<cell at 0x363e868: function object at 0x361aa28>),
And we change the function a little, like this
def _safe_log(log_func, msg, msg_data):
"""Sanitizes the msg_data field before logging."""
SANITIZE = ['_context_auth_token', 'auth_token', 'new_pass']
def _fix_passwords(d):
"""Sanitizes the password fields in the dictionary."""
for k in six.iterkeys(d):
if k.lower().find('password') != -1:
d[k] = '<SANITIZED>'
elif k.lower() in SANITIZE:
d[k] = '<SANITIZED>'
elif isinstance(d[k], list):
for e in d[k]:
if isinstance(e, dict):
_fix_passwords(e)
elif isinstance(d[k], dict):
_fix_passwords(d[k])
return d
res = log_func(msg, _fix_passwords(copy.deepcopy(msg_data)))
_fix_passwords = None
return res
After those, change, we find there are no more things find above.
Fix proposed to branch: master
Review: https:/