we first find this bug in ceilometer.collector, after create 50k instances, the ceilometer.collectos uses 5.7 G memory. So we think there is an memory leak. After some research on this issue we thinks the problem is in function _safe_log.
def _safe_log(log_func, msg, msg_data):
"""Sanitizes the msg_data field before logging."""
SANITIZE = ['_context_auth_token', 'auth_token', 'new_pass']
def _fix_passwords(d): """Sanitizes the password fields in the dictionary."""
for k in d.iterkeys():
if k.lower().find('password') != -1: d[k] = '<SANITIZED>'
elif k.lower() in SANITIZE: d[k] = '<SANITIZED>'
elif isinstance(d[k], dict): _fix_passwords(d[k])
return d
we find that in gc.garbage, there are many things like those:
['_context_auth_token', 'auth_token', 'new_pass'],
(<cell at 0x363e1a0: list object at 0x363cdd0>,
<cell at 0x363e130: function object at 0x361a7d0>),
<function _fix_passwords at 0x361a7d0>,
<cell at 0x363e2f0: list object at 0x361b368>,
<cell at 0x363e4e8: function object at 0x361a8c0>,
['_context_auth_token', 'auth_token', 'new_pass'],
(<cell at 0x363e2f0: list object at 0x361b368>,
<cell at 0x363e4e8: function object at 0x361a8c0>),
<function _fix_passwords at 0x361a8c0>,
<cell at 0x363e520: list object at 0x3640440>,
<cell at 0x363e4b0: function object at 0x361a9b0>,
['_context_auth_token', 'auth_token', 'new_pass'],
(<cell at 0x363e520: list object at 0x3640440>,
<cell at 0x363e4b0: function object at 0x361a9b0>),
<function _fix_passwords at 0x361a9b0>,
<cell at 0x363e670: list object at 0x361b4d0>,
<cell at 0x363e868: function object at 0x361aa28>,
['_context_auth_token', 'auth_token', 'new_pass'],
(<cell at 0x363e670: list object at 0x361b4d0>,
<cell at 0x363e868: function object at 0x361aa28>),
And we change the function a little, like this
def _safe_log(log_func, msg, msg_data):
"""Sanitizes the msg_data field before logging."""
SANITIZE = ['_context_auth_token', 'auth_token', 'new_pass']
def _fix_passwords(d): """Sanitizes the password fields in the dictionary."""
for k in six.iterkeys(d):
if k.lower().find('password') != -1: d[k] = '<SANITIZED>'
elif k.lower() in SANITIZE: d[k] = '<SANITIZED>'
elif isinstance(d[k], list):
for e in d[k]: if isinstance(e, dict): _fix_passwords(e)
elif isinstance(d[k], dict): _fix_passwords(d[k])
return d
res = log_func(msg, _fix_passwords(copy.deepcopy(msg_data)))
_fix_passwords = None
After those, change, we find there are on more things find above.
we first find this bug in ceilometer. collector, after create 50k instances, the ceilometer. collectos uses 5.7 G memory. So we think there is an memory leak. After some research on this issue we thinks the problem is in function _safe_log. auth_token' , 'auth_token', 'new_pass']
def _safe_log(log_func, msg, msg_data):
"""Sanitizes the msg_data field before logging."""
SANITIZE = ['_context_
def _fix_passwords(d):
"""Sanitizes the password fields in the dictionary.""" ).find( 'password' ) != -1:
d[k] = '<SANITIZED>'
d[k] = '<SANITIZED>'
_fix_ passwords( d[k])
for k in d.iterkeys():
if k.lower(
elif k.lower() in SANITIZE:
elif isinstance(d[k], dict):
return d
return log_func(msg, _fix_passwords( copy.deepcopy( msg_data) ))
we find that in gc.garbage, there are many things like those: auth_token' , 'auth_token', 'new_pass'], auth_token' , 'auth_token', 'new_pass'], auth_token' , 'auth_token', 'new_pass'], auth_token' , 'auth_token', 'new_pass'],
['_context_
(<cell at 0x363e1a0: list object at 0x363cdd0>,
<cell at 0x363e130: function object at 0x361a7d0>),
<function _fix_passwords at 0x361a7d0>,
<cell at 0x363e2f0: list object at 0x361b368>,
<cell at 0x363e4e8: function object at 0x361a8c0>,
['_context_
(<cell at 0x363e2f0: list object at 0x361b368>,
<cell at 0x363e4e8: function object at 0x361a8c0>),
<function _fix_passwords at 0x361a8c0>,
<cell at 0x363e520: list object at 0x3640440>,
<cell at 0x363e4b0: function object at 0x361a9b0>,
['_context_
(<cell at 0x363e520: list object at 0x3640440>,
<cell at 0x363e4b0: function object at 0x361a9b0>),
<function _fix_passwords at 0x361a9b0>,
<cell at 0x363e670: list object at 0x361b4d0>,
<cell at 0x363e868: function object at 0x361aa28>,
['_context_
(<cell at 0x363e670: list object at 0x361b4d0>,
<cell at 0x363e868: function object at 0x361aa28>),
And we change the function a little, like this auth_token' , 'auth_token', 'new_pass']
def _safe_log(log_func, msg, msg_data):
"""Sanitizes the msg_data field before logging."""
SANITIZE = ['_context_
def _fix_passwords(d):
"""Sanitizes the password fields in the dictionary.""" ).find( 'password' ) != -1:
d[k] = '<SANITIZED>'
d[k] = '<SANITIZED>'
if isinstance(e, dict):
_fix_ passwords( e)
_fix_ passwords( d[k]) copy.deepcopy( msg_data) ))
for k in six.iterkeys(d):
if k.lower(
elif k.lower() in SANITIZE:
elif isinstance(d[k], list):
for e in d[k]:
elif isinstance(d[k], dict):
return d
res = log_func(msg, _fix_passwords(
_fix_passwords = None
After those, change, we find there are on more things find above.