Comment 0 for bug 1365892

Revision history for this message
jazeltq (jazeltq-k) wrote :

we first find this bug in ceilometer.collector, after create 50k instances, the ceilometer.collectos uses 5.7 G memory. So we think there is an memory leak. After some research on this issue we thinks the problem is in function _safe_log.
def _safe_log(log_func, msg, msg_data):
    """Sanitizes the msg_data field before logging."""
    SANITIZE = ['_context_auth_token', 'auth_token', 'new_pass']

    def _fix_passwords(d):
        """Sanitizes the password fields in the dictionary."""
        for k in d.iterkeys():
            if k.lower().find('password') != -1:
                d[k] = '<SANITIZED>'
            elif k.lower() in SANITIZE:
                d[k] = '<SANITIZED>'
            elif isinstance(d[k], dict):
                _fix_passwords(d[k])
        return d

    return log_func(msg, _fix_passwords(copy.deepcopy(msg_data)))

we find that in gc.garbage, there are many things like those:
['_context_auth_token', 'auth_token', 'new_pass'],
 (<cell at 0x363e1a0: list object at 0x363cdd0>,
  <cell at 0x363e130: function object at 0x361a7d0>),
 <function _fix_passwords at 0x361a7d0>,
 <cell at 0x363e2f0: list object at 0x361b368>,
 <cell at 0x363e4e8: function object at 0x361a8c0>,
 ['_context_auth_token', 'auth_token', 'new_pass'],
 (<cell at 0x363e2f0: list object at 0x361b368>,
  <cell at 0x363e4e8: function object at 0x361a8c0>),
 <function _fix_passwords at 0x361a8c0>,
 <cell at 0x363e520: list object at 0x3640440>,
 <cell at 0x363e4b0: function object at 0x361a9b0>,
 ['_context_auth_token', 'auth_token', 'new_pass'],
 (<cell at 0x363e520: list object at 0x3640440>,
  <cell at 0x363e4b0: function object at 0x361a9b0>),
 <function _fix_passwords at 0x361a9b0>,
 <cell at 0x363e670: list object at 0x361b4d0>,
 <cell at 0x363e868: function object at 0x361aa28>,
 ['_context_auth_token', 'auth_token', 'new_pass'],
 (<cell at 0x363e670: list object at 0x361b4d0>,
  <cell at 0x363e868: function object at 0x361aa28>),

And we change the function a little, like this
def _safe_log(log_func, msg, msg_data):
    """Sanitizes the msg_data field before logging."""
    SANITIZE = ['_context_auth_token', 'auth_token', 'new_pass']

    def _fix_passwords(d):
        """Sanitizes the password fields in the dictionary."""
        for k in six.iterkeys(d):
            if k.lower().find('password') != -1:
                d[k] = '<SANITIZED>'
            elif k.lower() in SANITIZE:
                d[k] = '<SANITIZED>'
            elif isinstance(d[k], list):
                for e in d[k]:
                    if isinstance(e, dict):
                        _fix_passwords(e)
            elif isinstance(d[k], dict):
                _fix_passwords(d[k])
        return d
    res = log_func(msg, _fix_passwords(copy.deepcopy(msg_data)))
    _fix_passwords = None

After those, change, we find there are on more things find above.