api: '1.0'
astute_sha: 055b2d82fe8499b27c7047295e2e36a7a2c5d430
auth_required: true
build_id: 2015-04-16_21-30-10
build_number: '317'
feature_groups:
- mirantis
fuellib_sha: db5f39e96e7ab9f79691202755e547bf8242661f
fuelmain_sha: 0de2d2039e76839d339f977df45b111bef7200d6
nailgun_sha: 52d92c86e68602fb5dd2f3b8870a773d20a387ef
openstack_version: 2014.2-6.1
ostf_sha: b0991dbad159f53d335efa5d31cb94016ad5312e
production: docker
python-fuelclient_sha: 279ffc358e40dbdc162cfe76898cbd0874529f1f
release: '6.1'
Successfully deployed the following configuration:
Baremetal,Ubuntu,IBP,HA, Neutron-vlan,Ceph-all,Nova-debug,Nova-quotas, 6.1-317
Controllers:3 Computes:47
We have found that there is no archived logs on controller nodes:
root@node-1:~# ls -lah /var/log/*-all.log*
-rw-r----- 1 syslog syslog 279M May 14 13:59 /var/log/cinder-all.log
-rw-r----- 1 syslog syslog 356M May 14 13:59 /var/log/glance-all.log
-rw-r----- 1 syslog syslog 157M May 14 13:59 /var/log/heat-all.log
-rw-r----- 1 syslog syslog 1.8G May 14 13:59 /var/log/keystone-all.log
-rw-r----- 1 syslog syslog 3.3G May 14 13:59 /var/log/neutron-all.log
-rw-r----- 1 syslog syslog 718M May 14 13:59 /var/log/nova-all.log
For example cinder-all log hasn't been rotated:
root@node-1:~# head -1 /var/log/cinder-all.log
<159>May 12 22:41:11 node-1 cinder-manage backend <module 'cinder.db.sqlalchemy.migration' from '/usr/lib/python2.7/dist-packages/cinder/db/sqlalchemy/migration.pyc'>
root@node-1:~# tail -1 /var/log/cinder-all.log
<158>May 14 14:00:28 node-1 cinder-api 192.168.0.47 - - [14/May/2015 14:00:28] "OPTIONS / HTTP/1.0" 200 422 0.003460
root@node-1:~# ls -la /var/log/cinder/
total 416764
drwxr-x--- 2 cinder adm 4096 May 12 22:41 .
drwxrwxr-x 25 root syslog 4096 May 14 12:30 ..
-rw-r--r-- 1 cinder cinder 300518822 May 14 14:00 cinder-api.log
-rw-r--r-- 1 cinder cinder 709235 May 14 14:00 cinder-backup.log
-rw-r--r-- 1 cinder cinder 23014 May 12 22:41 cinder-manage.log
-rw-r--r-- 1 cinder cinder 29438943 May 14 14:00 cinder-scheduler.log
-rw-r--r-- 1 cinder cinder 96045179 May 14 14:00 cinder-volume.log
from messages log file:
<13>May 14 13:50:01 node-1 logrotate: ALERT exited abnormally with [1] (0 was expected)
We can see this alert every 10 minutes
Diagnostic snapshot is here http://mos-scale-share.mirantis.com/fuel-snapshot-2015-05-14_08-14-00.tar.xz
It's a really old iso. There were a few changes to logrotate configuration in the meantime:
https:/
https:/
https:/
If you think that the issue is still present, please post here the logs from logrotate runs, because I don't think those are included in diagnostic snapshot.
1. logrotate -v /etc/logrotate.conf
2. Edit our logrotate shell wrapper to include 'set -x' before anything gets executed and attach the output of run.