Activity log for bug #1730632

Date Who What changed Old value New value Message
2017-11-07 10:20:02 Ankit Jain bug added bug
2017-11-07 10:26:06 Ankit Jain nominated for series juniperopenstack/r4.0
2017-11-07 10:26:06 Ankit Jain bug task added juniperopenstack/r4.0
2017-11-07 10:26:21 Ankit Jain juniperopenstack/r4.0: assignee Abhay Joshi (abhayj)
2017-11-07 10:53:46 Ankit Jain description Steps: 1) Bring up the setup with the json file copied below using the cmd /opt/contrail/contrail_server_manager/provision_containers.sh -ni -j /root/regress_provision_with_compute.json The setup comes up as expected. 2) Now, destroy all the servers (VMs in this case). Don't delete the cluster from the server manager which is running on a different node (nodei5) root@nodei5:~# server-manager display all { "cluster": [ { "id": "nodel11_cluster" } ], "dhcp_host": [], "dhcp_subnet": [ { "default_lease_time": 21600, "dhcp_range": null, "dns_server_list": "['__$IPADDRESS__']", "max_lease_time": 43200, "search_domains_list": "['__$DOMAIN__']", "subnet_address": "__$SUBNETADDRESS__", "subnet_domain": "__$DOMAIN__", "subnet_gateway": "__$IPADDRESS__", "subnet_mask": "__$SUBNETMASK__" } ], "image": [ { "id": "regress_image" } ], "server": [ { "id": "nodel11-vm1", "ip_address": "10.204.217.31", "mac_address": "52:53:58:01:00:01" }, { "id": "nodel11-vm2", "ip_address": "10.204.217.32", "mac_address": "52:53:58:01:00:02" }, { "id": "nodel11-vm3", "ip_address": "10.204.217.33", "mac_address": "52:53:58:01:00:03" }, { "id": "nodel11-vm4", "ip_address": "10.204.217.34", "mac_address": "52:53:58:01:00:04" }, { "id": "nodel11-vm5", "ip_address": "10.204.217.36", "mac_address": "52:53:58:01:00:05" } ], "tag": {} } 3) Trigger the provisioning again using the following cmd /opt/contrail/contrail_server_manager/provision_containers.sh -ni -j /root/regress_provision_with_compute.json It was seen puppet-agent could not get started due to authentication issue. Logs: "2017-11-07 12:50:14,201-DEBUG-server_mgr_main.py:6744-prepare_provision(): 10.204.217.34 added in the provision server list" "2017-11-07 12:50:14,218-INFO-server_mgr_main.py:306-merge_dict(): adding metadata_proxy_secret:None" "2017-11-07 12:50:14,219-INFO-server_mgr_main.py:306-merge_dict(): adding management_ip:None" "2017-11-07 12:50:14,220-INFO-server_mgr_main.py:306-merge_dict(): adding ctrl_data_ip:None" "2017-11-07 12:50:14,221-INFO-server_mgr_main.py:306-merge_dict(): adding ip:None" "2017-11-07 12:50:14,222-INFO-server_mgr_main.py:306-merge_dict(): adding admin_password:None" "2017-11-07 12:50:14,223-INFO-server_mgr_main.py:306-merge_dict(): adding analytics_ip:None" "2017-11-07 12:50:14,224-INFO-server_mgr_main.py:306-merge_dict(): adding controller_ip:None" "2017-11-07 12:50:14,225-INFO-server_mgr_main.py:306-merge_dict(): adding external_rabbitmq_servers:None" "2017-11-07 12:50:14,226-INFO-server_mgr_main.py:306-merge_dict(): adding config_ip:None" "2017-11-07 12:50:14,228-INFO-server_mgr_main.py:306-merge_dict(): adding password:None" "2017-11-07 12:50:14,229-INFO-server_mgr_main.py:306-merge_dict(): adding user:None" "2017-11-07 12:50:14,233-DEBUG-server_mgr_main.py:6744-prepare_provision(): 10.204.217.36 added in the provision server list" "2017-11-07 12:50:14,234-DEBUG-server_mgr_main.py:6818-provision_server(): provision queued. Number of servers provisioned is 5:" "2017-11-07 12:50:14,340-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm1.englab.juniper.net" "2017-11-07 12:50:14,573-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm2.englab.juniper.net" "2017-11-07 12:50:14,794-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm3.englab.juniper.net" "2017-11-07 12:50:46,435-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm1" "2017-11-07 12:50:46,437-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm2" "2017-11-07 12:50:46,439-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm3" "2017-11-07 12:50:46,601-INFO-server_mgr_ssh_client.py:60-connect(): CONNECT FAILED: Host => 10.204.217.32, option => key, ERROR => Authentication failed." "2017-11-07 12:50:46,603-ERROR-server_mgr_mon_base_plugin.py:825-gevent_puppet_agent_action(): Gevent SSH Connect Exception for server id: nodel11-vm2 Error : Authentication failed. json file: server-manager is on nodei5 { "cluster" : [ { "id" : "nodel11_cluster", "parameters" : { "provision": { "contrail_4": { "enable_lbaas": true, "global_config": { "xmpp_auth_enable": true, "xmpp_dns_auth_enable": true }, "kernel_upgrade": false, "ha": { "contrail_external_vip": "10.204.217.34" } }, "openstack":{ "ha": { "external_vip": "10.204.217.89", "internal_vip": "10.204.217.89", "external_virtual_router_id": "201" }, "multi_tenancy": true, "openstack_manage_amqp": true, "keystone": { "admin_password": "contrail123", "admin_token": "admin" } } } } } ], "server": [ { "cluster_id": "nodel11_cluster", "domain": "englab.juniper.net", "id": "nodel11-vm1", "password": "c0ntrail123", "network": { "management_interface": "eth0", "interfaces": [ { "default_gateway": "10.204.217.254", "dhcp": true, "ip_address": "10.204.217.31/24", "mac_address": "52:53:58:01:00:01", "name": "eth0" } ] }, "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"] }, { "cluster_id": "nodel11_cluster", "domain": "englab.juniper.net", "id": "nodel11-vm2", "password": "c0ntrail123", "network": { "management_interface": "eth0", "interfaces": [ { "default_gateway": "10.204.217.254", "dhcp": true, "ip_address": "10.204.217.32/24", "mac_address": "52:53:58:01:00:02", "name": "eth0" } ] }, "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"] }, { "cluster_id": "nodel11_cluster", "domain": "englab.juniper.net", "id": "nodel11-vm3", "password": "c0ntrail123", "network": { "management_interface": "eth0", "interfaces": [ { "default_gateway": "10.204.217.254", "dhcp": true, "ip_address": "10.204.217.33/24", "mac_address": "52:53:58:01:00:03", "name": "eth0" } ] }, "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"] }, { "cluster_id": "nodel11_cluster", "domain": "englab.juniper.net", "id": "nodel11-vm4", "password": "c0ntrail123", "network": { "management_interface": "eth0", "interfaces": [ { "default_gateway": "10.204.217.254", "dhcp": true, "ip_address": "10.204.217.34/24", "mac_address": "52:53:58:01:00:04", "name": "eth0" } ] }, "roles": ["contrail-lb"] }, { "cluster_id": "nodel11_cluster", "domain": "englab.juniper.net", "id": "nodel11-vm5", "password": "c0ntrail123", "network": { "management_interface": "eth0", "interfaces": [ { "default_gateway": "10.204.217.254", "dhcp": true, "ip_address": "10.204.217.36/24", "mac_address": "52:53:58:01:00:05", "name": "eth0" } ] }, "roles": ["contrail-compute"] } ], "image": [ { "category": "package", "id": "regress_image", "type": "contrail-ubuntu-package", "path": "/root/contrail-cloud-docker_4.0.3.0-113-mitaka_trusty.tgz", "version": "R4.1" } ] } Logs and Json file copied at : /cs-shared/bugs/<bug-id>/ Steps: 1) Bring up the setup with the json file copied below using the cmd /opt/contrail/contrail_server_manager/provision_containers.sh -ni -j /root/regress_provision_with_compute.json The setup comes up as expected. 2) Now, destroy all the servers (VMs in this case) fab -f setup_virtmanager.py delete_vm:nodel11-vm1 Don't delete the servers from the server manager which is running on a different node (nodei5) root@nodei5:~# server-manager display all {     "cluster": [         {             "id": "nodel11_cluster"         }     ],     "dhcp_host": [],     "dhcp_subnet": [         {             "default_lease_time": 21600,             "dhcp_range": null,             "dns_server_list": "['__$IPADDRESS__']",             "max_lease_time": 43200,             "search_domains_list": "['__$DOMAIN__']",             "subnet_address": "__$SUBNETADDRESS__",             "subnet_domain": "__$DOMAIN__",             "subnet_gateway": "__$IPADDRESS__",             "subnet_mask": "__$SUBNETMASK__"         }     ],     "image": [         {             "id": "regress_image"         }     ],     "server": [         {             "id": "nodel11-vm1",             "ip_address": "10.204.217.31",             "mac_address": "52:53:58:01:00:01"         },         {             "id": "nodel11-vm2",             "ip_address": "10.204.217.32",             "mac_address": "52:53:58:01:00:02"         },         {             "id": "nodel11-vm3",             "ip_address": "10.204.217.33",             "mac_address": "52:53:58:01:00:03"         },         {             "id": "nodel11-vm4",             "ip_address": "10.204.217.34",             "mac_address": "52:53:58:01:00:04"         },         {             "id": "nodel11-vm5",             "ip_address": "10.204.217.36",             "mac_address": "52:53:58:01:00:05"         }     ],     "tag": {} } 3) Bring up all the VMs again 4) Trigger the provisioning again using the following cmd /opt/contrail/contrail_server_manager/provision_containers.sh -ni -j /root/regress_provision_with_compute.json It was seen puppet-agent could not get started due to authentication issue. Logs: "2017-11-07 12:50:14,201-DEBUG-server_mgr_main.py:6744-prepare_provision(): 10.204.217.34 added in the provision server list" "2017-11-07 12:50:14,218-INFO-server_mgr_main.py:306-merge_dict(): adding metadata_proxy_secret:None" "2017-11-07 12:50:14,219-INFO-server_mgr_main.py:306-merge_dict(): adding management_ip:None" "2017-11-07 12:50:14,220-INFO-server_mgr_main.py:306-merge_dict(): adding ctrl_data_ip:None" "2017-11-07 12:50:14,221-INFO-server_mgr_main.py:306-merge_dict(): adding ip:None" "2017-11-07 12:50:14,222-INFO-server_mgr_main.py:306-merge_dict(): adding admin_password:None" "2017-11-07 12:50:14,223-INFO-server_mgr_main.py:306-merge_dict(): adding analytics_ip:None" "2017-11-07 12:50:14,224-INFO-server_mgr_main.py:306-merge_dict(): adding controller_ip:None" "2017-11-07 12:50:14,225-INFO-server_mgr_main.py:306-merge_dict(): adding external_rabbitmq_servers:None" "2017-11-07 12:50:14,226-INFO-server_mgr_main.py:306-merge_dict(): adding config_ip:None" "2017-11-07 12:50:14,228-INFO-server_mgr_main.py:306-merge_dict(): adding password:None" "2017-11-07 12:50:14,229-INFO-server_mgr_main.py:306-merge_dict(): adding user:None" "2017-11-07 12:50:14,233-DEBUG-server_mgr_main.py:6744-prepare_provision(): 10.204.217.36 added in the provision server list" "2017-11-07 12:50:14,234-DEBUG-server_mgr_main.py:6818-provision_server(): provision queued. Number of servers provisioned is 5:" "2017-11-07 12:50:14,340-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm1.englab.juniper.net" "2017-11-07 12:50:14,573-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm2.englab.juniper.net" "2017-11-07 12:50:14,794-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm3.englab.juniper.net" "2017-11-07 12:50:46,435-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm1" "2017-11-07 12:50:46,437-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm2" "2017-11-07 12:50:46,439-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm3" "2017-11-07 12:50:46,601-INFO-server_mgr_ssh_client.py:60-connect(): CONNECT FAILED: Host => 10.204.217.32, option => key, ERROR => Authentication failed." "2017-11-07 12:50:46,603-ERROR-server_mgr_mon_base_plugin.py:825-gevent_puppet_agent_action(): Gevent SSH Connect Exception for server id: nodel11-vm2 Error : Authentication failed. json file: server-manager is on nodei5 {     "cluster" : [         {             "id" : "nodel11_cluster",             "parameters" : {                 "provision": {                     "contrail_4": {                         "enable_lbaas": true,                         "global_config": {                             "xmpp_auth_enable": true,                             "xmpp_dns_auth_enable": true                         },                         "kernel_upgrade": false,                         "ha": {                             "contrail_external_vip": "10.204.217.34"                         }                     },                     "openstack":{                         "ha": {                             "external_vip": "10.204.217.89",                             "internal_vip": "10.204.217.89",                             "external_virtual_router_id": "201"                         },                         "multi_tenancy": true,                         "openstack_manage_amqp": true,                         "keystone": {                             "admin_password": "contrail123",                             "admin_token": "admin"                         }                     }                 }             }         }     ],     "server": [         {               "cluster_id": "nodel11_cluster",               "domain": "englab.juniper.net",               "id": "nodel11-vm1",               "password": "c0ntrail123",               "network": {                   "management_interface": "eth0",                   "interfaces": [                       {                           "default_gateway": "10.204.217.254",                           "dhcp": true,                           "ip_address": "10.204.217.31/24",                           "mac_address": "52:53:58:01:00:01",                           "name": "eth0"                       }                   ]               },               "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"]         },         {               "cluster_id": "nodel11_cluster",               "domain": "englab.juniper.net",               "id": "nodel11-vm2",               "password": "c0ntrail123",               "network": {                   "management_interface": "eth0",                   "interfaces": [                       {                           "default_gateway": "10.204.217.254",                           "dhcp": true,                           "ip_address": "10.204.217.32/24",                           "mac_address": "52:53:58:01:00:02",                           "name": "eth0"                       }                   ]               },               "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"]         },         {               "cluster_id": "nodel11_cluster",               "domain": "englab.juniper.net",               "id": "nodel11-vm3",               "password": "c0ntrail123",               "network": {                   "management_interface": "eth0",                   "interfaces": [                       {                           "default_gateway": "10.204.217.254",                           "dhcp": true,                           "ip_address": "10.204.217.33/24",                           "mac_address": "52:53:58:01:00:03",                           "name": "eth0"                       }                   ]               },               "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"]         },         {               "cluster_id": "nodel11_cluster",               "domain": "englab.juniper.net",               "id": "nodel11-vm4",               "password": "c0ntrail123",               "network": {                   "management_interface": "eth0",                   "interfaces": [                       {                           "default_gateway": "10.204.217.254",                           "dhcp": true,                           "ip_address": "10.204.217.34/24",                           "mac_address": "52:53:58:01:00:04",                           "name": "eth0"                       }                   ]               },               "roles": ["contrail-lb"]         },         {               "cluster_id": "nodel11_cluster",               "domain": "englab.juniper.net",               "id": "nodel11-vm5",               "password": "c0ntrail123",               "network": {                   "management_interface": "eth0",                   "interfaces": [                       {                           "default_gateway": "10.204.217.254",                           "dhcp": true,                           "ip_address": "10.204.217.36/24",                           "mac_address": "52:53:58:01:00:05",                           "name": "eth0"                       }                   ]               },               "roles": ["contrail-compute"]         }     ],     "image": [         {             "category": "package",             "id": "regress_image",             "type": "contrail-ubuntu-package",             "path": "/root/contrail-cloud-docker_4.0.3.0-113-mitaka_trusty.tgz",             "version": "R4.1"         }     ] } Logs and Json file copied at : /cs-shared/bugs/<bug-id>/