[ 4.0.3.0-113] Puppet-agent failed to start due to authentication issue

Steps:

1) Bring up the setup with the json file copied below using the cmd
/opt/contrail/contrail_server_manager/provision_containers.sh -ni -j /root/regress_provision_with_compute.json

The setup comes up as expected.

2) Now, destroy all the servers (VMs in this case)
fab -f setup_virtmanager.py delete_vm:nodel11-vm1

Don't delete the servers from the server manager which is running on a different node (nodei5)

root@nodei5:~# server-manager display all
{
    "cluster": [
        {
            "id": "nodel11_cluster"
        }
    ],
    "dhcp_host": [],
    "dhcp_subnet": [
        {
            "default_lease_time": 21600,
            "dhcp_range": null,
            "dns_server_list": "['__$IPADDRESS__']",
            "max_lease_time": 43200,
            "search_domains_list": "['__$DOMAIN__']",
            "subnet_address": "__$SUBNETADDRESS__",
            "subnet_domain": "__$DOMAIN__",
            "subnet_gateway": "__$IPADDRESS__",
            "subnet_mask": "__$SUBNETMASK__"
        }
    ],
    "image": [
        {
            "id": "regress_image"
        }
    ],
    "server": [
        {
            "id": "nodel11-vm1",
            "ip_address": "10.204.217.31",
            "mac_address": "52:53:58:01:00:01"
        },
        {
            "id": "nodel11-vm2",
            "ip_address": "10.204.217.32",
            "mac_address": "52:53:58:01:00:02"
        },
        {
            "id": "nodel11-vm3",
            "ip_address": "10.204.217.33",
            "mac_address": "52:53:58:01:00:03"
        },
        {
            "id": "nodel11-vm4",
            "ip_address": "10.204.217.34",
            "mac_address": "52:53:58:01:00:04"
        },
        {
            "id": "nodel11-vm5",
            "ip_address": "10.204.217.36",
            "mac_address": "52:53:58:01:00:05"
        }
    ],
    "tag": {}
}

3) Bring up all the VMs again

4) Trigger the provisioning again using the following cmd
/opt/contrail/contrail_server_manager/provision_containers.sh -ni -j /root/regress_provision_with_compute.json

It was seen puppet-agent could not get started due to authentication issue.
Logs:

"2017-11-07 12:50:14,201-DEBUG-server_mgr_main.py:6744-prepare_provision(): 10.204.217.34 added in the provision server list"
"2017-11-07 12:50:14,218-INFO-server_mgr_main.py:306-merge_dict(): adding metadata_proxy_secret:None"
"2017-11-07 12:50:14,219-INFO-server_mgr_main.py:306-merge_dict(): adding management_ip:None"
"2017-11-07 12:50:14,220-INFO-server_mgr_main.py:306-merge_dict(): adding ctrl_data_ip:None"
"2017-11-07 12:50:14,221-INFO-server_mgr_main.py:306-merge_dict(): adding ip:None"
"2017-11-07 12:50:14,222-INFO-server_mgr_main.py:306-merge_dict(): adding admin_password:None"
"2017-11-07 12:50:14,223-INFO-server_mgr_main.py:306-merge_dict(): adding analytics_ip:None"
"2017-11-07 12:50:14,224-INFO-server_mgr_main.py:306-merge_dict(): adding controller_ip:None"
"2017-11-07 12:50:14,225-INFO-server_mgr_main.py:306-merge_dict(): adding external_rabbitmq_servers:None"
"2017-11-07 12:50:14,226-INFO-server_mgr_main.py:306-merge_dict(): adding config_ip:None"
"2017-11-07 12:50:14,228-INFO-server_mgr_main.py:306-merge_dict(): adding password:None"
"2017-11-07 12:50:14,229-INFO-server_mgr_main.py:306-merge_dict(): adding user:None"
"2017-11-07 12:50:14,233-DEBUG-server_mgr_main.py:6744-prepare_provision(): 10.204.217.36 added in the provision server list"
"2017-11-07 12:50:14,234-DEBUG-server_mgr_main.py:6818-provision_server(): provision queued. Number of servers provisioned is 5:"
"2017-11-07 12:50:14,340-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm1.englab.juniper.net"
"2017-11-07 12:50:14,573-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm2.englab.juniper.net"
"2017-11-07 12:50:14,794-DEBUG-server_mgr_puppet.py:576-update_node_map_file(): Add/Modify map file with env_name regress_image for server nodel11-vm3.englab.juniper.net"
"2017-11-07 12:50:46,435-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm1"
"2017-11-07 12:50:46,437-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm2"
"2017-11-07 12:50:46,439-DEBUG-server_mgr_mon_base_plugin.py:780-gevent_puppet_agent_action(): Going to start the puppet agent on the server nodel11-vm3"
"2017-11-07 12:50:46,601-INFO-server_mgr_ssh_client.py:60-connect(): CONNECT FAILED: Host => 10.204.217.32, option => key, ERROR => Authentication failed."
"2017-11-07 12:50:46,603-ERROR-server_mgr_mon_base_plugin.py:825-gevent_puppet_agent_action(): Gevent SSH Connect Exception for server id: nodel11-vm2 Error : Authentication failed.

json file:

server-manager is on nodei5

{
    "cluster" : [
        {
            "id" : "nodel11_cluster",
            "parameters" : {
                "provision": {
                    "contrail_4": {
                        "enable_lbaas": true,
                        "global_config": {
                            "xmpp_auth_enable": true,
                            "xmpp_dns_auth_enable": true
                        },
                        "kernel_upgrade": false,
                        "ha": {
                            "contrail_external_vip": "10.204.217.34"
                        }
                    },
                    "openstack":{
                        "ha": {
                            "external_vip": "10.204.217.89",
                            "internal_vip": "10.204.217.89",
                            "external_virtual_router_id": "201"
                        },
                        "multi_tenancy": true,
                        "openstack_manage_amqp": true,
                        "keystone": {
                            "admin_password": "contrail123",
                            "admin_token": "admin"
                        }
                    }
                }
            }
        }
    ],
    "server": [
        {
              "cluster_id": "nodel11_cluster",
              "domain": "englab.juniper.net",
              "id": "nodel11-vm1",
              "password": "c0ntrail123",
              "network": {
                  "management_interface": "eth0",
                  "interfaces": [
                      {
                          "default_gateway": "10.204.217.254",
                          "dhcp": true,
                          "ip_address": "10.204.217.31/24",
                          "mac_address": "52:53:58:01:00:01",
                          "name": "eth0"
                      }
                  ]
              },
              "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"]
        },
        {
              "cluster_id": "nodel11_cluster",
              "domain": "englab.juniper.net",
              "id": "nodel11-vm2",
              "password": "c0ntrail123",
              "network": {
                  "management_interface": "eth0",
                  "interfaces": [
                      {
                          "default_gateway": "10.204.217.254",
                          "dhcp": true,
                          "ip_address": "10.204.217.32/24",
                          "mac_address": "52:53:58:01:00:02",
                          "name": "eth0"
                      }
                  ]
              },
              "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"]
        },
        {
              "cluster_id": "nodel11_cluster",
              "domain": "englab.juniper.net",
              "id": "nodel11-vm3",
              "password": "c0ntrail123",
              "network": {
                  "management_interface": "eth0",
                  "interfaces": [
                      {
                          "default_gateway": "10.204.217.254",
                          "dhcp": true,
                          "ip_address": "10.204.217.33/24",
                          "mac_address": "52:53:58:01:00:03",
                          "name": "eth0"
                      }
                  ]
              },
              "roles": ["openstack", "contrail-controller", "contrail-analytics", "contrail-analyticsdb"]
        },
        {
              "cluster_id": "nodel11_cluster",
              "domain": "englab.juniper.net",
              "id": "nodel11-vm4",
              "password": "c0ntrail123",
              "network": {
                  "management_interface": "eth0",
                  "interfaces": [
                      {
                          "default_gateway": "10.204.217.254",
                          "dhcp": true,
                          "ip_address": "10.204.217.34/24",
                          "mac_address": "52:53:58:01:00:04",
                          "name": "eth0"
                      }
                  ]
              },
              "roles": ["contrail-lb"]
        },
        {
              "cluster_id": "nodel11_cluster",
              "domain": "englab.juniper.net",
              "id": "nodel11-vm5",
              "password": "c0ntrail123",
              "network": {

                  "management_interface": "eth0",
                  "interfaces": [
                      {
                          "default_gateway": "10.204.217.254",
                          "dhcp": true,
                          "ip_address": "10.204.217.36/24",
                          "mac_address": "52:53:58:01:00:05",
                          "name": "eth0"
                      }
                  ]
              },
              "roles": ["contrail-compute"]
        }
    ],

    "image": [
        {
            "category": "package",
            "id": "regress_image",
            "type": "contrail-ubuntu-package",
            "path": "/root/contrail-cloud-docker_4.0.3.0-113-mitaka_trusty.tgz",
            "version": "R4.1"
        }
    ]
}

Logs and Json file copied at :

/cs-shared/bugs/<bug-id>/