Ah thanks for pointing out where the folders should get created. So this failure happens with the 'container-puppet-haproxy' container which happens on step1:
2019-03-18 17:47:57,152 p=486 u=mistral | TASK [Run container-puppet tasks (generate config) during step 1] **************
2019-03-18 17:47:57,152 p=486 u=mistral | Monday 18 March 2019 17:47:57 +0000 (0:00:00.929) 0:08:23.294 **********
2019-03-18 17:50:35,097 p=486 u=mistral | ok: [compute-0] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false}
2019-03-18 17:52:11,404 p=486 u=mistral | ok: [controller-0] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false}
2019-03-18 17:52:27,130 p=486 u=mistral | ok: [controller-1] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false}
2019-03-18 17:52:30,301 p=486 u=mistral | ok: [controller-2] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false}
2019-03-18 17:52:30,352 p=486 u=mistral | TASK [Debug output for task: Run container-puppet tasks (generate config) during step 1] ***
2019-03-18 17:52:30,352 p=486 u=mistral | Monday 18 March 2019 17:52:30 +0000 (0:04:33.199) 0:12:56.493 **********
2019-03-18 17:52:30,476 p=486 u=mistral | fatal: [controller-0]: FAILED! => {
"failed_when_result": true,
"outputs.stdout_lines | default([]) | union(outputs.stderr_lines | default([]))": [
"2019-03-18 17:47:57,755 INFO: 27003 -- Running container-puppet",
"2019-03-18 17:47:57,756 INFO: 27003 -- Service compilation completed.",
"2019-03-18 17:47:57,756 INFO: 27003 -- Starting multiprocess configuration steps. Using 6 processes.",
"2019-03-18 17:47:57,769 INFO: 27004 -- Starting configuration of aodh using image brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-aodh-api:latest",
"2019-03-18 17:47:57,770 INFO: 27006 -- Starting configuration of glance_api using image brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-glance-api:latest",
"2019-03-18 17:47:57,769 INFO: 27005 -- Starting configuration of cinder using image brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-cinder-api:latest",
"2019-03-18 17:47:57,771 INFO: 27007 -- Starting configuration of haproxy using image brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-haproxy:latest",
"2019-03-18 17:47:57,772 INFO: 27008 -- Starting configuration of heat_api_cfn using image brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-heat-api-cfn:latest",
"2019-03-18 17:47:57,772 INFO: 27009 -- Starting configuration of horizon using image brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-horizon:latest",
"2019-03-18 17:47:58,108 INFO: 27008 -- Removing container: container-puppet-heat_api_cfn",
"2019-03-18 17:47:58,169 INFO: 27005 -- Removing container: container-puppet-cinder",
"2019-03-18 17:47:58,231 INFO: 27009 -- Removing container: container-puppet-horizon",
"2019-03-18 17:47:58,355 INFO: 27006 -- Removing container: container-puppet-glance_api",
"2019-03-18 17:47:58,469 INFO: 27004 -- Removing container: container-puppet-aodh",
"2019-03-18 17:47:58,877 INFO: 27007 -- Removing container: container-puppet-haproxy",
"2019-03-18 17:47:59,039 INFO: 27008 -- Pulling image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-heat-api-cfn:latest",
"2019-03-18 17:47:59,096 INFO: 27004 -- Pulling image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-aodh-api:latest",
"2019-03-18 17:47:59,532 INFO: 27009 -- Pulling image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-horizon:latest",
"2019-03-18 17:47:59,588 INFO: 27005 -- Pulling image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-cinder-api:latest",
"2019-03-18 17:47:59,836 INFO: 27007 -- Image already exists: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-haproxy:latest",
"2019-03-18 17:47:59,966 INFO: 27006 -- Pulling image: brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-glance-api:latest",
"2019-03-18 17:48:03,153 WARNING: 27007 -- ['/usr/bin/podman', 'run', '--user', 'root', '--name', 'container-puppet-haproxy', '--env', 'PUPPET_TAGS=file,file_line,concat,augeas,cron,haproxy_config', '--env', 'NAME=haproxy', '--env', 'HOSTNAME=controller-0', '--env', 'NO_ARCHIVE=', '--env', 'STEP=6', '--env', 'NET_HOST=true', '--log-driver', 'json-file', '--volume', '/etc/localtime:/etc/localtime:ro', '--volume', '/tmp/tmpvtrdil17:/etc/config.pp:ro', '--volume', '/etc/puppet/:/tmp/puppet-etc/:ro', '--volume', '/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro', '--volume', '/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro', '--volume', '/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro', '--volume', '/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro', '--volume', '/var/lib/config-data:/var/lib/config-data/:rw', '--volume', '/dev/log:/dev/log:rw', '--log-opt', 'path=/var/log/containers/stdouts/container-puppet-haproxy.log', '--security-opt', 'label=disable', '--volume', '/usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro', '--volume', '/etc/pki/tls/private/haproxy:/etc/pki/tls/private/haproxy:ro', '--volume', '/etc/pki/tls/certs/haproxy:/etc/pki/tls/certs/haproxy:ro', '--volume', '/etc/pki/tls/private/overcloud_endpoint.pem:/etc/pki/tls/private/overcloud_endpoint.pem:ro', '--entrypoint', '/var/lib/container-puppet/container-puppet.sh', '--net', 'host', '--volume', '/etc/hosts:/etc/hosts:ro', '--volume', '/var/lib/container-puppet/container-puppet.sh:/var/lib/container-puppet/container-puppet.sh:ro', 'brew-pulp-docker01.web.prod.ext.phx2.redhat.com:8888/rhosp15/openstack-haproxy:latest'] run failed after error checking path \"/etc/pki/tls/private/haproxy\": stat /etc/pki/tls/private/haproxy: no such file or directory",
" attempt(s): 1",
"2019-03-18 17:48:03,153 WARNING: 27007 -- Retrying running container: haproxy",
"2019-03-18 17:48:06,286 WARNING: 27007 -- ['/usr/bin/podman', 'start', '-a', 'container-puppet-haproxy'] run failed after unable to find container container-puppet-haproxy: no container with name or ID container-puppet-haproxy found: no such container",
" attempt(s): 2",
"2019-03-18 17:48:06,286 WARNING: 27007 -- Retrying running container: haproxy",
"2019-03-18 17:48:09,430 WARNING: 27007 -- ['/usr/bin/podman', 'start', '-a', 'container-puppet-haproxy'] run failed after unable to find container container-puppet-haproxy: no container with name or ID container-puppet-haproxy found: no such container",
" attempt(s): 3",
"2019-03-18 17:48:09,430 WARNING: 27007 -- Retrying running container: haproxy",
"2019-03-18 17:48:09,430 ERROR: 27007 -- Failed running container for haproxy",
In my host puppet manifest I do have certmonger:
[root@controller-0 tripleo-config]# grep certmonger /var/lib/tripleo-config/puppet_step_config.pp
include ::tripleo::profile::base::certmonger_user
Ah thanks for pointing out where the folders should get created. So this failure happens with the 'container- puppet- haproxy' container which happens on step1: when_result" : true, stdout_ lines | default([]) | union(outputs. stderr_ lines | default([]))": [ docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- aodh-api: latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- glance- api:latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- cinder- api:latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- haproxy: latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- heat-api- cfn:latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- horizon: latest" , puppet- heat_api_ cfn", puppet- cinder" , puppet- horizon" , puppet- glance_ api", puppet- aodh", puppet- haproxy" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- heat-api- cfn:latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- aodh-api: latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- horizon: latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- cinder- api:latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- haproxy: latest" , docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- glance- api:latest" , puppet- haproxy' , '--env', 'PUPPET_ TAGS=file, file_line, concat, augeas, cron,haproxy_ config' , '--env', 'NAME=haproxy', '--env', 'HOSTNAME= controller- 0', '--env', 'NO_ARCHIVE=', '--env', 'STEP=6', '--env', 'NET_HOST=true', '--log-driver', 'json-file', '--volume', '/etc/localtime :/etc/localtime :ro', '--volume', '/tmp/tmpvtrdil 17:/etc/ config. pp:ro', '--volume', '/etc/puppet/ :/tmp/puppet- etc/:ro' , '--volume', '/etc/pki/ ca-trust/ extracted: /etc/pki/ ca-trust/ extracted: ro', '--volume', '/etc/pki/ tls/certs/ ca-bundle. crt:/etc/ pki/tls/ certs/ca- bundle. crt:ro' , '--volume', '/etc/pki/ tls/certs/ ca-bundle. trust.crt: /etc/pki/ tls/certs/ ca-bundle. trust.crt: ro', '--volume', '/etc/pki/ tls/cert. pem:/etc/ pki/tls/ cert.pem: ro', '--volume', '/var/lib/ config- data:/var/ lib/config- data/:rw' , '--volume', '/dev/log: /dev/log: rw', '--log-opt', 'path=/ var/log/ containers/ stdouts/ container- puppet- haproxy. log', '--security-opt', 'label=disable', '--volume', '/usr/share/ openstack- puppet/ modules/ :/usr/share/ openstack- puppet/ modules/ :ro', '--volume', '/etc/pki/ tls/private/ haproxy: /etc/pki/ tls/private/ haproxy: ro', '--volume', '/etc/pki/ tls/certs/ haproxy: /etc/pki/ tls/certs/ haproxy: ro', '--volume', '/etc/pki/ tls/private/ overcloud_ endpoint. pem:/etc/ pki/tls/ private/ overcloud_ endpoint. pem:ro' , '--entrypoint', '/var/lib/ container- puppet/ container- puppet. sh', '--net', 'host', '--volume', '/etc/hosts: /etc/hosts: ro', '--volume', '/var/lib/ container- puppet/ container- puppet. sh:/var/ lib/container- puppet/ container- puppet. sh:ro', 'brew-pulp- docker01. web.prod. ext.phx2. redhat. com:8888/ rhosp15/ openstack- haproxy: latest' ] run failed after error checking path \"/etc/ pki/tls/ private/ haproxy\ ": stat /etc/pki/ tls/private/ haproxy: no such file or directory", puppet- haproxy' ] run failed after unable to find container container- puppet- haproxy: no container with name or ID container- puppet- haproxy found: no such container", puppet- haproxy' ] run failed after unable to find container container- puppet- haproxy: no container with name or ID container- puppet- haproxy found: no such container",
2019-03-18 17:47:57,152 p=486 u=mistral | TASK [Run container-puppet tasks (generate config) during step 1] **************
2019-03-18 17:47:57,152 p=486 u=mistral | Monday 18 March 2019 17:47:57 +0000 (0:00:00.929) 0:08:23.294 **********
2019-03-18 17:50:35,097 p=486 u=mistral | ok: [compute-0] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false}
2019-03-18 17:52:11,404 p=486 u=mistral | ok: [controller-0] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false}
2019-03-18 17:52:27,130 p=486 u=mistral | ok: [controller-1] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false}
2019-03-18 17:52:30,301 p=486 u=mistral | ok: [controller-2] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": false}
2019-03-18 17:52:30,352 p=486 u=mistral | TASK [Debug output for task: Run container-puppet tasks (generate config) during step 1] ***
2019-03-18 17:52:30,352 p=486 u=mistral | Monday 18 March 2019 17:52:30 +0000 (0:04:33.199) 0:12:56.493 **********
2019-03-18 17:52:30,476 p=486 u=mistral | fatal: [controller-0]: FAILED! => {
"failed_
"outputs.
"2019-03-18 17:47:57,755 INFO: 27003 -- Running container-puppet",
"2019-03-18 17:47:57,756 INFO: 27003 -- Service compilation completed.",
"2019-03-18 17:47:57,756 INFO: 27003 -- Starting multiprocess configuration steps. Using 6 processes.",
"2019-03-18 17:47:57,769 INFO: 27004 -- Starting configuration of aodh using image brew-pulp-
"2019-03-18 17:47:57,770 INFO: 27006 -- Starting configuration of glance_api using image brew-pulp-
"2019-03-18 17:47:57,769 INFO: 27005 -- Starting configuration of cinder using image brew-pulp-
"2019-03-18 17:47:57,771 INFO: 27007 -- Starting configuration of haproxy using image brew-pulp-
"2019-03-18 17:47:57,772 INFO: 27008 -- Starting configuration of heat_api_cfn using image brew-pulp-
"2019-03-18 17:47:57,772 INFO: 27009 -- Starting configuration of horizon using image brew-pulp-
"2019-03-18 17:47:58,108 INFO: 27008 -- Removing container: container-
"2019-03-18 17:47:58,169 INFO: 27005 -- Removing container: container-
"2019-03-18 17:47:58,231 INFO: 27009 -- Removing container: container-
"2019-03-18 17:47:58,355 INFO: 27006 -- Removing container: container-
"2019-03-18 17:47:58,469 INFO: 27004 -- Removing container: container-
"2019-03-18 17:47:58,877 INFO: 27007 -- Removing container: container-
"2019-03-18 17:47:59,039 INFO: 27008 -- Pulling image: brew-pulp-
"2019-03-18 17:47:59,096 INFO: 27004 -- Pulling image: brew-pulp-
"2019-03-18 17:47:59,532 INFO: 27009 -- Pulling image: brew-pulp-
"2019-03-18 17:47:59,588 INFO: 27005 -- Pulling image: brew-pulp-
"2019-03-18 17:47:59,836 INFO: 27007 -- Image already exists: brew-pulp-
"2019-03-18 17:47:59,966 INFO: 27006 -- Pulling image: brew-pulp-
"2019-03-18 17:48:03,153 WARNING: 27007 -- ['/usr/bin/podman', 'run', '--user', 'root', '--name', 'container-
" attempt(s): 1",
"2019-03-18 17:48:03,153 WARNING: 27007 -- Retrying running container: haproxy",
"2019-03-18 17:48:06,286 WARNING: 27007 -- ['/usr/bin/podman', 'start', '-a', 'container-
" attempt(s): 2",
"2019-03-18 17:48:06,286 WARNING: 27007 -- Retrying running container: haproxy",
"2019-03-18 17:48:09,430 WARNING: 27007 -- ['/usr/bin/podman', 'start', '-a', 'container-
" attempt(s): 3",
"2019-03-18 17:48:09,430 WARNING: 27007 -- Retrying running container: haproxy",
"2019-03-18 17:48:09,430 ERROR: 27007 -- Failed running container for haproxy",
In my host puppet manifest I do have certmonger: tripleo- config/ puppet_ step_config. pp :profile: :base:: certmonger_ user
[root@controller-0 tripleo-config]# grep certmonger /var/lib/
include ::tripleo: