Linking k8s pod's VM to node is failing

R4.0.1.0 Build 41 ubuntu 16.04.2 container setup

Python 2.7.12: /usr/bin/python
Mon Sep 4 10:46:36 2017

A problem occurred in a Python script. Here is the sequence of
function calls leading up to the error, in the order they occurred.

 /usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_kubernetes.py in vnc_process(self=<kube_manager.vnc.vnc_kubernetes.VncKubernetes object>)
  374 uid = metadata.get('uid')
  375 if kind == 'Pod':
  376 self.pod_mgr.process(event)
  377 elif kind == 'Service':
  378 self.service_mgr.process(event)
self = <kube_manager.vnc.vnc_kubernetes.VncKubernetes object>
self.pod_mgr = <kube_manager.vnc.vnc_pod.VncPod object>
self.pod_mgr.process = <bound method VncPod.process of <kube_manager.vnc.vnc_pod.VncPod object>>
event = {u'object': {u'apiVersion': u'v1', u'kind': u'Pod', u'metadata': {u'annotations': {u'kubernetes.io/created-by': u'{"kind":"SerializedReference","apiVersion":"v1...ersion":"extensions","resourceVersion":"576"}}\n', u'scheduler.alpha.kubernetes.io/critical-pod': u''}, u'creationTimestamp': u'2017-09-01T20:33:20Z', u'generateName': u'kube-dns-1378942307-', u'labels': {u'k8s-app': u'kube-dns', u'pod-template-hash': u'1378942307'}, u'name': u'kube-dns-1378942307-dngbw', u'namespace': u'kube-system', u'ownerReferences': [{u'apiVersion': u'extensions/v1beta1', u'blockOwnerDeletion': True, u'controller': True, u'kind': u'ReplicaSet', u'name': u'kube-dns-1378942307', u'uid': u'cacd793d-8f54-11e7-bcd3-002590c30af2'}], u'resourceVersion': u'1960', u'selfLink': u'/api/v1/namespaces/kube-system/pods/kube-dns-1378942307-dngbw', u'uid': u'caf3ffcf-8f54-11e7-bcd3-002590c30af2'}, u'spec': {u'containers': [{u'args': [u'--domain=cluster.local.', u'--dns-port=10053', u'--config-dir=/kube-dns-config', u'--v=2'], u'env': [{...}], u'image': u'gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.4', u'imagePullPolicy': u'IfNotPresent', u'livenessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'name': u'kubedns', u'ports': [{...}, {...}, {...}], u'readinessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'resources': {u'limits': {...}, u'requests': {...}}, u'terminationMessagePath': u'/dev/termination-log', ...}, {u'args': [u'-v=2', u'-logtostderr', u'-configDir=/etc/k8s/dns/dnsmasq-nanny', u'-restartDnsmasq=true', u'--', u'-k', u'--cache-size=1000', u'--log-facility=-', u'--server=/cluster.local/127.0.0.1#10053', u'--server=/in-addr.arpa/127.0.0.1#10053', u'--server=/ip6.arpa/127.0.0.1#10053'], u'image': u'gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.4', u'imagePullPolicy': u'IfNotPresent', u'livenessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'name': u'dnsmasq', u'ports': [{...}, {...}], u'readinessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'resources': {u'requests': {...}}, u'terminationMessagePath': u'/dev/termination-log', u'terminationMessagePolicy': u'File', ...}, {u'args': [u'--v=2', u'--logtostderr', u'--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A', u'--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A'], u'image': u'gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.4', u'imagePullPolicy': u'IfNotPresent', u'livenessProbe': {u'exec': {...}, u'failureThreshold': 3, u'periodSeconds': 10, u'successThreshold': 1, u'timeoutSeconds': 1}, u'name': u'sidecar', u'ports': [{...}], u'resources': {u'requests': {...}}, u'terminationMessagePath': u'/dev/termination-log', u'terminationMessagePolicy': u'File', u'volumeMounts': [{...}]}], u'dnsPolicy': u'Default', u'nodeName': u'nodek2', u'restartPolicy': u'Always', u'schedulerName': u'default-scheduler', u'securityContext': {}, u'serviceAccount': u'kube-dns', u'serviceAccountName': u'kube-dns', u'terminationGracePeriodSeconds': 30, u'tolerations': [{u'effect': u'NoExecute', u'key': u'node.alpha.kubernetes.io/notReady', u'operator': u'Exists', u'tolerationSeconds': 300}, {u'effect': u'NoExecute', u'key': u'node.alpha.kubernetes.io/unreachable', u'operator': u'Exists', u'tolerationSeconds': 300}], ...}, u'status': {u'conditions': [{u'lastProbeTime': None, u'lastTransitionTime': u'2017-09-01T20:48:47Z', u'status': u'True', u'type': u'Initialized'}, {u'lastProbeTime': None, u'lastTransitionTime': u'2017-09-01T20:48:47Z', u'message': u'containers with unready status: [kubedns dnsmasq sidecar]', u'reason': u'ContainersNotReady', u'status': u'False', u'type': u'Ready'}, {u'lastProbeTime': None, u'lastTransitionTime': u'2017-09-01T20:48:47Z', u'status': u'True', u'type': u'PodScheduled'}], u'containerStatuses': [{u'image': u'gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.4', u'imageID': u'', u'lastState': {}, u'name': u'dnsmasq', u'ready': False, u'restartCount': 0, u'state': {u'waiting': {...}}}, {u'image': u'gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.4', u'imageID': u'', u'lastState': {}, u'name': u'kubedns', u'ready': False, u'restartCount': 0, u'state': {u'waiting': {...}}}, {u'image': u'gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.4', u'imageID': u'', u'lastState': {}, u'name': u'sidecar', u'ready': False, u'restartCount': 0, u'state': {u'waiting': {...}}}], u'hostIP': u'10.204.216.222', u'phase': u'Pending', u'qosClass': u'Burstable', u'startTime': u'2017-09-01T20:48:47Z'}}, u'type': u'ADDED'}
  502 if event['type'] == 'ADDED':
  503 vm = self.vnc_pod_add(pod_id, pod_name, pod_namespace,
  504 pod_node, labels, vm_vmi)
  505 if vm:
  506 self._network_policy_mgr.update_pod_np(pod_namespace, pod_id, labels)
pod_node = u'nodek2'
labels = {u'k8s-app': u'kube-dns', u'pod-template-hash': u'1378942307'}
vm_vmi = None

 /usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py in vnc_pod_add(self=<kube_manager.vnc.vnc_pod.VncPod object>, pod_id=u'caf3ffcf-8f54-11e7-bcd3-002590c30af2', pod_name=u'kube-dns-1378942307-dngbw', pod_namespace=u'kube-system', pod_node=u'nodek2', labels={u'k8s-app': u'kube-dns', u'pod-template-hash': u'1378942307'}, vm_vmi=None)
  322 vm.pod_namespace = pod_namespace
  323 if not vm.virtual_router:
  324 self._link_vm_to_node(vm, pod_node)
  325 self._set_label_to_pod_cache(labels, vm)
  326 return vm
self = <kube_manager.vnc.vnc_pod.VncPod object>
self._link_vm_to_node = <bound method VncPod._link_vm_to_node of <kube_manager.vnc.vnc_pod.VncPod object>>
vm = <kube_manager.vnc.config_db.VirtualMachineKM object>
pod_node = u'nodek2'

 /usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py in _link_vm_to_node(self=<kube_manager.vnc.vnc_pod.VncPod object>, vm_obj=<kube_manager.vnc.config_db.VirtualMachineKM object>, pod_node=u'nodek2')
  298 self._logger.debug("%s - Vrouter %s Not Found for Pod %s"
  299 %(self._name, vrouter_fq_name, vm_obj.uuid))
  300 string_buf = StringIO()
  301 cgitb_hook(file=string_buf, format="text")
  302 err_msg = string_buf.getvalue()
string_buf undefined
StringIO undefined
<type 'exceptions.NameError'>: global name 'StringIO' is not defined
    __class__ = <type 'exceptions.NameError'>
    __delattr__ = <method-wrapper '__delattr__' of exceptions.NameError object>
    __dict__ = {}
    __doc__ = 'Name not found globally.'
    __format__ = <built-in method __format__ of exceptions.NameError object>
    __getattribute__ = <method-wrapper '__getattribute__' of exceptions.NameError object>
    __getitem__ = <method-wrapper '__getitem__' of exceptions.NameError object>
    __getslice__ = <method-wrapper '__getslice__' of exceptions.NameError object>
    __hash__ = <method-wrapper '__hash__' of exceptions.NameError object>
    __init__ = <method-wrapper '__init__' of exceptions.NameError object>
    __new__ = <built-in method __new__ of type object>
    __reduce__ = <built-in method __reduce__ of exceptions.NameError object>
    __reduce_ex__ = <built-in method __reduce_ex__ of exceptions.NameError object>
    __repr__ = <method-wrapper '__repr__' of exceptions.NameError object>
    __setattr__ = <method-wrapper '__setattr__' of exceptions.NameError object>
    __setstate__ = <built-in method __setstate__ of exceptions.NameError object>
    __sizeof__ = <built-in method __sizeof__ of exceptions.NameError object>
    __str__ = <method-wrapper '__str__' of exceptions.NameError object>
    __subclasshook__ = <built-in method __subclasshook__ of type object>
    __unicode__ = <built-in method __unicode__ of exceptions.NameError object>
    args = ("global name 'StringIO' is not defined",)
    message = "global name 'StringIO' is not defined"

The above is a description of an error in a Python program. Here is
the original traceback:

Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_kubernetes.py", line 376, in vnc_process
    self.pod_mgr.process(event)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 504, in process
    pod_node, labels, vm_vmi)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 324, in vnc_pod_add
    self._link_vm_to_node(vm, pod_node)
  File "/usr/lib/python2.7/dist-packages/kube_manager/vnc/vnc_pod.py", line 300, in _link_vm_to_node
    string_buf = StringIO()
NameError: global name 'StringIO' is not defined