External OpenStack Cloud Controller Manager implementation (#5491)

* External OpenStack Cloud Controller Manager implementation

* Adding controller image tag

* Minor fixes

* Restructuring the external cloud controller to work with KubeADM

roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+# The external cloud controller will need credentials to access
+# openstack apis. Per default these values will be
+# read from the environment.
+external_openstack_auth_url: "{{ lookup('env','OS_AUTH_URL')  }}"
+external_openstack_username: "{{ lookup('env','OS_USERNAME')  }}"
+external_openstack_password: "{{ lookup('env','OS_PASSWORD')  }}"
+external_openstack_region: "{{ lookup('env','OS_REGION_NAME')  }}"
+external_openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}"
+external_openstack_tenant_name: "{{ lookup('env','OS_TENANT_NAME') }}"
+external_openstack_domain_name: "{{ lookup('env','OS_USER_DOMAIN_NAME') }}"
+external_openstack_domain_id: "{{ lookup('env','OS_USER_DOMAIN_ID') }}"
+external_openstack_cacert: "{{ lookup('env','OS_CACERT') }}"
+
+external_openstack_cloud_controller_image_tag: "latest"

roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- include_tasks: openstack-credential-check.yml
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Write cacert file
+  copy:
+    src: "{{ external_openstack_cacert }}"
+    dest: "{{ kube_config_dir }}/external-openstack-cacert.pem"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  when:
+    - inventory_hostname in groups['k8s-cluster']
+    - external_openstack_cacert is defined
+    - external_openstack_cacert | length > 0
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Write External OpenStack cloud-config
+  template:
+    src: "external-openstack-cloud-config.j2"
+    dest: "{{ kube_config_dir }}/external_openstack_cloud_config"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Get base64 cloud-config
+  slurp:
+    src: "{{ kube_config_dir }}/external_openstack_cloud_config"
+  register: external_openstack_cloud_config_secret
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: external-openstack-cloud-config-secret, file: external-openstack-cloud-config-secret.yml}
+    - {name: external-openstack-cloud-controller-manager-roles, file: external-openstack-cloud-controller-manager-roles.yml}
+    - {name: external-openstack-cloud-controller-manager-role-bindings, file: external-openstack-cloud-controller-manager-role-bindings.yml}
+    - {name: external-openstack-cloud-controller-manager-ds, file: external-openstack-cloud-controller-manager-ds.yml}
+  register: external_openstack_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: external-openstack
+
+- name: External OpenStack Cloud Controller | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ external_openstack_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: external-openstack

roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml

@@ -0,0 +1,34 @@
+---
+- name: External OpenStack Cloud Controller | check external_openstack_auth_url value
+  fail:
+    msg: "external_openstack_auth_url is missing"
+  when: external_openstack_auth_url is not defined or not external_openstack_auth_url
+
+- name: External OpenStack Cloud Controller | check external_openstack_username value
+  fail:
+    msg: "external_openstack_username is missing"
+  when: external_openstack_username is not defined or not external_openstack_username
+
+- name: External OpenStack Cloud Controller | check external_openstack_password value
+  fail:
+    msg: "external_openstack_password is missing"
+  when: external_openstack_password is not defined or not external_openstack_password
+
+- name: External OpenStack Cloud Controller | check external_openstack_region value
+  fail:
+    msg: "external_openstack_region is missing"
+  when: external_openstack_region is not defined or not external_openstack_region
+
+- name: External OpenStack Cloud Controller | check external_openstack_tenant_id value
+  fail:
+    msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified"
+  when:
+    - external_openstack_tenant_id is not defined or not external_openstack_tenant_id
+    - external_openstack_tenant_name is not defined
+
+- name: External OpenStack Cloud Controller | check external_openstack_tenant_name value
+  fail:
+    msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified"
+  when:
+    - external_openstack_tenant_name is not defined or not external_openstack_tenant_name
+    - external_openstack_tenant_id is not defined

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2

@@ -0,0 +1,10 @@
+# This YAML file contains secret objects,
+# which are necessary to run external openstack cloud controller.
+
+kind: Secret
+apiVersion: v1
+metadata:
+  name: external-openstack-cloud-config
+  namespace: kube-system
+data:
+  cloud.conf: {{ external_openstack_cloud_config_secret.content }}

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2

@@ -0,0 +1,41 @@
+[Global]
+auth-url="{{ external_openstack_auth_url }}"
+username="{{ external_openstack_username }}"
+password="{{ external_openstack_password }}"
+region="{{ external_openstack_region }}"
+{% if external_openstack_tenant_id is defined and external_openstack_tenant_id != "" %}
+tenant-id="{{ external_openstack_tenant_id }}"
+{% endif %}
+{% if external_openstack_tenant_name is defined and external_openstack_tenant_name != "" %}
+tenant-name="{{ external_openstack_tenant_name }}"
+{% endif %}
+{% if external_openstack_domain_name is defined and external_openstack_domain_name != "" %}
+domain-name="{{ external_openstack_domain_name }}"
+{% elif external_openstack_domain_id is defined and external_openstack_domain_id != "" %}
+domain-id ="{{ external_openstack_domain_id }}"
+{% endif %}
+{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
+ca-file="{{ kube_config_dir }}/external-openstack-cacert.pem"
+{% endif %}
+
+[LoadBalancer]
+use-octavia={{ external_openstack_lbaas_use_octavia }}
+create-monitor={{ openstack_lbaas_create_monitor }}
+monitor-delay={{ openstack_lbaas_monitor_delay }}
+monitor-timeout={{ openstack_lbaas_monitor_timeout }}
+monitor-max-retries={{ openstack_lbaas_monitor_max_retries }}
+{% if external_openstack_lbaas_method is defined %}
+lb-method={{ external_openstack_lbaas_method }}
+{% endif %}
+{% if external_openstack_lbaas_network_id is defined %}
+network-id={{ external_openstack_lbaas_network_id }}
+{% endif %}
+{% if external_openstack_lbaas_subnet_id is defined %}
+subnet-id={{ external_openstack_lbaas_subnet_id }}
+{% endif %}
+{% if external_openstack_lbaas_floating_network_id is defined %}
+floating-network-id={{ external_openstack_lbaas_floating_network_id }}
+{% endif %}
+{% if external_openstack_lbaas_flaoting_subnet_id is defined %}
+floating-subnet-id={{ external_openstack_lbaas_floating_subnet_id }}
+{% endif %}

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2

@@ -0,0 +1,92 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: openstack-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: openstack-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: openstack-cloud-controller-manager
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: openstack-cloud-controller-manager
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsUser: 1001
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      serviceAccountName: cloud-controller-manager
+      containers:
+        - name: openstack-cloud-controller-manager
+          image: {{ docker_image_repo }}/k8scloudprovider/openstack-cloud-controller-manager:{{ external_openstack_cloud_controller_image_tag }}
+          args:
+            - /bin/openstack-cloud-controller-manager
+            - --v=1
+            - --cloud-config=$(CLOUD_CONFIG)
+            - --cloud-provider=openstack
+            - --use-service-account-credentials=true
+            - --address=127.0.0.1
+          volumeMounts:
+            - mountPath: /etc/kubernetes/pki
+              name: k8s-certs
+              readOnly: true
+            - mountPath: /etc/ssl/certs
+              name: ca-certs
+              readOnly: true
+            - mountPath: /etc/config
+              name: cloud-config-volume
+              readOnly: true
+{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
+            - mountPath: {{ kube_config_dir }}/external-openstack-cacert.pem
+              name: openstack-cacert
+              readOnly: true
+{% endif %}
+            - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+              name: flexvolume-dir
+          resources:
+            requests:
+              cpu: 200m
+          env:
+            - name: CLOUD_CONFIG
+              value: /etc/config/cloud.conf
+      hostNetwork: true
+      volumes:
+      - hostPath:
+          path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+          type: DirectoryOrCreate
+        name: flexvolume-dir
+      - hostPath:
+          path: /etc/kubernetes/pki
+          type: DirectoryOrCreate
+        name: k8s-certs
+      - hostPath:
+          path: /etc/ssl/certs
+          type: DirectoryOrCreate
+        name: ca-certs
+      - name: cloud-config-volume
+        secret:
+          secretName: external-openstack-cloud-config
+{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
+      - hostPath:
+          path: {{ kube_config_dir }}/external-openstack-cacert.pem
+          type: FileOrCreate
+        name: openstack-cacert
+{% endif %}

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2

@@ -0,0 +1,40 @@
+apiVersion: v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: system:cloud-node-controller
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:cloud-node-controller
+  subjects:
+  - kind: ServiceAccount
+    name: cloud-node-controller
+    namespace: kube-system
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: system:pvl-controller
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:pvl-controller
+  subjects:
+  - kind: ServiceAccount
+    name: pvl-controller
+    namespace: kube-system
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: system:cloud-controller-manager
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:cloud-controller-manager
+  subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+kind: List
+metadata: {}

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-roles.yml.j2

@@ -0,0 +1,129 @@
+apiVersion: v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    name: system:cloud-controller-manager
+  rules:
+  - apiGroups:
+    - coordination.k8s.io
+    resources:
+    - leases
+    verbs:
+    - get
+    - create
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - events
+    verbs:
+    - create
+    - patch
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - nodes
+    verbs:
+    - '*'
+  - apiGroups:
+    - ""
+    resources:
+    - nodes/status
+    verbs:
+    - patch
+  - apiGroups:
+    - ""
+    resources:
+    - services
+    verbs:
+    - list
+    - patch
+    - update
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - serviceaccounts
+    verbs:
+    - create
+    - get
+  - apiGroups:
+    - ""
+    resources:
+    - persistentvolumes
+    verbs:
+    - '*'
+  - apiGroups:
+    - ""
+    resources:
+    - endpoints
+    verbs:
+    - create
+    - get
+    - list
+    - watch
+    - update
+  - apiGroups:
+    - ""
+    resources:
+    - configmaps
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - ""
+    resources:
+    - secrets
+    verbs:
+    - list
+    - get
+    - watch
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    name: system:cloud-node-controller
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - nodes
+    verbs:
+    - '*'
+  - apiGroups:
+    - ""
+    resources:
+    - nodes/status
+    verbs:
+    - patch
+  - apiGroups:
+    - ""
+    resources:
+    - events
+    verbs:
+    - create
+    - patch
+    - update
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    name: system:pvl-controller
+  rules:
+  - apiGroups:
+    - ""
+    resources:
+    - persistentvolumes
+    verbs:
+    - '*'
+  - apiGroups:
+    - ""
+    resources:
+    - events
+    verbs:
+    - create
+    - patch
+    - update
+kind: List
+metadata: {}