Add vSphere external cloud provider (#5959)

roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml

@@ -0,0 +1,14 @@
+---
+external_vsphere_vcenter_port: "443"
+external_vsphere_insecure: "true"
+external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
+
+vsphere_syncer_image_tag: "v1.0.2"
+vsphere_csi_attacher_image_tag: "v1.1.1"
+vsphere_csi_controller: "v1.0.2"
+vsphere_csi_liveness_probe_image_tag: "v1.1.0"
+vsphere_csi_provisioner_image_tag: "v1.2.2"
+vsphere_csi_node_driver_registrar_image_tag: "v1.1.0"
+vsphere_csi_driver_image_tag: "v1.0.2"
+
+vsphere_csi_controller_replicas: 1

roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml

@@ -0,0 +1,44 @@
+---
+- include_tasks: vsphere-credentials-check.yml
+  tags: vsphere-csi-driver
+
+- name: vSphere CSI Driver | Generate CSI cloud-config
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item }}"
+    mode: 0640
+  with_items:
+    - vsphere-csi-cloud-config
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: vsphere-csi-driver
+
+- name: vSphere CSI Driver | Generate Manifests
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item }}"
+  with_items:
+    - vsphere-csi-controller-rbac.yml
+    - vsphere-csi-controller-ss.yml
+    - vsphere-csi-node.yml
+  register: vsphere_csi_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: vsphere-csi-driver
+
+- name: vSphere CSI Driver | Create a CSI secret
+  command: "{{ bin_dir }}/kubectl create secret generic vsphere-config-secret --from-file=csi-vsphere.conf={{ kube_config_dir }}/vsphere-csi-cloud-config -n kube-system"
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: vsphere-csi-driver
+
+- name: vSphere CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item }}"
+    state: "latest"
+  with_items:
+    - "{{ vsphere_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item }}"
+  tags: vsphere-csi-driver

roles/kubernetes-apps/csi_driver/vsphere/tasks/vsphere-credentials-check.yml

@@ -0,0 +1,38 @@
+---
+- name: External vSphere Cloud Provider | check external_vsphere_vcenter_ip value
+  fail:
+    msg: "external_vsphere_vcenter_ip is missing"
+  when: external_vsphere_vcenter_ip is not defined or not external_vsphere_vcenter_ip
+
+- name: External vSphere Cloud Provider | check external_vsphere_vcenter_port value
+  fail:
+    msg: "external_vsphere_vcenter_port is missing"
+  when: external_vsphere_vcenter_port is not defined or not external_vsphere_vcenter_port
+
+- name: External vSphere Cloud Provider | check external_vsphere_insecure value
+  fail:
+    msg: "external_vsphere_insecure is missing"
+  when: external_vsphere_insecure is not defined or not external_vsphere_insecure
+
+- name: External vSphere Cloud Provider | check external_vsphere_user value
+  fail:
+    msg: "external_vsphere_user is missing"
+  when: external_vsphere_user is not defined or not external_vsphere_user
+
+- name: External vSphere Cloud Provider | check external_vsphere_password value
+  fail:
+    msg: "external_vsphere_password is missing"
+  when:
+    - external_vsphere_password is not defined or not external_vsphere_password
+
+- name: External vSphere Cloud Provider | check external_vsphere_datacenter value
+  fail:
+    msg: "external_vsphere_datacenter is missing"
+  when:
+    - external_vsphere_datacenter is not defined or not external_vsphere_datacenter
+
+- name: External vSphere Cloud Provider | check external_vsphere_kubernetes_cluster_id value
+  fail:
+    msg: "external_vsphere_kubernetes_cluster_id is missing"
+  when:
+    - external_vsphere_kubernetes_cluster_id is not defined or not external_vsphere_kubernetes_cluster_id

roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-cloud-config.j2

@@ -0,0 +1,9 @@
+[Global]
+cluster-id = "{{ external_vsphere_kubernetes_cluster_id }}"
+
+[VirtualCenter "{{ external_vsphere_vcenter_ip }}"]
+insecure-flag = "{{ external_vsphere_insecure }}"
+user = "{{ external_vsphere_user }}"
+password = "{{ external_vsphere_password }}"
+port = "{{ external_vsphere_vcenter_port }}"
+datacenters = "{{ external_vsphere_datacenter }}"

roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2

@@ -0,0 +1,42 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: vsphere-csi-controller
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vsphere-csi-controller-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "persistentvolumeclaims", "pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vsphere-csi-controller-binding
+subjects:
+  - kind: ServiceAccount
+    name: vsphere-csi-controller
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: vsphere-csi-controller-role
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-ss.yml.j2

@@ -0,0 +1,128 @@
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: vsphere-csi-controller
+  namespace: kube-system
+spec:
+  serviceName: vsphere-csi-controller
+  replicas: {{ vsphere_csi_controller_replicas }}
+  updateStrategy:
+    type: "RollingUpdate"
+  selector:
+    matchLabels:
+      app: vsphere-csi-controller
+  template:
+    metadata:
+      labels:
+        app: vsphere-csi-controller
+        role: vsphere-csi
+    spec:
+      serviceAccountName: vsphere-csi-controller
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+        - operator: "Exists"
+          key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      dnsPolicy: "Default"
+      containers:
+        - name: csi-attacher
+          image: {{ quay_image_repo }}/k8scsi/csi-attacher:{{ vsphere_csi_attacher_image_tag }}
+          args:
+            - "--v=4"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+        - name: vsphere-csi-controller
+          image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_controller }}
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /var/lib/csi/sockets/pluginproxy/csi.vsphere.vmware.com"]
+          args:
+            - "--v=4"
+          imagePullPolicy: "Always"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: X_CSI_MODE
+              value: "controller"
+            - name: VSPHERE_CSI_CONFIG
+              value: "/etc/cloud/csi-vsphere.conf"
+          volumeMounts:
+            - mountPath: /etc/cloud
+              name: vsphere-config-volume
+              readOnly: true
+            - mountPath: /var/lib/csi/sockets/pluginproxy/
+              name: socket-dir
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 5
+            failureThreshold: 3
+        - name: liveness-probe
+          image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - mountPath: /var/lib/csi/sockets/pluginproxy/
+              name: socket-dir
+        - name: vsphere-syncer
+          image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/syncer:{{ vsphere_syncer_image_tag }}
+          args:
+            - "--v=2"
+          imagePullPolicy: "Always"
+          env:
+            - name: FULL_SYNC_INTERVAL_MINUTES
+              value: "30"
+            - name: VSPHERE_CSI_CONFIG
+              value: "/etc/cloud/csi-vsphere.conf"
+          volumeMounts:
+            - mountPath: /etc/cloud
+              name: vsphere-config-volume
+              readOnly: true
+        - name: csi-provisioner
+          image: {{ quay_image_repo }}/k8scsi/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }}
+          args:
+            - "--v=4"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+            - "--feature-gates=Topology=true"
+            - "--strict-topology"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+      volumes:
+        - name: vsphere-config-volume
+          secret:
+            secretName: vsphere-config-secret
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/csi/sockets/pluginproxy/csi.vsphere.vmware.com
+            type: DirectoryOrCreate
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: csi.vsphere.vmware.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false

roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2

@@ -0,0 +1,121 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: vsphere-csi-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: vsphere-csi-node
+  updateStrategy:
+    type: "RollingUpdate"
+  template:
+    metadata:
+      labels:
+        app: vsphere-csi-node
+        role: vsphere-csi
+    spec:
+      dnsPolicy: "Default"
+      containers:
+        - name: node-driver-registrar
+          image: {{ quay_image_repo }}/k8scsi/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }}
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com-reg.sock"]
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com/csi.sock
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: vsphere-csi-node
+          image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }}
+          imagePullPolicy: "Always"
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+            - name: X_CSI_MODE
+              value: "node"
+            - name: X_CSI_SPEC_REQ_VALIDATION
+              value: "false"
+            # needed only for topology aware setups
+            #- name: VSPHERE_CSI_CONFIG
+            #  value: "/etc/cloud/csi-vsphere.conf" # here csi-vsphere.conf is the name of the file used for creating secret using "--from-file" flag
+          args:
+            - "--v=4"
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          volumeMounts:
+            # needed only for topology aware setups
+            #- name: vsphere-config-volume
+            #  mountPath: /etc/cloud
+            #  readOnly: true
+            - name: plugin-dir
+              mountPath: /csi
+            - name: pods-mount-dir
+              mountPath: /var/lib/kubelet
+              # needed so that any mounts setup inside this container are
+              # propagated back to the host machine.
+              mountPropagation: "Bidirectional"
+            - name: device-dir
+              mountPath: /dev
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 5
+            failureThreshold: 3
+        - name: liveness-probe
+          image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+      volumes:
+        # needed only for topology aware setups
+        #- name: vsphere-config-volume
+        #  secret:
+        #    secretName: vsphere-config-secret
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: DirectoryOrCreate
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com
+            type: DirectoryOrCreate
+        - name: pods-mount-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev