Change MetalLB to one of addons (#6238)

This changes MetalLB contrib to one of addons for deploying MetalLB with
Kubernetes cluster deployment. By the default, Kubespray doesn't deploy
MetalLB addon.

roles/kubernetes-apps/meta/main.yml

@@ -97,3 +97,10 @@ dependencies:
       - inventory_hostname == groups['kube-master'][0]
     tags:
       - oci
+
+  - role: kubernetes-apps/metallb
+    when:
+      - metallb_enabled
+      - inventory_hostname == groups['kube-master'][0]
+    tags:
+      - metallb

roles/kubernetes-apps/metallb/README.md

@@ -0,0 +1,19 @@
+# Deploy MetalLB into Kubespray/Kubernetes
+
+MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation.
+In short, it allows you to create Kubernetes services of type "LoadBalancer" in clusters that
+don't run on a cloud provider, and thus cannot simply hook into paid products to provide load-balancers.
+This playbook aims to automate [this](https://metallb.universe.tf/concepts/layer2/).
+It deploys MetalLB into Kubernetes and sets up a layer 2 load-balancer.
+
+## Install
+
+Defaults can be found in contrib/metallb/roles/provision/defaults/main.yml.
+You can override the defaults by copying the contents of this file to somewhere in inventory/mycluster/group_vars
+such as inventory/mycluster/groups_vars/k8s-cluster/addons.yml and making any adjustments as required.
+MetalLB allocates external IP addresses from this ip_range option, so you need to update this ip_range option
+at least for suiting your network environment.
+
+```
+ansible-playbook --ask-become -i inventory/sample/hosts.ini contrib/metallb/metallb.yml
+```

roles/kubernetes-apps/metallb/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+metallb_enabled: false
+metallb_version: v0.9.3
+metallb_protocol: "layer2"
+metallb_port: "7472"
+metallb_limits_cpu: "100m"
+metallb_limits_mem: "100Mi"

roles/kubernetes-apps/metallb/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+- name: "Kubernetes Apps | Check cluster settings for MetalLB"
+  fail:
+    msg: "MetalLB require kube_proxy_strict_arp = true, see https://github.com/danderson/metallb/issues/153#issuecomment-518651132"
+  when:
+    - "kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp"
+
+- name: Kubernetes Apps | Check cluster settings for MetalLB
+  fail:
+    msg: "metallb_ip_range is mandatory to be specified for MetalLB"
+  when:
+    - metallb_ip_range is not defined or not metallb_ip_range
+
+- name: Kubernetes Apps | Check AppArmor status
+  command: which apparmor_parser
+  register: apparmor_status
+  when:
+    - podsecuritypolicy_enabled
+    - inventory_hostname == groups['kube-master'][0]
+  failed_when: false
+
+- name: Kubernetes Apps | Set apparmor_enabled
+  set_fact:
+    apparmor_enabled: "{{ apparmor_status.rc == 0 }}"
+  when:
+    - podsecuritypolicy_enabled
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: "Kubernetes Apps | Lay Down MetalLB"
+  become: true
+  template: { src: "{{ item }}.j2", dest: "{{ kube_config_dir }}/{{ item }}" }
+  with_items: ["metallb.yml", "metallb-config.yml"]
+  register: "rendering"
+  when:
+    - "inventory_hostname == groups['kube-master'][0]"
+
+- name: "Kubernetes Apps | Install and configure MetalLB"
+  kube:
+    name: "MetalLB"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item }}"
+    state: "{{ item.changed | ternary('latest','present') }}"
+  become: true
+  with_items: "{{ rendering.results }}"
+  when:
+    - "inventory_hostname == groups['kube-master'][0]"
+
+- name: Kubernetes Apps | Check existing secret of MetalLB
+  command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n metallb-system get secret memberlist"
+  register: metallb_secret
+  become: true
+  ignore_errors: yes
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Apps | Create random bytes for MetalLB
+  command: "openssl rand -base64 32"
+  register: metallb_rand
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - metallb_secret.rc != 0
+
+- name: Kubernetes Apps | Install secret of MetalLB if not existing
+  command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf -n metallb-system create secret generic memberlist --from-literal=secretkey={{ metallb_rand.stdout }}"
+  become: true
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - metallb_secret.rc != 0

roles/kubernetes-apps/metallb/templates/metallb-config.yml.j2

@@ -0,0 +1,25 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: loadbalanced
+      protocol: {{ metallb_protocol }}
+      addresses:
+{% for ip_range in metallb_ip_range %}
+      - {{ ip_range }}
+{% endfor %}
+{% if metallb_additional_address_pools is defined %}{% for pool in metallb_additional_address_pools %}
+    - name: {{ pool }}
+      protocol: {{ metallb_additional_address_pools[pool].protocol }}
+      addresses:
+{% for ip_range in metallb_additional_address_pools[pool].ip_range %}
+      - {{ ip_range }}
+{% endfor %}
+      auto-assign: {{ metallb_additional_address_pools[pool].auto_assign }}
+{% endfor %}
+{% endif %}

roles/kubernetes-apps/metallb/templates/metallb.yml.j2

@@ -0,0 +1,398 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: metallb-system
+  labels:
+    app: metallb
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  labels:
+    app: metallb
+  name: controller
+  namespace: metallb-system
+spec:
+  allowPrivilegeEscalation: false
+  allowedCapabilities: []
+  allowedHostPaths: []
+  defaultAddCapabilities: []
+  defaultAllowPrivilegeEscalation: false
+  fsGroup:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  hostIPC: false
+  hostNetwork: false
+  hostPID: false
+  privileged: false
+  readOnlyRootFilesystem: true
+  requiredDropCapabilities:
+  - ALL
+  runAsUser:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  volumes:
+  - configMap
+  - secret
+  - emptyDir
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  labels:
+    app: metallb
+  name: speaker
+  namespace: metallb-system
+spec:
+  allowPrivilegeEscalation: false
+  allowedCapabilities:
+  - NET_ADMIN
+  - NET_RAW
+  - SYS_ADMIN
+  allowedHostPaths: []
+  defaultAddCapabilities: []
+  defaultAllowPrivilegeEscalation: false
+  fsGroup:
+    rule: RunAsAny
+  hostIPC: false
+  hostNetwork: true
+  hostPID: false
+  hostPorts:
+  - max: {{ metallb_port }}
+    min: {{ metallb_port }}
+  privileged: true
+  readOnlyRootFilesystem: true
+  requiredDropCapabilities:
+  - ALL
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+  - configMap
+  - secret
+  - emptyDir
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: metallb
+  name: controller
+  namespace: metallb-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: metallb
+  name: speaker
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:controller
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ''
+  resources:
+  - services/status
+  verbs:
+  - update
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - policy
+  resourceNames:
+  - controller
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:speaker
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - services
+  - endpoints
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ''
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - policy
+  resourceNames:
+  - speaker
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: metallb
+  name: config-watcher
+  namespace: metallb-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: metallb
+  name: pod-lister
+  namespace: metallb-system
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - pods
+  verbs:
+  - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:controller
+subjects:
+- kind: ServiceAccount
+  name: controller
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: metallb-system:speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:speaker
+subjects:
+- kind: ServiceAccount
+  name: speaker
+  namespace: metallb-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: config-watcher
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: config-watcher
+subjects:
+- kind: ServiceAccount
+  name: controller
+- kind: ServiceAccount
+  name: speaker
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: metallb
+  name: pod-lister
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pod-lister
+subjects:
+- kind: ServiceAccount
+  name: speaker
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: metallb
+    component: speaker
+  name: speaker
+  namespace: metallb-system
+spec:
+  selector:
+    matchLabels:
+      app: metallb
+      component: speaker
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '{{ metallb_port }}'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: metallb
+        component: speaker
+    spec:
+      containers:
+      - args:
+        - --port={{ metallb_port }}
+        - --config=config
+        env:
+        - name: METALLB_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: METALLB_HOST
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: METALLB_ML_BIND_ADDR
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: METALLB_ML_LABELS
+          value: "app=metallb,component=speaker"
+        - name: METALLB_ML_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: METALLB_ML_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: memberlist
+              key: secretkey
+        image: metallb/speaker:{{ metallb_version }}
+        imagePullPolicy: Always
+        name: speaker
+        ports:
+        - containerPort: {{ metallb_port }}
+          name: monitoring
+        resources:
+          limits:
+            cpu: {{ metallb_limits_cpu }}
+            memory: {{ metallb_limits_mem }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_ADMIN
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      serviceAccountName: speaker
+      terminationGracePeriodSeconds: 2
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: metallb
+    component: controller
+  name: controller
+  namespace: metallb-system
+spec:
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app: metallb
+      component: controller
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: '{{ metallb_port }}'
+        prometheus.io/scrape: 'true'
+      labels:
+        app: metallb
+        component: controller
+    spec:
+      containers:
+      - args:
+        - --port={{ metallb_port }}
+        - --config=config
+        image: metallb/controller:{{ metallb_version }}
+        imagePullPolicy: Always
+        name: controller
+        ports:
+        - containerPort: {{ metallb_port }}
+          name: monitoring
+        resources:
+          limits:
+            cpu: {{ metallb_limits_cpu }}
+            memory: {{ metallb_limits_mem }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: controller
+      terminationGracePeriodSeconds: 0

roles/kubespray-defaults/defaults/main.yaml

@@ -324,6 +324,7 @@ ingress_ambassador_enabled: false
 ingress_alb_enabled: false
 cert_manager_enabled: false
 expand_persistent_volumes: false
+metallb_enabled: false
 
 ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
 # openstack_blockstorage_version: "v1/v2/auto (default)"