Fix Bionic networking restart error #3430 (#3431)

* Fix the jinja expression for openstack_tenant_id

OS_PROJECT_ID is obsolete in keystone v3 and jinja expression
doesn't set openstack_tenant_id as expected because of
undefined env var. Fixed the expression.

* Fix the dic iteration method in the kubelet template

Kubelet template rendering errors when additional Node lables are
added and using Python3. Update the method to be compatible to both
python2/3

Node lables doesn't work

.gitlab-ci.yml

@@ -93,7 +93,7 @@ before_script:
     # Check out latest tag if testing upgrade
     # Uncomment when gitlab kubespray repo has tags
     #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
-    - test "${UPGRADE_TEST}" != "false" && git checkout 02cd5418c22d51e40261775908d55bc562206023
+    - test "${UPGRADE_TEST}" != "false" && git checkout 8b3ce6e418ccf48171eb5b3888ee1af84f8d71ba
     # Checkout the CI vars file so it is available
     - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
     # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021
@@ -240,6 +240,10 @@ before_script:
 # stage: deploy-part1
   MOVED_TO_GROUP_VARS: "true"
 
+.ubuntu18_flannel_aio_variables: &ubuntu18_flannel_aio_variables
+# stage: deploy-part1
+  MOVED_TO_GROUP_VARS: "true"
+
 .ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
 # stage: deploy-part1
   UPGRADE_TEST: "graceful"
@@ -263,7 +267,7 @@ before_script:
 .ubuntu_cilium_sep_variables: &ubuntu_cilium_sep_variables
 # stage: deploy-special
   MOVED_TO_GROUP_VARS: "true"
-  
+
 .rhel7_weave_variables: &rhel7_weave_variables
 # stage: deploy-part1
   MOVED_TO_GROUP_VARS: "true"
@@ -331,6 +335,18 @@ gce_coreos-calico-aio:
   only: [/^pr-.*$/]
 
 ### PR JOBS PART2
+
+gce_ubuntu18-flannel-aio:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *ubuntu18_flannel_aio_variables
+    <<: *gce_variables
+  when: manual
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
 gce_centos7-flannel-addons:
   stage: deploy-part2
   <<: *job
@@ -342,6 +358,17 @@ gce_centos7-flannel-addons:
   except: ['triggers']
   only: [/^pr-.*$/]
 
+gce_centos-weave-kubeadm:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *centos_weave_kubeadm_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
 gce_ubuntu-weave-sep:
   stage: deploy-part2
   <<: *job
@@ -438,17 +465,6 @@ gce_ubuntu-canal-kubeadm-triggers:
   when: on_success
   only: ['triggers']
 
-gce_centos-weave-kubeadm:
-  stage: deploy-part2
-  <<: *job
-  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *centos_weave_kubeadm_variables
-  when: manual
-  except: ['triggers']
-  only: ['master', /^pr-.*$/]
-
 gce_centos-weave-kubeadm-triggers:
   stage: deploy-part2
   <<: *job

OWNERS

@@ -1,9 +1,7 @@
 # See the OWNERS file documentation:
-#  https://github.com/kubernetes/kubernetes/blob/master/docs/devel/owners.md
+# https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md
 
-owners:
-  - Smana
-  - ant31
-  - bogdando
-  - mattymo
-  - rsmitty
+approvers:
+  - kubespray-approvers
+reviewers:
+  - kubespray-reviewers

OWNERS_ALIASES

@@ -0,0 +1,18 @@
+aliases:
+  kubespray-approvers:
+    - ant31
+    - mattymo
+    - atoms
+    - chadswen
+    - rsmitty
+    - bogdando
+    - bradbeam
+    - woopstar
+    - riverzhang
+    - holser
+    - smana
+  kubespray-reviewers:
+    - jjungnickel
+    - archifleks
+    - chapsuk
+    - mirwan

README.md

@@ -4,8 +4,9 @@ Deploy a Production Ready Kubernetes Cluster
 ============================================
 
 If you have questions, join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
+You can get your invite [here](http://slack.k8s.io/)
 
--   Can be deployed on **AWS, GCE, Azure, OpenStack, vSphere or Baremetal**
+-   Can be deployed on **AWS, GCE, Azure, OpenStack, vSphere, Oracle Cloud Infrastructure (Experimental), or Baremetal**
 -   **Highly available** cluster
 -   **Composable** (Choice of the network plugin for instance)
 -   Supports most popular **Linux distributions**
@@ -22,19 +23,29 @@ To deploy the cluster you can use :
     sudo pip install -r requirements.txt
 
     # Copy ``inventory/sample`` as ``inventory/mycluster``
-    cp -rfp inventory/sample inventory/mycluster
+    cp -rfp inventory/sample/* inventory/mycluster
 
     # Update Ansible inventory file with inventory builder
     declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5)
     CONFIG_FILE=inventory/mycluster/hosts.ini python3 contrib/inventory_builder/inventory.py ${IPS[@]}
 
     # Review and change parameters under ``inventory/mycluster/group_vars``
-    cat inventory/mycluster/group_vars/all.yml
-    cat inventory/mycluster/group_vars/k8s-cluster.yml
+    cat inventory/mycluster/group_vars/all/all.yml
+    cat inventory/mycluster/group_vars/k8s-cluster/k8s-cluster.yml
 
     # Deploy Kubespray with Ansible Playbook
     ansible-playbook -i inventory/mycluster/hosts.ini cluster.yml
 
+Note: When Ansible is already installed via system packages on the control machine, other python packages installed via `sudo pip install -r requirements.txt` will go to a different directory tree (e.g. `/usr/local/lib/python2.7/dist-packages` on Ubuntu) from Ansible's (e.g. `/usr/lib/python2.7/dist-packages/ansible` still on Ubuntu).
+As a consequence, `ansible-playbook` command will fail with:
+```
+ERROR! no action detected in task. This often indicates a misspelled module name, or incorrect module path.
+```
+probably pointing on a task depending on a module present in requirements.txt (i.e. "unseal vault").
+
+One way of solving this would be to uninstall the Ansible package and then, to install it via pip but it is not always possible.
+A workaround consists of setting `ANSIBLE_LIBRARY` and `ANSIBLE_MODULE_UTILS` environment variables respectively to the `ansible/modules` and `ansible/module_utils` subdirectories of pip packages installation location, which can be found in the Location field of the output of `pip show [package]` before executing `ansible-playbook`.
+
 ### Vagrant
 
 For Vagrant we need to install python dependencies for provisioning tasks.
@@ -79,8 +90,9 @@ Supported Linux Distributions
 
 -   **Container Linux by CoreOS**
 -   **Debian** Jessie, Stretch, Wheezy
--   **Ubuntu** 16.04
+-   **Ubuntu** 16.04, 18.04
 -   **CentOS/RHEL** 7
+-   **Fedora** 28
 -   **Fedora/CentOS** Atomic
 -   **openSUSE** Leap 42.3/Tumbleweed
 
@@ -90,23 +102,25 @@ Supported Components
 --------------------
 
 -   Core
-    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.10.4
+    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.11.3
     -   [etcd](https://github.com/coreos/etcd) v3.2.18
     -   [docker](https://www.docker.com/) v17.03 (see note)
     -   [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)
+    -   [cri-o](http://cri-o.io/) v1.11.5 (experimental: see [CRI-O Note](docs/cri-o.md). Only on centos based OS)
 -   Network Plugin
-    -   [calico](https://github.com/projectcalico/calico) v2.6.8
+    -   [calico](https://github.com/projectcalico/calico) v3.1.3
     -   [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-    -   [cilium](https://github.com/cilium/cilium) v1.1.2
+    -   [cilium](https://github.com/cilium/cilium) v1.2.0
     -   [contiv](https://github.com/contiv/install) v1.1.7
     -   [flanneld](https://github.com/coreos/flannel) v0.10.0
-    -   [weave](https://github.com/weaveworks/weave) v2.4.0
+    -   [weave](https://github.com/weaveworks/weave) v2.4.1
 -   Application
-    -   [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v1.1.0-k8s1.10
-    -   [cert-manager](https://github.com/jetstack/cert-manager) v0.4.0
-    -   [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.17.1
+    -   [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
+    -   [cert-manager](https://github.com/jetstack/cert-manager) v0.5.0
+    -   [coredns](https://github.com/coredns/coredns) v1.2.2
+    -   [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.19.0
 
-Note: kubernetes doesn't support newer docker versions. Among other things kubelet currently breaks on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
+Note: kubernetes doesn't support newer docker versions ("Version 17.03 is recommended... Versions 17.06+ might work, but have not yet been tested and verified by the Kubernetes node team" cf. [Bootstrapping Clusters with kubeadm](https://kubernetes.io/docs/setup/independent/install-kubeadm/#installing-docker)). Among other things kubelet currently breaks on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin). 
 
 Note 2: rkt support as docker alternative is limited to control plane (etcd and
 kubelet). Docker is still used for Kubernetes cluster workloads and network

Vagrantfile

@@ -18,6 +18,7 @@ SUPPORTED_OS = {
   "coreos-beta"   => {box: "coreos-beta",        bootstrap_os: "coreos", user: "core", box_url: COREOS_URL_TEMPLATE % ["beta"]},
   "ubuntu"        => {box: "bento/ubuntu-16.04", bootstrap_os: "ubuntu", user: "vagrant"},
   "centos"        => {box: "centos/7",           bootstrap_os: "centos", user: "vagrant"},
+  "fedora"        => {box: "fedora/28-cloud-base", bootstrap_os: "fedora", user: "vagrant"},
   "opensuse"      => {box: "opensuse/openSUSE-42.3-x86_64", bootstrap_os: "opensuse", use: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/openSUSE-Tumbleweed-x86_64", bootstrap_os: "opensuse", use: "vagrant"},
 }
@@ -44,6 +45,8 @@ $kube_node_instances_with_disks = false
 $kube_node_instances_with_disks_size = "20G"
 $kube_node_instances_with_disks_number = 2
 
+$playbook = "cluster.yml"
+
 $local_release_dir = "/vagrant/temp"
 
 host_vars = {}
@@ -125,6 +128,10 @@ Vagrant.configure("2") do |config|
 
      config.vm.provider :libvirt do |lv|
        lv.memory = $vm_memory
+       # Fix kernel panic on fedora 28
+       if $os == "fedora"
+        lv.cpu_mode = "host-passthrough"
+       end
      end
 
       ip = "#{$subnet}.#{i+100}"
@@ -157,7 +164,7 @@ Vagrant.configure("2") do |config|
       # when all the machines are up and ready.
       if i == $num_instances
         config.vm.provision "ansible" do |ansible|
-          ansible.playbook = "cluster.yml"
+          ansible.playbook = $playbook
           if File.exist?(File.join(File.dirname($inventory), "hosts"))
             ansible.inventory_path = $inventory
           end

ansible.cfg

@@ -13,3 +13,5 @@ callback_whitelist = profile_tasks
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds
+[inventory]
+ignore_patterns = artifacts, credentials

cluster.yml

@@ -33,7 +33,8 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: docker, tags: docker, when: manage_docker|default(true) }
+    - { role: docker, tags: docker, when: container_manager == 'docker' }
+    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
@@ -93,6 +94,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
+    - { role: win_nodes/kubernetes_patch, tags: win_nodes, when: "kubeadm_enabled" }
 
 - hosts: kube-master
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
@@ -117,7 +119,7 @@
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf }
   environment: "{{proxy_env}}"
 
-- hosts: kube-master[0]
+- hosts: kube-master
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}

contrib/metallb/README.md

@@ -0,0 +1,10 @@
+# Deploy MetalLB into Kubespray/Kubernetes
+```
+MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation. In short, it allows you to create Kubernetes services of type “LoadBalancer” in clusters that don’t run on a cloud provider, and thus cannot simply hook into paid products to provide load-balancers.
+```
+This playbook aims to automate [this](https://metallb.universe.tf/tutorial/layer2/tutorial). It deploys MetalLB into kubernetes and sets up a layer 2 loadbalancer.
+
+## Install
+```
+ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/metallb/metallb.yml
+```

contrib/metallb/metallb.yml

@@ -0,0 +1,6 @@
+---
+- hosts: kube-master[0]
+  tags:
+    - "provision"
+  roles:
+    - { role: provision }

contrib/metallb/roles/provision/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+metallb:
+  ip_range: "10.5.0.50-10.5.0.99"
+  limits:
+    cpu: "100m"
+    memory: "100Mi"
+  port: "7472"

contrib/metallb/roles/provision/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: "Kubernetes Apps | Lay Down MetalLB"
+  become: true
+  template: { src: "{{ item }}.j2", dest: "{{ kube_config_dir }}/{{ item }}" }
+  with_items: ["metallb.yml", "metallb-config.yml"]
+  register: "rendering"
+  when:
+    - "inventory_hostname == groups['kube-master'][0]"
+- name: "Kubernetes Apps | Install and configure MetalLB"
+  kube:
+    name: "MetalLB"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item }}"
+    state: "{{ item.changed | ternary('latest','present') }}"
+  with_items: "{{ rendering.results }}"
+  when:
+    - "inventory_hostname == groups['kube-master'][0]"

contrib/metallb/roles/provision/templates/metallb-config.yml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: metallb-system
+  name: config
+data:
+  config: |
+    address-pools:
+    - name: loadbalanced
+      protocol: layer2
+      addresses:
+      - {{ metallb.ip_range }}

contrib/metallb/roles/provision/templates/metallb.yml.j2

@@ -0,0 +1,254 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: metallb-system
+  labels:
+    app: metallb
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: metallb-system
+  name: controller
+  labels:
+    app: metallb
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: metallb-system
+  name: speaker
+  labels:
+    app: metallb
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metallb-system:controller
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["get", "list", "watch", "update"]
+- apiGroups: [""]
+  resources: ["services/status"]
+  verbs: ["update"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metallb-system:speaker
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["services", "endpoints", "nodes"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: metallb-system
+  name: leader-election
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["endpoints"]
+  resourceNames: ["metallb-speaker"]
+  verbs: ["get", "update"]
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: metallb-system
+  name: config-watcher
+  labels:
+    app: metallb
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create"]
+---
+
+## Role bindings
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metallb-system:controller
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: controller
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:controller
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metallb-system:speaker
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: speaker
+  namespace: metallb-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metallb-system:speaker
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: metallb-system
+  name: config-watcher
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: controller
+- kind: ServiceAccount
+  name: speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: config-watcher
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: metallb-system
+  name: leader-election
+  labels:
+    app: metallb
+subjects:
+- kind: ServiceAccount
+  name: speaker
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: leader-election
+---
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  namespace: metallb-system
+  name: speaker
+  labels:
+    app: metallb
+    component: speaker
+spec:
+  selector:
+    matchLabels:
+      app: metallb
+      component: speaker
+  template:
+    metadata:
+      labels:
+        app: metallb
+        component: speaker
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ metallb.port }}"
+    spec:
+      serviceAccountName: speaker
+      terminationGracePeriodSeconds: 0
+      hostNetwork: true
+      containers:
+      - name: speaker
+        image: metallb/speaker:v0.6.2
+        imagePullPolicy: IfNotPresent
+        args:
+        - --port={{ metallb.port }}
+        - --config=config
+        env:
+        - name: METALLB_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - name: monitoring
+          containerPort: {{ metallb.port }}
+        resources:
+          limits:
+            cpu: {{ metallb.limits.cpu }}
+            memory: {{ metallb.limits.memory }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop:
+            - all
+            add:
+            - net_raw
+
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  namespace: metallb-system
+  name: controller
+  labels:
+    app: metallb
+    component: controller
+spec:
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app: metallb
+      component: controller
+  template:
+    metadata:
+      labels:
+        app: metallb
+        component: controller
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ metallb.port }}"
+    spec:
+      serviceAccountName: controller
+      terminationGracePeriodSeconds: 0
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534 # nobody
+      containers:
+      - name: controller
+        image: metallb/controller:v0.6.2
+        imagePullPolicy: IfNotPresent
+        args:
+        - --port={{ metallb.port }}
+        - --config=config
+        ports:
+        - name: monitoring
+          containerPort: {{ metallb.port }}
+        resources:
+          limits:
+            cpu: {{ metallb.limits.cpu }}
+            memory: {{ metallb.limits.memory }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+
+---
+
+

contrib/network-storage/glusterfs/inventory.example

@@ -12,7 +12,7 @@
 # ## As in the previous case, you can set ip to give direct communication on internal IPs
 # gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
 # gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8 
-# gfs_node1 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 
+# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 
 
 # [kube-master]
 # node1

contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml

@@ -2,7 +2,7 @@
 # For Ubuntu.
 glusterfs_default_release: ""
 glusterfs_ppa_use: yes
-glusterfs_ppa_version: "3.8"
+glusterfs_ppa_version: "4.1"
 
 # Gluster configuration.
 gluster_mount_dir: /mnt/gluster

contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml

@@ -2,7 +2,7 @@
 # For Ubuntu.
 glusterfs_default_release: ""
 glusterfs_ppa_use: yes
-glusterfs_ppa_version: "3.8"
+glusterfs_ppa_version: "3.12"
 
 # Gluster configuration.
 gluster_mount_dir: /mnt/gluster

contrib/network-storage/glusterfs/roles/glusterfs/server/vars/Debian.yml

@@ -1,2 +1,2 @@
 ---
-glusterfs_daemon: glusterfs-server
+glusterfs_daemon: glusterd

contrib/network-storage/heketi/README.md

@@ -0,0 +1,16 @@
+# Deploy Heketi/Glusterfs into Kubespray/Kubernetes
+This playbook aims to automate [this](https://github.com/heketi/heketi/blob/master/docs/admin/install-kubernetes.md) tutorial. It deploys heketi/glusterfs into kubernetes and sets up a storageclass.
+
+## Client Setup
+Heketi provides a CLI that provides users with a means to administer the deployment and configuration of GlusterFS in Kubernetes. [Download and install the heketi-cli](https://github.com/heketi/heketi/releases) on your client machine.
+
+## Install
+Copy the inventory.yml.sample over to inventory/sample/k8s_heketi_inventory.yml and change it according to your setup.
+```
+ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi.yml
+```
+
+## Tear down
+```
+ansible-playbook --ask-become -i inventory/sample/k8s_heketi_inventory.yml contrib/network-storage/heketi/heketi-tear-down.yml
+```

contrib/network-storage/heketi/heketi-tear-down.yml

@@ -0,0 +1,9 @@
+---
+- hosts: kube-master[0]
+  roles:
+    - { role: tear-down }
+
+- hosts: heketi-node
+  become: yes
+  roles:
+    - { role: tear-down-disks }

contrib/network-storage/heketi/heketi.yml

@@ -0,0 +1,10 @@
+---
+- hosts: heketi-node
+  roles:
+    - { role: prepare }
+
+- hosts: kube-master[0]
+  tags:
+    - "provision"
+  roles:
+    - { role: provision }

contrib/network-storage/heketi/inventory.yml.sample

@@ -0,0 +1,26 @@
+all:
+    vars:
+        heketi_admin_key: "11elfeinhundertundelf"
+        heketi_user_key: "!!einseinseins"
+    children:
+        k8s-cluster:
+            vars:
+                kubelet_fail_swap_on: false
+            children:
+                kube-master:
+                    hosts:
+                        node1:
+                etcd:
+                    hosts:
+                        node2:
+                kube-node:
+                    hosts: &kube_nodes
+                        node1:
+                        node2:
+                        node3:
+                        node4:
+                heketi-node:
+                    vars:
+                        disk_volume_device_1: "/dev/vdb"
+                    hosts:
+                        <<: *kube_nodes

contrib/network-storage/heketi/requirements.txt

@@ -0,0 +1 @@
+jmespath

contrib/network-storage/heketi/roles/prepare/tasks/main.yml

@@ -0,0 +1,24 @@
+---
+- name: "Load lvm kernel modules"
+  become: true
+  with_items:
+      - "dm_snapshot"
+      - "dm_mirror"
+      - "dm_thin_pool"
+  modprobe:
+      name: "{{ item }}"
+      state: "present"
+
+- name: "Install glusterfs mount utils (RedHat)"
+  become: true
+  yum:
+      name: "glusterfs-fuse"
+      state: "present"
+  when: "ansible_os_family == 'RedHat'"
+
+- name: "Install glusterfs mount utils (Debian)"
+  become: true
+  apt:
+      name: "glusterfs-client"
+      state: "present"
+  when: "ansible_os_family == 'Debian'"

contrib/network-storage/heketi/roles/provision/defaults/main.yml

@@ -0,0 +1 @@
+---

contrib/network-storage/heketi/roles/provision/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: "stop port forwarding"
+  command: "killall "

contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml

@@ -0,0 +1,56 @@
+# Bootstrap heketi
+- name: "Get state of heketi service, deployment and pods."
+  register: "initial_heketi_state"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+- name: "Bootstrap heketi."
+  when:
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0"
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
+    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0"
+  include_tasks: "bootstrap/deploy.yml"
+
+# Prepare heketi topology
+- name: "Get heketi initial pod state."
+  register: "initial_heketi_pod"
+  command: "{{ bin_dir }}/kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
+  changed_when: false
+- name: "Ensure heketi bootstrap pod is up."
+  assert:
+    that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1"
+- set_fact:
+    initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}"
+- name: "Test heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+- name: "Load heketi topology."
+  when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
+  include_tasks: "bootstrap/topology.yml"
+
+# Provision heketi database volume
+- name: "Prepare heketi volumes."
+  include_tasks: "bootstrap/volumes.yml"
+
+# Remove bootstrap heketi
+- name: "Tear down bootstrap."
+  include_tasks: "bootstrap/tear-down.yml"
+
+# Prepare heketi storage
+- name: "Test heketi storage."
+  command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+# ensure endpoints actually exist before trying to move database data to it
+- name: "Create heketi storage."
+  include_tasks: "bootstrap/storage.yml"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -0,0 +1,24 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi Bootstrap"
+  become: true
+  template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" }
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-bootstrap.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Wait for heketi bootstrap to complete."
+  changed_when: false
+  register: "initial_heketi_state"
+  vars:
+    initial_heketi_state: { stdout: "{}" }
+    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
+  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+  until:
+      - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
+      - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml

@@ -0,0 +1,33 @@
+---
+- name: "Test heketi storage."
+  command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+- name: "Create heketi storage."
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-storage-bootstrap.json"
+    state: "present"
+  vars:
+    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
+    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
+    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+  when:
+    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
+    - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
+  register: "heketi_storage_result"
+- name: "Get state of heketi database copy job."
+  command: "{{ bin_dir }}/kubectl get jobs --output=json"
+  changed_when: false
+  register: "heketi_storage_state"
+  vars:
+    heketi_storage_state: { stdout: "{}" }
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]"
+  until:
+    - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/tear-down.yml

@@ -0,0 +1,14 @@
+---
+- name: "Get existing Heketi deploy resources."
+  command: "{{ bin_dir }}/kubectl get all --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_resources"
+  changed_when: false
+- name: "Delete bootstrap Heketi."
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\""
+  when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0"
+- name: "Ensure there is nothing left over."
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

@@ -0,0 +1,26 @@
+---
+- name: "Get heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+- name: "Render heketi topology template."
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  register: "render"
+  template:
+    src: "topology.json.j2"
+    dest: "{{ kube_config_dir }}/topology.json"
+- name: "Copy topology configuration into container."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"
+- name: "Load heketi topology."
+  when: "render.changed"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json"
+  register: "load_heketi"
+- name: "Get heketi topology."
+  changed_when: false
+  register: "heketi_topology"
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+  until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml

@@ -0,0 +1,41 @@
+---
+- name: "Get heketi volume ids."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json"
+  changed_when: false
+  register: "heketi_volumes"
+- name: "Get heketi volumes."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json"
+  with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
+  loop_control: { loop_var: "volume_id" }
+  register: "volumes_information"
+- name: "Test heketi database volume."
+  set_fact: { heketi_database_volume_exists: true }
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout|from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+- name: "Provision database volume."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} setup-openshift-heketi-storage"
+  when: "heketi_database_volume_exists is undefined"
+- name: "Copy configuration from pod."
+  become: true
+  command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage-bootstrap.json"
+- name: "Get heketi volume ids."
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json"
+  changed_when: false
+  register: "heketi_volumes"
+- name: "Get heketi volumes."
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume info {{ volume_id }} --json"
+  with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
+  loop_control: { loop_var: "volume_id" }
+  register: "volumes_information"
+- name: "Test heketi database volume."
+  set_fact: { heketi_database_volume_created: true }
+  with_items: "{{ volumes_information.results }}"
+  loop_control: { loop_var: "volume_information" }
+  vars: { volume: "{{ volume_information.stdout|from_json }}" }
+  when: "volume.name == 'heketidbstorage'"
+- name: "Ensure heketi database volume exists."
+  assert: { that: "heketi_database_volume_created is defined" , msg: "Heketi database volume does not exist." }

contrib/network-storage/heketi/roles/provision/tasks/cleanup.yml

@@ -0,0 +1,4 @@
+---
+- name: "Clean up left over jobs."
+  command: "{{ bin_dir }}/kubectl delete jobs,pods --selector=\"deploy-heketi\""
+  changed_when: false

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -0,0 +1,38 @@
+---
+- name: "Kubernetes Apps | Lay Down GlusterFS Daemonset"
+  template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" }
+  become: true
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/glusterfs-daemonset.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Kubernetes Apps | Label GlusterFS nodes"
+  include_tasks: "glusterfs/label.yml"
+  with_items: "{{ groups['heketi-node'] }}"
+  loop_control:
+    loop_var: "node"
+- name: "Kubernetes Apps | Wait for daemonset to become available."
+  register: "daemonset_state"
+  command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
+  changed_when: false
+  vars:
+    daemonset_state: { stdout: "{}" }
+    ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}"
+    desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}"
+  until: "ready | int >= 3"
+  retries: 60
+  delay: 5
+
+- name: "Kubernetes Apps | Lay Down Heketi Service Account"
+  template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" }
+  become: true
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Service Account"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-service-account.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/glusterfs/label.yml

@@ -0,0 +1,11 @@
+---
+- register: "label_present"
+  command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+- name: "Assign storage label"
+  when: "label_present.stdout_lines|length == 0"
+  command: "{{ bin_dir }}/kubectl label node {{ node }} storagenode=glusterfs"
+- register: "label_present"
+  command: "{{ bin_dir }}/kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." }

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -0,0 +1,26 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi"
+  become: true
+  template: { src: "heketi-deployment.json.j2", dest: "{{ kube_config_dir }}/heketi-deployment.json" }
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-deployment.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Ensure heketi is up and running."
+  changed_when: false
+  register: "heketi_state"
+  vars:
+    heketi_state: { stdout: "{}" }
+    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
+  command: "{{ bin_dir }}/kubectl get deployments,pods --selector=glusterfs --output=json"
+  until:
+      - "heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
+      - "heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
+  retries: 60
+  delay: 5
+- set_fact:
+    heketi_pod_name: "{{ heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.name|[0]\") }}"

contrib/network-storage/heketi/roles/provision/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: "Kubernetes Apps | GlusterFS"
+  include_tasks: "glusterfs.yml"
+
+- name: "Kubernetes Apps | Heketi Secrets"
+  include_tasks: "secret.yml"
+
+- name: "Kubernetes Apps | Test Heketi"
+  register: "heketi_service_state"
+  command: "{{bin_dir}}/kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true"
+  changed_when: false
+
+- name: "Kubernetes Apps | Bootstrap Heketi"
+  when: "heketi_service_state.stdout == \"\""
+  include_tasks: "bootstrap.yml"
+
+- name: "Kubernetes Apps | Heketi"
+  include_tasks: "heketi.yml"
+
+- name: "Kubernetes Apps | Heketi Topology"
+  include_tasks: "topology.yml"
+
+- name: "Kubernetes Apps | Heketi Storage"
+  include_tasks: "storage.yml"
+
+- name: "Kubernetes Apps | Storage Class"
+  include_tasks: "storageclass.yml"
+
+- name: "Clean up"
+  include_tasks: "cleanup.yml"

contrib/network-storage/heketi/roles/provision/tasks/secret.yml

@@ -0,0 +1,27 @@
+---
+- register: "clusterrolebinding_state"
+  command: "{{bin_dir}}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Kubernetes Apps | Deploy cluster role binding."
+  when: "clusterrolebinding_state.stdout == \"\""
+  command: "{{bin_dir}}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
+- register: "clusterrolebinding_state"
+  command: "{{bin_dir}}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." }
+
+- register: "secret_state"
+  command: "{{bin_dir}}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Render Heketi secret configuration."
+  become: true
+  template:
+    src: "heketi.json.j2"
+    dest: "{{ kube_config_dir }}/heketi.json"
+- name: "Deploy Heketi config secret"
+  when: "secret_state.stdout == \"\""
+  command: "{{bin_dir}}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
+- register: "secret_state"
+  command: "{{bin_dir}}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." }

contrib/network-storage/heketi/roles/provision/tasks/storage.yml

@@ -0,0 +1,12 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi Storage"
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" }
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Storage"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-storage.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -0,0 +1,25 @@
+---
+- name: "Test storage class."
+  command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json"
+  register: "storageclass"
+  changed_when: false
+- name: "Test heketi service."
+  command: "{{ bin_dir }}/kubectl get service heketi --ignore-not-found=true --output=json"
+  register: "heketi_service"
+  changed_when: false
+- name: "Ensure heketi service is available."
+  assert: { that: "heketi_service.stdout != \"\""  }
+- name: "Render storage class configuration."
+  become: true
+  vars:
+    endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}"
+  template:
+    src: "storageclass.yml.j2"
+    dest: "{{ kube_config_dir }}/storageclass.yml"
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Storace Class"
+  kube:
+    name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/storageclass.yml"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -0,0 +1,25 @@
+---
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+- name: "Render heketi topology template."
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  register: "rendering"
+  template:
+    src: "topology.json.j2"
+    dest: "{{ kube_config_dir }}/topology.json"
+- name: "Copy topology configuration into container."
+  when: "rendering.changed"
+  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"
+- name: "Load heketi topology."
+  when: "rendering.changed"
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json"
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
+  until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2

@@ -0,0 +1,144 @@
+{
+    "kind": "DaemonSet",
+    "apiVersion": "extensions/v1beta1",
+    "metadata": {
+        "name": "glusterfs",
+        "labels": {
+            "glusterfs": "deployment"
+        },
+        "annotations": {
+            "description": "GlusterFS Daemon Set",
+            "tags": "glusterfs"
+        }
+    },
+    "spec": {
+        "template": {
+            "metadata": {
+                "name": "glusterfs",
+                "labels": {
+                    "glusterfs-node": "daemonset"
+                }
+            },
+            "spec": {
+                "nodeSelector": {
+                    "storagenode" : "glusterfs"
+                },
+                "hostNetwork": true,
+                "containers": [
+                    {
+                        "image": "gluster/gluster-centos:gluster4u0_centos7",
+                        "imagePullPolicy": "IfNotPresent",
+                        "name": "glusterfs",
+                        "volumeMounts": [
+                            {
+                                "name": "glusterfs-heketi",
+                                "mountPath": "/var/lib/heketi"
+                            },
+                            {
+                                "name": "glusterfs-run",
+                                "mountPath": "/run"
+                            },
+                            {
+                                "name": "glusterfs-lvm",
+                                "mountPath": "/run/lvm"
+                            },
+                            {
+                                "name": "glusterfs-etc",
+                                "mountPath": "/etc/glusterfs"
+                            },
+                            {
+                                "name": "glusterfs-logs",
+                                "mountPath": "/var/log/glusterfs"
+                            },
+                            {
+                                "name": "glusterfs-config",
+                                "mountPath": "/var/lib/glusterd"
+                            },
+                            {
+                                "name": "glusterfs-dev",
+                                "mountPath": "/dev"
+                            },
+                            {
+                                "name": "glusterfs-cgroup",
+                                "mountPath": "/sys/fs/cgroup"
+                            }
+                        ],
+                        "securityContext": {
+                            "capabilities": {},
+                            "privileged": true
+                        },
+                        "readinessProbe": {
+                            "timeoutSeconds": 3,
+                            "initialDelaySeconds": 60,
+                            "exec": {
+                                "command": [
+                                    "/bin/bash",
+                                    "-c",
+                                    "systemctl status glusterd.service"
+                                ]
+                            }
+                        },
+                        "livenessProbe": {
+                            "timeoutSeconds": 3,
+                            "initialDelaySeconds": 60,
+                            "exec": {
+                                "command": [
+                                    "/bin/bash",
+                                    "-c",
+                                    "systemctl status glusterd.service"
+                                ]
+                            }
+                        }
+                    }
+                ],
+                "volumes": [
+                    {
+                        "name": "glusterfs-heketi",
+                        "hostPath": {
+                            "path": "/var/lib/heketi"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-run"
+                    },
+                    {
+                        "name": "glusterfs-lvm",
+                        "hostPath": {
+                            "path": "/run/lvm"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-etc",
+                        "hostPath": {
+                            "path": "/etc/glusterfs"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-logs",
+                        "hostPath": {
+                            "path": "/var/log/glusterfs"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-config",
+                        "hostPath": {
+                            "path": "/var/lib/glusterd"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-dev",
+                        "hostPath": {
+                            "path": "/dev"
+                        }
+                    },
+                    {
+                        "name": "glusterfs-cgroup",
+                        "hostPath": {
+                            "path": "/sys/fs/cgroup"
+                        }
+                    }
+                ]
+            }
+        }
+    }
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-bootstrap.json.j2

@@ -0,0 +1,133 @@
+{
+  "kind": "List",
+  "apiVersion": "v1",
+  "items": [
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "deploy-heketi",
+        "labels": {
+          "glusterfs": "heketi-service",
+          "deploy-heketi": "support"
+        },
+        "annotations": {
+          "description": "Exposes Heketi Service"
+        }
+      },
+      "spec": {
+        "selector": {
+          "name": "deploy-heketi"
+        },
+        "ports": [
+          {
+            "name": "deploy-heketi",
+            "port": 8080,
+            "targetPort": 8080
+          }
+        ]
+      }
+    },
+    {
+      "kind": "Deployment",
+      "apiVersion": "extensions/v1beta1",
+      "metadata": {
+        "name": "deploy-heketi",
+        "labels": {
+          "glusterfs": "heketi-deployment",
+          "deploy-heketi": "deployment"
+        },
+        "annotations": {
+          "description": "Defines how to deploy Heketi"
+        }
+      },
+      "spec": {
+        "replicas": 1,
+        "template": {
+          "metadata": {
+            "name": "deploy-heketi",
+            "labels": {
+              "name": "deploy-heketi",
+              "glusterfs": "heketi-pod",
+              "deploy-heketi": "pod"
+            }
+          },
+          "spec": {
+            "serviceAccountName": "heketi-service-account",
+            "containers": [
+              {
+                "image": "heketi/heketi:7",
+                "imagePullPolicy": "Always",
+                "name": "deploy-heketi",
+                "env": [
+                  {
+                    "name": "HEKETI_EXECUTOR",
+                    "value": "kubernetes"
+                  },
+                  {
+                    "name": "HEKETI_DB_PATH",
+                    "value": "/var/lib/heketi/heketi.db"
+                  },
+                  {
+                    "name": "HEKETI_FSTAB",
+                    "value": "/var/lib/heketi/fstab"
+                  },
+                  {
+                    "name": "HEKETI_SNAPSHOT_LIMIT",
+                    "value": "14"
+                  },
+                  {
+                    "name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
+                    "value": "y"
+                  }
+                ],
+                "ports": [
+                  {
+                    "containerPort": 8080
+                  }
+                ],
+                "volumeMounts": [
+                  {
+                    "name": "db",
+                    "mountPath": "/var/lib/heketi"
+                  },
+                  {
+                    "name": "config",
+                    "mountPath": "/etc/heketi"
+                  }
+                ],
+                "readinessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 3,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                },
+                "livenessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 30,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                }
+              }
+            ],
+            "volumes": [
+              {
+                "name": "db"
+              },
+              {
+                "name": "config",
+                "secret": {
+                  "secretName": "heketi-config-secret"
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-deployment.json.j2

@@ -0,0 +1,159 @@
+{
+  "kind": "List",
+  "apiVersion": "v1",
+  "items": [
+    {
+      "kind": "Secret",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-db-backup",
+        "labels": {
+          "glusterfs": "heketi-db",
+          "heketi": "db"
+        }
+      },
+      "data": {
+      },
+      "type": "Opaque"
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi",
+        "labels": {
+          "glusterfs": "heketi-service",
+          "deploy-heketi": "support"
+        },
+        "annotations": {
+          "description": "Exposes Heketi Service"
+        }
+      },
+      "spec": {
+        "selector": {
+          "name": "heketi"
+        },
+        "ports": [
+          {
+            "name": "heketi",
+            "port": 8080,
+            "targetPort": 8080
+          }
+        ]
+      }
+    },
+    {
+      "kind": "Deployment",
+      "apiVersion": "extensions/v1beta1",
+      "metadata": {
+        "name": "heketi",
+        "labels": {
+          "glusterfs": "heketi-deployment"
+        },
+        "annotations": {
+          "description": "Defines how to deploy Heketi"
+        }
+      },
+      "spec": {
+        "replicas": 1,
+        "template": {
+          "metadata": {
+            "name": "heketi",
+            "labels": {
+              "name": "heketi",
+              "glusterfs": "heketi-pod"
+            }
+          },
+          "spec": {
+            "serviceAccountName": "heketi-service-account",
+            "containers": [
+              {
+                "image": "heketi/heketi:7",
+                "imagePullPolicy": "Always",
+                "name": "heketi",
+                "env": [
+                  {
+                    "name": "HEKETI_EXECUTOR",
+                    "value": "kubernetes"
+                  },
+                  {
+                    "name": "HEKETI_DB_PATH",
+                    "value": "/var/lib/heketi/heketi.db"
+                  },
+                  {
+                    "name": "HEKETI_FSTAB",
+                    "value": "/var/lib/heketi/fstab"
+                  },
+                  {
+                    "name": "HEKETI_SNAPSHOT_LIMIT",
+                    "value": "14"
+                  },
+                  {
+                    "name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
+                    "value": "y"
+                  }
+                ],
+                "ports": [
+                  {
+                    "containerPort": 8080
+                  }
+                ],
+                "volumeMounts": [
+                  {
+                    "mountPath": "/backupdb",
+                    "name": "heketi-db-secret"
+                  },
+                  {
+                    "name": "db",
+                    "mountPath": "/var/lib/heketi"
+                  },
+                  {
+                    "name": "config",
+                    "mountPath": "/etc/heketi"
+                  }
+                ],
+                "readinessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 3,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                },
+                "livenessProbe": {
+                  "timeoutSeconds": 3,
+                  "initialDelaySeconds": 30,
+                  "httpGet": {
+                    "path": "/hello",
+                    "port": 8080
+                  }
+                }
+              }
+            ],
+            "volumes": [
+              {
+                "name": "db",
+                "glusterfs": {
+                  "endpoints": "heketi-storage-endpoints",
+                  "path": "heketidbstorage"
+                }
+              },
+              {
+                "name": "heketi-db-secret",
+                "secret": {
+                  "secretName": "heketi-db-backup"
+                }
+              },
+              {
+                "name": "config",
+                "secret": {
+                  "secretName": "heketi-config-secret"
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-service-account.json.j2

@@ -0,0 +1,7 @@
+{
+  "apiVersion": "v1",
+  "kind": "ServiceAccount",
+  "metadata": {
+    "name": "heketi-service-account"
+  }
+}

contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2

@@ -0,0 +1,54 @@
+{
+  "apiVersion": "v1",
+  "kind": "List",
+  "items": [
+    {
+      "kind": "Endpoints",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-storage-endpoints",
+        "creationTimestamp": null
+      },
+      "subsets": [
+{% set nodeblocks = [] %}
+{% for node in nodes %}
+{% set nodeblock %}
+        {
+          "addresses": [
+            {
+              "ip": "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}"
+            }
+          ],
+          "ports": [
+            {
+              "port": 1
+            }
+          ]
+        }
+{% endset %}
+{% if nodeblocks.append(nodeblock) %}{% endif %}
+{% endfor %}
+{{ nodeblocks|join(',') }}
+      ]
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-storage-endpoints",
+        "creationTimestamp": null
+      },
+      "spec": {
+        "ports": [
+          {
+            "port": 1,
+            "targetPort": 0
+          }
+        ]
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    }
+  ]
+}

contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2

@@ -0,0 +1,44 @@
+{
+  "_port_comment": "Heketi Server Port Number",
+  "port": "8080",
+
+  "_use_auth": "Enable JWT authorization. Please enable for deployment",
+  "use_auth": true,
+
+  "_jwt": "Private keys for access",
+  "jwt": {
+    "_admin": "Admin has access to all APIs",
+    "admin": {
+      "key": "{{ heketi_admin_key }}"
+    },
+    "_user": "User only has access to /volumes endpoint",
+    "user": {
+      "key": "{{ heketi_user_key }}"
+    }
+  },
+
+  "_glusterfs_comment": "GlusterFS Configuration",
+  "glusterfs": {
+    "_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
+    "executor": "kubernetes",
+
+    "_db_comment": "Database file name",
+    "db": "/var/lib/heketi/heketi.db",
+
+    "kubeexec": {
+      "rebalance_on_expansion": true
+    },
+
+    "sshexec": {
+      "rebalance_on_expansion": true,
+      "keyfile": "/etc/heketi/private_key",
+      "fstab": "/etc/fstab",
+      "port": "22",
+      "user": "root",
+      "sudo": false
+    }
+  },
+
+  "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
+  "backup_db_to_kube_secret": false
+}

contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2

@@ -0,0 +1,12 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: gluster
+  annotations:
+    storageclass.beta.kubernetes.io/is-default-class: "true"
+provisioner: kubernetes.io/glusterfs
+parameters:
+  resturl: "http://{{ endpoint_address }}:8080"
+  restuser: "admin"
+  restuserkey: "{{ heketi_admin_key }}"

contrib/network-storage/heketi/roles/provision/templates/topology.json.j2

@@ -0,0 +1,34 @@
+{
+  "clusters": [
+    {
+      "nodes": [
+{% set nodeblocks = [] %}
+{% for node in nodes %}
+{% set nodeblock %}
+        {
+          "node": {
+            "hostnames": {
+              "manage": [
+                "{{ node }}"
+              ],
+              "storage": [
+                "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}"
+              ]
+            },
+            "zone": 1
+          },
+          "devices": [
+            {
+              "name": "{{ hostvars[node]['disk_volume_device_1'] }}",
+              "destroydata": false
+            }
+          ]
+        }
+{% endset %}
+{% if nodeblocks.append(nodeblock) %}{% endif %}
+{% endfor %}
+{{ nodeblocks|join(',') }}
+      ]
+    }
+  ]
+}

contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml

@@ -0,0 +1,46 @@
+---
+- name: "Install lvm utils (RedHat)"
+  become: true
+  yum:
+      name: "lvm2"
+      state: "present"
+  when: "ansible_os_family == 'RedHat'"
+
+- name: "Install lvm utils (Debian)"
+  become: true
+  apt:
+      name: "lvm2"
+      state: "present"
+  when: "ansible_os_family == 'Debian'"
+
+- name: "Get volume group information."
+  become: true
+  shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2"
+  register: "volume_groups"
+  ignore_errors: true
+  changed_when: false
+
+- name: "Remove volume groups."
+  become: true
+  command: "vgremove {{ volume_group }} --yes"
+  with_items: "{{ volume_groups.stdout_lines }}"
+  loop_control: { loop_var: "volume_group" }
+
+- name: "Remove physical volume from cluster disks."
+  become: true
+  command: "pvremove {{ disk_volume_device_1 }} --yes"
+  ignore_errors: true
+
+- name: "Remove lvm utils (RedHat)"
+  become: true
+  yum:
+      name: "lvm2"
+      state: "absent"
+  when: "ansible_os_family == 'RedHat'"
+
+- name: "Remove lvm utils (Debian)"
+  become: true
+  apt:
+      name: "lvm2"
+      state: "absent"
+  when: "ansible_os_family == 'Debian'"

contrib/network-storage/heketi/roles/tear-down/tasks/main.yml

@@ -0,0 +1,51 @@
+---
+- name: "Remove storage class."
+  command: "{{ bin_dir }}/kubectl delete storageclass gluster"
+  ignore_errors: true
+- name: "Tear down heketi."
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\""
+  ignore_errors: true
+- name: "Tear down heketi."
+  command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\""
+  ignore_errors: true
+- name: "Tear down bootstrap."
+  include_tasks: "../provision/tasks/bootstrap/tear-down.yml"
+- name: "Ensure there is nothing left over."
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
+  retries: 60
+  delay: 5
+- name: "Ensure there is nothing left over."
+  command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json"
+  register: "heketi_result"
+  until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
+  retries: 60
+  delay: 5
+- name: "Tear down glusterfs."
+  command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs"
+  ignore_errors: true
+- name: "Remove heketi storage service."
+  command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints"
+  ignore_errors: true
+- name: "Remove heketi gluster role binding"
+  command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin"
+  ignore_errors: true
+- name: "Remove heketi config secret"
+  command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret"
+  ignore_errors: true
+- name: "Remove heketi db backup"
+  command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup"
+  ignore_errors: true
+- name: "Remove heketi service account"
+  command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account"
+  ignore_errors: true
+- name: "Get secrets"
+  command: "{{ bin_dir }}/kubectl get secrets --output=\"json\""
+  register: "secrets"
+  changed_when: false
+- name: "Remove heketi storage secret"
+  vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" }
+  command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}"
+  when: "storage_query is defined"
+  ignore_errors: true

contrib/terraform/aws/README.md

@@ -22,8 +22,6 @@ export TF_VAR_AWS_SECRET_ACCESS_KEY ="xxx"
 export TF_VAR_AWS_SSH_KEY_NAME="yyy"
 export TF_VAR_AWS_DEFAULT_REGION="zzz"
 ```
-- Rename `contrib/terraform/aws/terraform.tfvars.example` to `terraform.tfvars`
-
 - Update `contrib/terraform/aws/terraform.tfvars` with your data. By default, the Terraform scripts use CoreOS as base image. If you want to change this behaviour, see note "Using other distrib than CoreOs" below.
 - Create an AWS EC2 SSH Key
 - Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials

contrib/terraform/aws/create-infrastructure.tf

@@ -181,7 +181,7 @@ data "template_file" "inventory" {
 
 resource "null_resource" "inventories" {
   provisioner "local-exec" {
-      command = "echo '${data.template_file.inventory.rendered}' > ../../../inventory/hosts"
+      command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
   }
 
   triggers {

contrib/terraform/aws/terraform.tfvars

@@ -31,3 +31,5 @@ default_tags = {
 #  Env = "devtest"
 #  Product = "kubernetes"
 }
+
+inventory_file = "../../../inventory/hosts"

contrib/terraform/aws/variables.tf

@@ -103,3 +103,7 @@ variable "default_tags" {
   description = "Default tags for all resources"
   type = "map"
 }
+
+variable "inventory_file" {
+  description = "Where to store the generated inventory file"
+}

contrib/terraform/openstack/README.md

@@ -223,6 +223,8 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tf`.
 |`number_of_gfs_nodes_no_floating_ip` | Number of gluster servers to provision. |
 | `gfs_volume_size_in_gb` | Size of the non-ephemeral volumes to be attached to store the GlusterFS bricks |
 |`supplementary_master_groups` | To add ansible groups to the masters, such as `kube-node` for tainting them as nodes, empty by default. |
+|`supplementary_node_groups` | To add ansible groups to the nodes, such as `kube-ingress` for running ingress controller pods, empty by default. |
+|`bastion_allowed_remote_ips` | List of CIDR allowed to initiate a SSH connection, `["0.0.0.0/0"]` by default |
 
 #### Terraform state files
 
@@ -258,7 +260,7 @@ $ terraform apply -var-file=cluster.tf ../../contrib/terraform/openstack
 
 if you chose to create a bastion host, this script will create
 `contrib/terraform/openstack/k8s-cluster.yml` with an ssh command for Ansible to
-be able to access your machines tunneling  through the bastion's IP address. If
+be able to access your machines tunneling through the bastion's IP address. If
 you want to manually handle the ssh tunneling to these machines, please delete
 or move that file. If you want to use this, just leave it there, as ansible will
 pick it up automatically.

contrib/terraform/openstack/kubespray.tf

@@ -50,7 +50,9 @@ module "compute" {
   k8s_master_fips                              = "${module.ips.k8s_master_fips}"
   k8s_node_fips                                = "${module.ips.k8s_node_fips}"
   bastion_fips                                 = "${module.ips.bastion_fips}"
+  bastion_allowed_remote_ips                   = "${var.bastion_allowed_remote_ips}"
   supplementary_master_groups                  = "${var.supplementary_master_groups}"
+  supplementary_node_groups                    = "${var.supplementary_node_groups}"
 
   network_id = "${module.network.router_id}"
 }

contrib/terraform/openstack/modules/compute/main.tf

@@ -3,72 +3,62 @@ resource "openstack_compute_keypair_v2" "k8s" {
   public_key = "${chomp(file(var.public_key_path))}"
 }
 
-resource "openstack_compute_secgroup_v2" "k8s_master" {
+resource "openstack_networking_secgroup_v2" "k8s_master" {
   name        = "${var.cluster_name}-k8s-master"
   description = "${var.cluster_name} - Kubernetes Master"
-
-  rule {
-    ip_protocol = "tcp"
-    from_port   = "6443"
-    to_port     = "6443"
-    cidr        = "0.0.0.0/0"
-  }
 }
 
-resource "openstack_compute_secgroup_v2" "bastion" {
+resource "openstack_networking_secgroup_rule_v2" "k8s_master" {
+  direction = "ingress"
+  ethertype = "IPv4"
+  protocol = "tcp"
+  port_range_min = "6443"
+  port_range_max = "6443"
+  remote_ip_prefix = "0.0.0.0/0"
+  security_group_id = "${openstack_networking_secgroup_v2.k8s_master.id}"
+}
+
+resource "openstack_networking_secgroup_v2" "bastion" {
   name        = "${var.cluster_name}-bastion"
   description = "${var.cluster_name} - Bastion Server"
-
-  rule {
-    ip_protocol = "tcp"
-    from_port   = "22"
-    to_port     = "22"
-    cidr        = "0.0.0.0/0"
-  }
 }
 
-resource "openstack_compute_secgroup_v2" "k8s" {
+resource "openstack_networking_secgroup_rule_v2" "bastion" {
+  count = "${length(var.bastion_allowed_remote_ips)}"
+  direction = "ingress"
+  ethertype = "IPv4"
+  protocol = "tcp"
+  port_range_min = "22"
+  port_range_max = "22"
+  remote_ip_prefix = "${var.bastion_allowed_remote_ips[count.index]}"
+  security_group_id = "${openstack_networking_secgroup_v2.bastion.id}"
+}
+
+resource "openstack_networking_secgroup_v2" "k8s" {
   name        = "${var.cluster_name}-k8s"
   description = "${var.cluster_name} - Kubernetes"
-
-  rule {
-    ip_protocol = "icmp"
-    from_port   = "-1"
-    to_port     = "-1"
-    cidr        = "0.0.0.0/0"
-  }
-
-  rule {
-    ip_protocol = "tcp"
-    from_port   = "1"
-    to_port     = "65535"
-    self        = true
-  }
-
-  rule {
-    ip_protocol = "udp"
-    from_port   = "1"
-    to_port     = "65535"
-    self        = true
-  }
-
-  rule {
-    ip_protocol = "icmp"
-    from_port   = "-1"
-    to_port     = "-1"
-    self        = true
-  }
 }
-resource "openstack_compute_secgroup_v2" "worker" {
+
+resource "openstack_networking_secgroup_rule_v2" "k8s" {
+  direction = "ingress"
+  ethertype = "IPv4"
+  remote_group_id = "${openstack_networking_secgroup_v2.k8s.id}"
+  security_group_id = "${openstack_networking_secgroup_v2.k8s.id}"
+}
+
+resource "openstack_networking_secgroup_v2" "worker" {
   name        = "${var.cluster_name}-k8s-worker"
   description = "${var.cluster_name} - Kubernetes worker nodes"
+}
 
-  rule {
-    ip_protocol = "tcp"
-    from_port   = "30000"
-    to_port     = "32767"
-    cidr        = "0.0.0.0/0"
-  }
+resource "openstack_networking_secgroup_rule_v2" "worker" {
+  direction = "ingress"
+  ethertype = "IPv4"
+  protocol = "tcp"
+  port_range_min = "30000"
+  port_range_max = "32767"
+  remote_ip_prefix = "0.0.0.0/0"
+  security_group_id = "${openstack_networking_secgroup_v2.worker.id}"
 }
 
 resource "openstack_compute_instance_v2" "bastion" {
@@ -82,8 +72,8 @@ resource "openstack_compute_instance_v2" "bastion" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s.name}",
-    "${openstack_compute_secgroup_v2.bastion.name}",
+  security_groups = ["${openstack_networking_secgroup_v2.k8s.name}",
+    "${openstack_networking_secgroup_v2.bastion.name}",
     "default",
   ]
 
@@ -111,9 +101,9 @@ resource "openstack_compute_instance_v2" "k8s_master" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s_master.name}",
-    "${openstack_compute_secgroup_v2.bastion.name}",
-    "${openstack_compute_secgroup_v2.k8s.name}",
+  security_groups = ["${openstack_networking_secgroup_v2.k8s_master.name}",
+    "${openstack_networking_secgroup_v2.bastion.name}",
+    "${openstack_networking_secgroup_v2.k8s.name}",
     "default",
   ]
 
@@ -141,9 +131,9 @@ resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s_master.name}",
-    "${openstack_compute_secgroup_v2.bastion.name}",
-    "${openstack_compute_secgroup_v2.k8s.name}",
+  security_groups = ["${openstack_networking_secgroup_v2.k8s_master.name}",
+    "${openstack_networking_secgroup_v2.bastion.name}",
+    "${openstack_networking_secgroup_v2.k8s.name}",
   ]
 
   metadata = {
@@ -170,7 +160,7 @@ resource "openstack_compute_instance_v2" "etcd" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s.name}"]
+  security_groups = ["${openstack_networking_secgroup_v2.k8s.name}"]
 
   metadata = {
     ssh_user         = "${var.ssh_user}"
@@ -192,8 +182,8 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s_master.name}",
-    "${openstack_compute_secgroup_v2.k8s.name}",
+  security_groups = ["${openstack_networking_secgroup_v2.k8s_master.name}",
+    "${openstack_networking_secgroup_v2.k8s.name}",
     "default",
   ]
 
@@ -217,8 +207,8 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s_master.name}",
-    "${openstack_compute_secgroup_v2.k8s.name}",
+  security_groups = ["${openstack_networking_secgroup_v2.k8s_master.name}",
+    "${openstack_networking_secgroup_v2.k8s.name}",
   ]
 
   metadata = {
@@ -241,15 +231,15 @@ resource "openstack_compute_instance_v2" "k8s_node" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s.name}",
-    "${openstack_compute_secgroup_v2.bastion.name}",
-    "${openstack_compute_secgroup_v2.worker.name}",
+  security_groups = ["${openstack_networking_secgroup_v2.k8s.name}",
+    "${openstack_networking_secgroup_v2.bastion.name}",
+    "${openstack_networking_secgroup_v2.worker.name}",
     "default",
   ]
 
   metadata = {
     ssh_user         = "${var.ssh_user}"
-    kubespray_groups = "kube-node,k8s-cluster"
+    kubespray_groups = "kube-node,k8s-cluster,${var.supplementary_node_groups}"
     depends_on       = "${var.network_id}"
   }
 
@@ -271,14 +261,14 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s.name}",
-    "${openstack_compute_secgroup_v2.worker.name}",
+  security_groups = ["${openstack_networking_secgroup_v2.k8s.name}",
+    "${openstack_networking_secgroup_v2.worker.name}",
     "default",
   ]
 
   metadata = {
     ssh_user         = "${var.ssh_user}"
-    kubespray_groups = "kube-node,k8s-cluster,no-floating"
+    kubespray_groups = "kube-node,k8s-cluster,no-floating,${var.supplementary_node_groups}"
     depends_on       = "${var.network_id}"
   }
 
@@ -321,7 +311,7 @@ resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" {
     name = "${var.network_name}"
   }
 
-  security_groups = ["${openstack_compute_secgroup_v2.k8s.name}",
+  security_groups = ["${openstack_networking_secgroup_v2.k8s.name}",
     "default",
   ]
 

contrib/terraform/openstack/modules/compute/variables.tf

@@ -60,6 +60,14 @@ variable "bastion_fips" {
   type = "list"
 }
 
+variable "bastion_allowed_remote_ips" {
+  type = "list"
+}
+
 variable "supplementary_master_groups" {
   default = ""
 }
+
+variable "supplementary_node_groups" {
+  default = ""
+}

contrib/terraform/openstack/modules/network/outputs.tf

@@ -1,4 +1,8 @@
 output "router_id" {
+  value = "${openstack_networking_router_v2.k8s.id}"
+}
+
+output "router_internal_port_id" {
   value = "${openstack_networking_router_interface_v2.k8s.id}"
 }
 

contrib/terraform/openstack/sample-inventory/cluster.tf

@@ -43,4 +43,4 @@ network_name = "<network>"
 external_net = "<UUID>"
 subnet_cidr = "<cidr>"
 floatingip_pool = "<pool>"
-
+bastion_allowed_remote_ips = ["0.0.0.0/0"]

contrib/terraform/openstack/variables.tf

@@ -128,3 +128,14 @@ variable "supplementary_master_groups" {
   description = "supplementary kubespray ansible groups for masters, such kube-node"
   default = ""
 }
+
+variable "supplementary_node_groups" {
+  description = "supplementary kubespray ansible groups for worker nodes, such as kube-ingress"
+  default = ""
+}
+
+variable "bastion_allowed_remote_ips" {
+  description = "An array of CIDRs allowed to SSH to hosts"
+  type = "list"
+  default = ["0.0.0.0/0"]
+}

docs/ansible.md

@@ -158,7 +158,7 @@ And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/reso
 ```
 ansible-playbook -i inventory/sample/hosts.ini -e dnsmasq_dns_server='' cluster.yml --tags resolvconf
 ```
-And this prepares all container images localy (at the ansible runner node) without installing
+And this prepares all container images locally (at the ansible runner node) without installing
 or upgrading related stuff or trying to upload container to K8s cluster nodes:
 ```
 ansible-playbook -i inventory/sample/hosts.ini cluster.yml \

docs/aws.md

@@ -1,7 +1,7 @@
 AWS
 ===============
 
-To deploy kubespray on [AWS](https://aws.amazon.com/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'aws'`.
+To deploy kubespray on [AWS](https://aws.amazon.com/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'aws'`. Refer to the [Kubespray Configuration](#kubespray-configuration) for customizing the provider.
 
 Prior to creating your instances, you **must** ensure that you have created IAM roles and policies for both "kubernetes-master" and "kubernetes-node". You can find the IAM policies [here](https://github.com/kubernetes-incubator/kubespray/tree/master/contrib/aws_iam/). See the [IAM Documentation](https://aws.amazon.com/documentation/iam/) if guidance is needed on how to set these up. When you bring your instances online, associate them with the respective IAM role. Nodes that are only to be used for Etcd do not need a role.
 
@@ -58,3 +58,21 @@ export AWS_SECRET_ACCESS_KEY="yyyyy"
 export REGION="us-east-2"
 ```
 - We will now create our cluster. There will be either one or two small changes. The first is that we will specify `-i inventory/kubespray-aws-inventory.py` as our inventory script. The other is conditional. If your AWS instances are public facing, you can set the `VPC_VISIBILITY` variable to `public` and that will result in public IP and DNS names being passed into the inventory. This causes your cluster.yml command to look like `VPC_VISIBILITY="public" ansible-playbook ... cluster.yml`
+
+## Kubespray configuration
+
+Declare the cloud config variables for the `aws` provider as follows. Setting these variables are optional and depend on your use case.
+
+Variable|Type|Comment
+---|---|---
+aws_zone|string|Force set the AWS zone. Recommended to leave blank.
+aws_vpc|string|The AWS VPC flag enables the possibility to run the master components on a different aws account, on a different cloud provider or on-premise. If the flag is set also the KubernetesClusterTag must be provided
+aws_subnet_id|string|SubnetID enables using a specific subnet to use for ELB's
+aws_route_table_id|string|RouteTableID enables using a specific RouteTable
+aws_role_arn|string|RoleARN is the IAM role to assume when interaction with AWS APIs
+aws_kubernetes_cluster_tag|string|KubernetesClusterTag is the legacy cluster id we'll use to identify our cluster resources
+aws_kubernetes_cluster_id|string|KubernetesClusterID is the cluster id we'll use to identify our cluster resources
+aws_disable_security_group_ingress|bool|The aws provider creates an inbound rule per load balancer on the node security group. However, this can run into the AWS security group rule limit of 50 if many LoadBalancers are created. This flag disables the automatic ingress creation. It requires that the user has setup a rule that allows inbound traffic on kubelet ports from the local VPC subnet (so load balancers can access it). E.g. 10.82.0.0/16 30000-32000.
+aws_elb_security_group|string|Only in Kubelet version >= 1.7 : AWS has a hard limit of 500 security groups. For large clusters creating a security group for each ELB can cause the max number of security groups to be reached. If this is set instead of creating a new Security group for each ELB this security group will be used instead.
+aws_disable_strict_zone_check|bool|During the instantiation of an new AWS cloud provider, the detected region is validated against a known set of regions. In a non-standard, AWS like environment (e.g. Eucalyptus), this check may be undesirable.  Setting this to true will disable the check and provide a warning that the check was skipped.  Please note that this is an experimental feature and work-in-progress for the moment.
+

docs/calico.md

@@ -1,6 +1,13 @@
 Calico
 ===========
 
+---
+ **N.B. Version 2.6.5 upgrade to 3.1.1 is upgrading etcd store to etcdv3**
+ If you create automated backups of etcdv2 please switch for creating etcdv3 backups, as kubernetes and calico now uses etcdv3
+ After migration you can check `/tmp/calico_upgrade/` directory for converted items to etcdv3.
+ **PLEASE TEST upgrade before upgrading production cluster.**
+ ---
+
 Check if the calico-node container is running
 
 ```
@@ -14,7 +21,7 @@ The **calicoctl** command allows to check the status of the network workloads.
 calicoctl node status
 ```
 
-or for versions prior *v1.0.0*:
+or for versions prior to *v1.0.0*:
 
 ```
 calicoctl status
@@ -26,7 +33,7 @@ calicoctl status
 calicoctl get ippool -o wide
 ```
 
-or for versions prior *v1.0.0*:
+or for versions prior to *v1.0.0*:
 
 ```
 calicoctl pool show
@@ -66,7 +73,7 @@ In some cases you may want to route the pods subnet and so NAT is not needed on
 For instance if you have a cluster spread on different locations and you want your pods to talk each other no matter where they are located.
 The following variables need to be set:
 `peer_with_router` to enable the peering with the datacenter's border router (default value: false).
-you'll need to edit the inventory and add a and a hostvar `local_as` by node.
+you'll need to edit the inventory and add a hostvar `local_as` by node.
 
 ```
 node1 ansible_ssh_host=95.54.0.12 local_as=xxxxxx
@@ -86,7 +93,7 @@ To do so you can deploy BGP route reflectors and peer `calico-node` with them as
 recommended here:
 
 * https://hub.docker.com/r/calico/routereflector/
-* http://docs.projectcalico.org/v2.0/reference/private-cloud/l3-interconnect-fabric
+* https://docs.projectcalico.org/v3.1/reference/private-cloud/l3-interconnect-fabric
 
 You need to edit your inventory and add:
 
@@ -149,7 +156,7 @@ The inventory above will deploy the following topology assuming that calico's
 
 ##### Optional : Define default endpoint to host action
 
-By default Calico blocks traffic from endpoints to the host itself by using an iptables DROP action. When using it in kubernetes the action has to be changed to RETURN (default in kubespray) or ACCEPT (see https://github.com/projectcalico/felix/issues/660 and https://github.com/projectcalico/calicoctl/issues/1389). Otherwise all network packets from pods (with hostNetwork=False) to services endpoints (with hostNetwork=True) withing the same node are dropped.
+By default Calico blocks traffic from endpoints to the host itself by using an iptables DROP action. When using it in kubernetes the action has to be changed to RETURN (default in kubespray) or ACCEPT (see https://github.com/projectcalico/felix/issues/660 and https://github.com/projectcalico/calicoctl/issues/1389). Otherwise all network packets from pods (with hostNetwork=False) to services endpoints (with hostNetwork=True) within the same node are dropped.
 
 
 To re-define default action please set the following variable in your inventory:
@@ -157,6 +164,15 @@ To re-define default action please set the following variable in your inventory:
 calico_endpoint_to_host_action: "ACCEPT"
 ```
 
+##### Optional : Define address on which Felix will respond to health requests
+
+Since Calico 3.2.0, HealthCheck default behavior changed from listening on all interfaces to just listening on localhost.
+
+To re-define health host please set the following variable in your inventory:
+```
+calico_healthhost: "0.0.0.0"
+```
+
 Cloud providers configuration
 =============================
 

docs/contiv.md

@@ -54,16 +54,18 @@ The default configuration uses VXLAN to create an overlay. Two networks are crea
 
 You can change the default network configuration by overriding the `contiv_networks` variable.
 
-The default forward mode is set to routing:
+The default forward mode is set to routing and the default network mode is vxlan:
 
 ```yaml
 contiv_fwd_mode: routing
+contiv_net_mode: vxlan
 ```
 
 The following is an example of how you can use VLAN instead of VXLAN:
 
 ```yaml
 contiv_fwd_mode: bridge
+contiv_net_mode: vlan
 contiv_vlan_interface: eth0
 contiv_networks:
   - name: default-net

docs/cri-o.md

@@ -0,0 +1,31 @@
+cri-o
+===============
+
+cri-o is container developed by kubernetes project.
+Currently, only basic function is supported for cri-o.
+
+* cri-o is supported kubernetes 1.11.1 or later.
+* helm and other feature may not be supported due to docker dependency.
+* scale.yml and upgrade-cluster.yml are not supported.
+
+helm and other feature may not be supported due to docker dependency.
+
+Use cri-o instead of docker, set following variable:
+
+#### all.yml
+
+```
+kubeadm_enabled: true
+...
+download_container: false
+skip_downloads: false
+```
+
+#### k8s-cluster.yml
+
+```
+etcd_deployment_type: host
+kubelet_deployment_type: host
+container_manager: crio
+```
+

docs/downloads.md

@@ -40,3 +40,14 @@ The full list of available vars may be found in the download's ansible role defa
 Those also allow to specify custom urls and local repositories for binaries and container
 images as well. See also the DNS stack docs for the related intranet configuration,
 so the hosts can resolve those urls and repos.
+
+## Offline environment
+
+In case your servers don't have access to internet (for example when deploying on premises with security constraints), you'll have, first, to setup the appropriate proxies/caches/mirrors and/or internal repositories and registries and, then, adapt the following variables to fit your environment before deploying:
+
+* At least `foo_image_repo` and `foo_download_url` as described before (i.e. in case of use of proxies to registries and binaries repositories, checksums and versions do not necessarily need to be changed).
+  NB: Regarding `foo_image_repo`, when using insecure registries/proxies, you will certainly have to append them to the `docker_insecure_registries` variable in group_vars/all/docker.yml
+* Depending on the `container_manager`
+  * When `container_manager=docker`, `docker_foo_repo_base_url`, `docker_foo_repo_gpgkey`, `dockerproject_bar_repo_base_url` and `dockerproject_bar_repo_gpgkey` (where `foo` is the distribution and `bar` is system package manager)
+  * When `container_manager=crio`, `crio_rhel_repo_base_url` 
+* When using Helm, `helm_stable_repo_url`

docs/getting-started.md

@@ -89,7 +89,7 @@ authentication. One could generate a kubeconfig based on one installed
 kube-master hosts (needs improvement) or connect with a username and password.
 By default, a user with admin rights is created, named `kube`.
 The password can be viewed after deployment by looking at the file
-`PATH_TO_KUBESPRAY/credentials/kube_user.creds`. This contains a randomly generated
+`{{ credentials_dir }}/kube_user.creds` (`credentials_dir` is set to `{{ inventory_dir }}/credentials` by default). This contains a randomly generated
 password. If you wish to set your own password, just precreate/modify this
 file yourself.
 

docs/ha-mode.md

@@ -5,18 +5,38 @@ The following components require a highly available endpoints:
 * etcd cluster,
 * kube-apiserver service instances.
 
-The latter relies on a 3rd side reverse proxies, like Nginx or HAProxy, to
+The latter relies on a 3rd side reverse proxy, like Nginx or HAProxy, to
 achieve the same goal.
 
 Etcd
 ----
 
-The `etcd_access_endpoint` fact provides an access pattern for clients. And the
-`etcd_multiaccess` (defaults to `True`) group var controls that behavior.
-It makes deployed components to access the etcd cluster members
-directly: `http://ip1:2379, http://ip2:2379,...`. This mode assumes the clients
-do a loadbalancing and handle HA for connections.
+In order to use an external loadbalancing (L4/TCP or L7 w/ SSL Passthrough VIP), the following variables need to be overriden in group_vars
+* `etcd_access_addresses`
+* `etcd_client_url`
+* `etcd_cert_alt_names`
+* `etcd_cert_alt_ips`
 
+### Example of a VIP w/ FQDN
+```yaml
+etcd_access_addresses: https://etcd.example.com:2379
+etcd_client_url: https://etcd.example.com:2379
+etcd_cert_alt_names:
+  - "etcd.kube-system.svc.{{ dns_domain }}"
+  - "etcd.kube-system.svc"
+  - "etcd.kube-system"
+  - "etcd"
+  - "etcd.example.com" # This one needs to be added to the default etcd_cert_alt_names
+```
+
+### Example of a VIP w/o FQDN (IP only)
+
+```yaml
+etcd_access_addresses: https://2.3.7.9:2379
+etcd_client_url: https://2.3.7.9:2379
+etcd_cert_alt_ips:
+  - "2.3.7.9"
+```
 
 Kube-apiserver
 --------------
@@ -83,7 +103,7 @@ loadbalancer_apiserver:
   so you will need to use a different port for the vip from that the API is
   listening on, or set the `kube_apiserver_bind_address` so that the API only
   listens on a specific interface (to avoid conflict with haproxy binding the
-  port on the VIP adddress)
+  port on the VIP address)
 
 This domain name, or default "lb-apiserver.kubernetes.local", will be inserted
 into the `/etc/hosts` file of all servers in the `k8s-cluster` group and wired

docs/kubernetes-reliability.md

@@ -15,7 +15,7 @@ By default the normal behavior looks like:
 2. Kubernetes controller manager checks the statuses of Kubelets every
    `–-node-monitor-period`. The default value is **5s**.
 
-3. In case the status is updated  within `--node-monitor-grace-period` of time,
+3. In case the status is updated within `--node-monitor-grace-period` of time,
    Kubernetes controller manager considers healthy status of Kubelet. The
    default value is **40s**.
 
@@ -33,7 +33,7 @@ Kubelet will try to make `nodeStatusUpdateRetry` post attempts. Currently
 [kubelet.go](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet.go#L102).
 
 Kubelet will try to update the status in
-[tryUpdateNodeStatus](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet_node_status.go#L345)
+[tryUpdateNodeStatus](https://github.com/kubernetes/kubernetes/blob/release-1.5/pkg/kubelet/kubelet_node_status.go#L312)
 function. Kubelet uses `http.Client()` Golang method, but has no specified
 timeout. Thus there may be some glitches when API Server is overloaded while
 TCP connection is established.
@@ -69,7 +69,7 @@ minute which may require large etcd containers or even dedicated nodes for etcd.
 
 > If we calculate the number of tries, the division will give 5, but in reality
 > it will be from 3 to 5 with `nodeStatusUpdateRetry` attempts of each try. The
-> total number of attemtps will vary from 15 to 25 due to latency of all
+> total number of attempts will vary from 15 to 25 due to latency of all
 > components.
 
 ## Medium Update and Average Reaction
@@ -92,7 +92,7 @@ etcd updates per minute.
 Let's set `-–node-status-update-frequency` to **1m**.
 `--node-monitor-grace-period` will set to **5m** and `--pod-eviction-timeout`
 to **1m**. In this scenario, every kubelet will try to update the status every
-minute. There will be 5 * 5 = 25 attempts before unhealty status. After 5m,
+minute. There will be 5 * 5 = 25 attempts before unhealthy status. After 5m,
 Kubernetes controller manager will set unhealthy status. This means that pods
 will be evicted after 1m after being marked unhealthy. (6m in total).
 

docs/openstack.md

@@ -3,7 +3,7 @@ OpenStack
 
 To deploy kubespray on [OpenStack](https://www.openstack.org/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'openstack'`.
 
-After that make sure to source in your OpenStack credentials like you would do when using `nova-client` by using `source path/to/your/openstack-rc`.
+After that make sure to source in your OpenStack credentials like you would do when using `nova-client` or `neutron-client` by using `source path/to/your/openstack-rc` or `. path/to/your/openstack-rc`.
 
 The next step is to make sure the hostnames in your `inventory` file are identical to your instance names in OpenStack.
 Otherwise [cinder](https://wiki.openstack.org/wiki/Cinder) won't work as expected.
@@ -12,35 +12,34 @@ Unless you are using calico you can now run the playbook.
 
 **Additional step needed when using calico:**
 
-Calico does not encapsulate all packages with the hosts ip addresses. Instead the packages will be routed with the PODs ip addresses directly.
+Calico does not encapsulate all packages with the hosts' ip addresses. Instead the packages will be routed with the PODs ip addresses directly.
+
 OpenStack will filter and drop all packages from ips it does not know to prevent spoofing.
 
-In order to make calico work on OpenStack you will need to tell OpenStack to allow calicos packages by allowing the network it uses.
+In order to make calico work on OpenStack you will need to tell OpenStack to allow calico's packages by allowing the network it uses.
 
 First you will need the ids of your OpenStack instances that will run kubernetes:
 
-    nova list --tenant Your-Tenant
+    openstack server list --project YOUR_PROJECT
     +--------------------------------------+--------+----------------------------------+--------+-------------+
     | ID                                   | Name   | Tenant ID                        | Status | Power State |
     +--------------------------------------+--------+----------------------------------+--------+-------------+
     | e1f48aad-df96-4bce-bf61-62ae12bf3f95 | k8s-1  | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running     |
     | 725cd548-6ea3-426b-baaa-e7306d3c8052 | k8s-2  | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running     |
 
-Then you can use the instance ids to find the connected [neutron](https://wiki.openstack.org/wiki/Neutron) ports:
+Then you can use the instance ids to find the connected [neutron](https://wiki.openstack.org/wiki/Neutron) ports (though they are now configured through using OpenStack):
 
-    neutron port-list -c id -c device_id
+    openstack port list -c id -c device_id --project YOUR_PROJECT
     +--------------------------------------+--------------------------------------+
     | id                                   | device_id                            |
     +--------------------------------------+--------------------------------------+
     | 5662a4e0-e646-47f0-bf88-d80fbd2d99ef | e1f48aad-df96-4bce-bf61-62ae12bf3f95 |
     | e5ae2045-a1e1-4e99-9aac-4353889449a7 | 725cd548-6ea3-426b-baaa-e7306d3c8052 |
 
-Given the port ids on the left, you can set the `allowed_address_pairs` in neutron.
-Note that you have to allow both of `kube_service_addresses` (default `10.233.0.0/18`)
-and `kube_pods_subnet` (default `10.233.64.0/18`.)
+Given the port ids on the left, you can set the two `allowed_address`(es) in OpenStack. Note that you have to allow both `kube_service_addresses` (default `10.233.0.0/18`) and `kube_pods_subnet` (default `10.233.64.0/18`.)
 
     # allow kube_service_addresses and kube_pods_subnet network
-    neutron port-update 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed_address_pairs list=true type=dict ip_address=10.233.0.0/18 ip_address=10.233.64.0/18
-    neutron port-update e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed_address_pairs list=true type=dict ip_address=10.233.0.0/18 ip_address=10.233.64.0/18
+    openstack port set 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18
+    openstack port set e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18
 
 Now you can finally run the playbook.

docs/proxy.md

@@ -0,0 +1,16 @@
+# Setting up Environment Proxy
+
+If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray-defaults/defaults/main.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
+
+## Set proxy for http and https
+
+ `http_proxy:"http://example.proxy.tld:port"`
+ `https_proxy:"http://example.proxy.tld:port"`
+
+## Set default no_proxy (this will override default no_proxy generation)
+
+`no_proxy: "node1,node1_ip,node2,node2_ip...additional_host"`
+
+## Set additional addresses to default no_proxy (all cluster nodes and loadbalancer)
+
+`additional_no_proxy: "aditional_host,"`

docs/roadmap.md

@@ -24,7 +24,7 @@ Kubespray's roadmap
 
 ### Tests
 - [ ] Run kubernetes e2e tests
-- [ ] Test idempotency on on single OS but for all network plugins/container engines
+- [ ] Test idempotency on single OS but for all network plugins/container engines
 - [ ] single test on AWS per day
 - [ ] test scale up cluster:  +1 etcd, +1 master, +1 node
 - [ ] Reorganize CI test vars into group var files

docs/upgrades.md

@@ -35,7 +35,7 @@ ansible-playbook cluster.yml -i inventory/sample/hosts.ini -e kube_version=v1.4.
 
 #### Graceful upgrade
 
-Kubespray also supports cordon, drain and uncordoning of nodes when performing 
+Kubespray also supports cordon, drain and uncordoning of nodes when performing
 a cluster upgrade. There is a separate playbook used for this purpose. It is
 important to note that upgrade-cluster.yml can only be used for upgrading an
 existing cluster. That means there must be at least 1 kube-master already
@@ -86,7 +86,7 @@ for impact to user deployed pods.
 
 A deployer may want to upgrade specific components in order to minimize risk
 or save time. This strategy is not covered by CI as of this writing, so it is
-not guaranteed to work. 
+not guaranteed to work.
 
 These commands are useful only for upgrading fully-deployed, healthy, existing
 hosts. This will definitely not work for undeployed or partially deployed

docs/vars.md

@@ -126,9 +126,20 @@ node_labels:
   label1_name: label1_value
   label2_name: label2_value
 ```
+* *podsecuritypolicy_enabled* - When set to `true`, enables the PodSecurityPolicy admission controller and defines two policies `privileged` (applying to all resources in `kube-system` namespace and kubelet) and `restricted` (applying all other namespaces).
+  Addons deployed in kube-system namespaces are handled.
+* *kubernetes_audit* - When set to `true`, enables Auditing.
+  The auditing parameters can be tuned via the following variables (which default values are shown below):
+  * `audit_log_path`: /var/log/audit/kube-apiserver-audit.log
+  * `audit_log_maxage`: 30
+  * `audit_log_maxbackups`: 1
+  * `audit_log_maxsize`: 100
+  * `audit_policy_file`: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+
+  By default, the `audit_policy_file` contains [default rules](https://github.com/kubernetes-incubator/kubespray/blob/master/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2) that can be overriden with the `audit_policy_custom_rules` variable.
 
 ##### Custom flags for Kube Components
-For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example:
+For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not masters. Example:
 ```
 kubelet_custom_flags:
   - "--eviction-hard=memory.available<100Mi"
@@ -140,11 +151,12 @@ The possible vars are:
 * *controller_mgr_custom_flags*
 * *scheduler_custom_flags*
 * *kubelet_custom_flags*
+* *kubelet_node_custom_flags*
 
 #### User accounts
 
 By default, a user with admin rights is created, named `kube`.
 The password can be viewed after deployment by looking at the file
-`PATH_TO_KUBESPRAY/credentials/kube_user.creds`. This contains a randomly generated
+`{{ credentials_dir }}/kube_user.creds` (`credentials_dir` is set to `{{ inventory_dir }}/credentials` by default). This contains a randomly generated
 password. If you wish to set your own password, just precreate/modify this
 file yourself or change `kube_api_pwd` var.

docs/weave.md

@@ -9,7 +9,7 @@ Weave uses [**consensus**](https://www.weave.works/docs/net/latest/ipam/##consen
 
 Weave encryption is supported for all communication
 
-* To use Weave encryption, specify a strong password (if no password, no encrytion)
+* To use Weave encryption, specify a strong password (if no password, no encryption)
 
 ```
 # In file ./inventory/sample/group_vars/k8s-cluster.yml

inventory/sample/group_vars/all.yml

@@ -1,133 +0,0 @@
-# Valid bootstrap options (required): ubuntu, coreos, centos, none
-bootstrap_os: none
-
-#Directory where etcd data stored
-etcd_data_dir: /var/lib/etcd
-
-# Directory where the binaries will be installed
-bin_dir: /usr/local/bin
-
-## The access_ip variable is used to define how other nodes should access
-## the node.  This is used in flannel to allow other flannel nodes to see
-## this node for example.  The access_ip is really useful AWS and Google
-## environments where the nodes are accessed remotely by the "public" ip,
-## but don't know about that address themselves.
-#access_ip: 1.1.1.1
-
-### LOADBALANCING AND ACCESS MODES
-## Enable multiaccess to configure etcd clients to access all of the etcd members directly
-## as the "http://hostX:port, http://hostY:port, ..." and ignore the proxy loadbalancers.
-## This may be the case if clients support and loadbalance multiple etcd servers natively.
-#etcd_multiaccess: true
-
-### ETCD: disable peer client cert authentication.
-# This affects ETCD_PEER_CLIENT_CERT_AUTH variable
-#etcd_peer_client_auth: true
-
-## External LB example config
-## apiserver_loadbalancer_domain_name: "elb.some.domain"
-#loadbalancer_apiserver:
-#  address: 1.2.3.4
-#  port: 1234
-
-## Internal loadbalancers for apiservers
-#loadbalancer_apiserver_localhost: true
-
-## Local loadbalancer should use this port instead, if defined.
-## Defaults to kube_apiserver_port (6443)
-#nginx_kube_apiserver_port: 8443
-
-### OTHER OPTIONAL VARIABLES
-## For some things, kubelet needs to load kernel modules.  For example, dynamic kernel services are needed
-## for mounting persistent volumes into containers.  These may not be loaded by preinstall kubernetes
-## processes.  For example, ceph and rbd backed volumes.  Set to true to allow kubelet to load kernel
-## modules.
-#kubelet_load_modules: false
-
-## Internal network total size. This is the prefix of the
-## entire network. Must be unused in your environment.
-#kube_network_prefix: 18
-
-## With calico it is possible to distributed routes with border routers of the datacenter.
-## Warning : enabling router peering will disable calico's default behavior ('node mesh').
-## The subnets of each nodes will be distributed by the datacenter router
-#peer_with_router: false
-
-## Upstream dns servers used by dnsmasq
-#upstream_dns_servers:
-#  - 8.8.8.8
-#  - 8.8.4.4
-
-## There are some changes specific to the cloud providers
-## for instance we need to encapsulate packets with some network plugins
-## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', or 'external'
-## When openstack is used make sure to source in the openstack credentials
-## like you would do when using nova-client before starting the playbook.
-#cloud_provider:
-
-## When azure is used, you need to also set the following variables.
-## see docs/azure.md for details on how to get these values
-#azure_tenant_id:
-#azure_subscription_id:
-#azure_aad_client_id:
-#azure_aad_client_secret:
-#azure_resource_group:
-#azure_location:
-#azure_subnet_name:
-#azure_security_group_name:
-#azure_vnet_name:
-#azure_vnet_resource_group:
-#azure_route_table_name:
-
-## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
-#openstack_blockstorage_version: "v1/v2/auto (default)"
-## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
-#openstack_lbaas_enabled: True
-#openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP"
-## To enable automatic floating ip provisioning, specify a subnet.
-#openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default"
-## Override default LBaaS behavior
-#openstack_lbaas_use_octavia: False
-#openstack_lbaas_method: "ROUND_ROBIN"
-#openstack_lbaas_provider: "haproxy"
-#openstack_lbaas_create_monitor: "yes"
-#openstack_lbaas_monitor_delay: "1m"
-#openstack_lbaas_monitor_timeout: "30s"
-#openstack_lbaas_monitor_max_retries: "3"
-
-## Uncomment to enable experimental kubeadm deployment mode
-#kubeadm_enabled: false
-## Set these proxy values in order to update package manager and docker daemon to use proxies
-#http_proxy: ""
-#https_proxy: ""
-## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy
-#no_proxy: ""
-
-## Uncomment this if you want to force overlay/overlay2 as docker storage driver
-## Please note that overlay2 is only supported on newer kernels
-#docker_storage_options: -s overlay2
-
-# Uncomment this if you have more than 3 nameservers, then we'll only use the first 3.
-#docker_dns_servers_strict: false
-
-## Certificate Management
-## This setting determines whether certs are generated via scripts or whether a
-## cluster of Hashicorp's Vault is started to issue certificates (using etcd
-## as a backend). Options are "script" or "vault"
-#cert_management: script
-
-# Set to true to allow pre-checks to fail and continue deployment
-#ignore_assert_errors: false
-
-## Etcd auto compaction retention for mvcc key value store in hour
-#etcd_compaction_retention: 0
-
-## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
-#etcd_metrics: basic
-
-## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.
-## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.
-#etcd_memory_limit: "512M"
-
-# The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable.
-#kube_read_only_port: 10255

inventory/sample/group_vars/all/all.yml

@@ -0,0 +1,85 @@
+## Valid bootstrap options (required): ubuntu, coreos, centos, none
+## If the OS is not listed here, it means it doesn't require extra/bootstrap steps.
+## In example, python is not available on 'coreos' so it must be installed before
+## anything else. In the opposite, Debian has already all its dependencies fullfiled, then bootstrap_os should be set to `none`.
+bootstrap_os: none
+
+## Directory where etcd data stored
+etcd_data_dir: /var/lib/etcd
+
+## Directory where the binaries will be installed
+bin_dir: /usr/local/bin
+
+## The access_ip variable is used to define how other nodes should access
+## the node.  This is used in flannel to allow other flannel nodes to see
+## this node for example.  The access_ip is really useful AWS and Google
+## environments where the nodes are accessed remotely by the "public" ip,
+## but don't know about that address themselves.
+#access_ip: 1.1.1.1
+
+
+## External LB example config
+## apiserver_loadbalancer_domain_name: "elb.some.domain"
+#loadbalancer_apiserver:
+#  address: 1.2.3.4
+#  port: 1234
+
+## Internal loadbalancers for apiservers
+#loadbalancer_apiserver_localhost: true
+
+## Local loadbalancer should use this port instead, if defined.
+## Defaults to kube_apiserver_port (6443)
+#nginx_kube_apiserver_port: 8443
+
+### OTHER OPTIONAL VARIABLES
+## For some things, kubelet needs to load kernel modules.  For example, dynamic kernel services are needed
+## for mounting persistent volumes into containers.  These may not be loaded by preinstall kubernetes
+## processes.  For example, ceph and rbd backed volumes.  Set to true to allow kubelet to load kernel
+## modules.
+#kubelet_load_modules: false
+
+## Upstream dns servers used by dnsmasq
+#upstream_dns_servers:
+#  - 8.8.8.8
+#  - 8.8.4.4
+
+## There are some changes specific to the cloud providers
+## for instance we need to encapsulate packets with some network plugins
+## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external'
+## When openstack is used make sure to source in the openstack credentials
+## like you would do when using nova-client before starting the playbook.
+#cloud_provider:
+
+
+## Uncomment to enable experimental kubeadm deployment mode
+#kubeadm_enabled: false
+
+## Set these proxy values in order to update package manager and docker daemon to use proxies
+#http_proxy: ""
+#https_proxy: ""
+
+## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy
+#no_proxy: ""
+
+## Some problems may occur when downloading files over https proxy due to ansible bug
+## https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable
+## SSL validation of get_url module. Note that kubespray will still be performing checksum validation.
+#download_validate_certs: False
+
+## If you need exclude all cluster nodes from proxy and other resources, add other resources here.
+#additional_no_proxy: ""
+
+## Certificate Management
+## This setting determines whether certs are generated via scripts or whether a
+## cluster of Hashicorp's Vault is started to issue certificates (using etcd
+## as a backend). Options are "script" or "vault"
+#cert_management: script
+
+## Set to true to allow pre-checks to fail and continue deployment
+#ignore_assert_errors: false
+
+## The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable.
+#kube_read_only_port: 10255
+
+## Set true to download and cache container
+#download_container: true

inventory/sample/group_vars/all/azure.yml

@@ -0,0 +1,14 @@
+## When azure is used, you need to also set the following variables.
+## see docs/azure.md for details on how to get these values
+
+#azure_tenant_id:
+#azure_subscription_id:
+#azure_aad_client_id:
+#azure_aad_client_secret:
+#azure_resource_group:
+#azure_location:
+#azure_subnet_name:
+#azure_security_group_name:
+#azure_vnet_name:
+#azure_vnet_resource_group:
+#azure_route_table_name:

inventory/sample/group_vars/all/coreos.yml

@@ -0,0 +1,2 @@
+## Does coreos need auto upgrade, default is true
+#coreos_auto_upgrade: true

inventory/sample/group_vars/all/docker.yml

@@ -0,0 +1,61 @@
+## Uncomment this if you want to force overlay/overlay2 as docker storage driver
+## Please note that overlay2 is only supported on newer kernels
+#docker_storage_options: -s overlay2
+
+## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7.
+docker_container_storage_setup: false
+
+## It must be define a disk path for docker_container_storage_setup_devs.
+## Otherwise docker-storage-setup will be executed incorrectly.
+#docker_container_storage_setup_devs: /dev/vdb
+
+## Uncomment this if you have more than 3 nameservers, then we'll only use the first 3.
+docker_dns_servers_strict: false
+
+# Path used to store Docker data
+docker_daemon_graph: "/var/lib/docker"
+
+## Used to set docker daemon iptables options to true
+docker_iptables_enabled: "false"
+
+# Docker log options
+# Rotate container stderr/stdout logs at 50m and keep last 5
+docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5"
+
+# define docker bin_dir
+docker_bin_dir: "/usr/bin"
+
+## An obvious use case is allowing insecure-registry access to self hosted registries.
+## Can be ipddress and domain_name.
+## example define 172.19.16.11 or mirror.registry.io
+#docker_insecure_registries:
+#   - mirror.registry.io
+#   - 172.19.16.11
+
+## Add other registry,example China registry mirror.
+#docker_registry_mirrors:
+#   - https://registry.docker-cn.com
+#   - https://mirror.aliyuncs.com
+
+## If non-empty will override default system MounFlags value.
+## This option takes a mount propagation flag: shared, slave
+## or private, which control whether mounts in the file system
+## namespace set up for docker will receive or propagate mounts
+## and unmounts. Leave empty for system default
+#docker_mount_flags:
+
+## A string of extra options to pass to the docker daemon.
+## This string should be exactly as you wish it to appear.
+docker_options: >-
+  {%- if docker_insecure_registries is defined -%}
+  {{ docker_insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }}
+  {%- endif %}
+  {% if docker_registry_mirrors is defined -%}
+  {{ docker_registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }}
+  {%- endif %}
+  --graph={{ docker_daemon_graph }} {{ docker_log_opts }}
+  {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %}
+  --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current
+  --default-runtime=docker-runc --exec-opt native.cgroupdriver=systemd
+  --userland-proxy-path=/usr/libexec/docker/docker-proxy-current --signature-verification=false
+  {%- endif -%}

inventory/sample/group_vars/all/oci.yml

@@ -0,0 +1,15 @@
+## When Oracle Cloud Infrastructure is used, set these variables
+#oci_private_key:
+#oci_region_id:
+#oci_tenancy_id:
+#oci_user_id:
+#oci_user_fingerprint:
+#oci_compartment_id:
+#oci_vnc_id:
+#oci_subnet1_id:
+#oci_subnet2_id:
+## Overide these default behaviors if you wish
+#oci_security_list_management: All
+# If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint
+#oci_use_instance_principals: false
+#oci_cloud_controller_version: 0.5.0

inventory/sample/group_vars/all/openstack.yml

@@ -0,0 +1,16 @@
+## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
+#openstack_blockstorage_version: "v1/v2/auto (default)"
+#openstack_blockstorage_ignore_volume_az: yes
+## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
+#openstack_lbaas_enabled: True
+#openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP"
+## To enable automatic floating ip provisioning, specify a subnet.
+#openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default"
+## Override default LBaaS behavior
+#openstack_lbaas_use_octavia: False
+#openstack_lbaas_method: "ROUND_ROBIN"
+#openstack_lbaas_provider: "haproxy"
+#openstack_lbaas_create_monitor: "yes"
+#openstack_lbaas_monitor_delay: "1m"
+#openstack_lbaas_monitor_timeout: "30s"
+#openstack_lbaas_monitor_max_retries: "3"

inventory/sample/group_vars/etcd.yml

@@ -0,0 +1,18 @@
+## Etcd auto compaction retention for mvcc key value store in hour
+#etcd_compaction_retention: 0
+
+## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
+#etcd_metrics: basic
+
+## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.
+## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.
+#etcd_memory_limit: "512M"
+
+## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than
+## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check
+## etcd documentation for more information.
+#etcd_quota_backend_bytes: "2G"
+
+### ETCD: disable peer client cert authentication.
+# This affects ETCD_PEER_CLIENT_CERT_AUTH variable
+#etcd_peer_client_auth: true

inventory/sample/group_vars/k8s-cluster/addons.yml

@@ -0,0 +1,51 @@
+# Kubernetes dashboard
+# RBAC required. see docs/getting-started.md for access details.
+dashboard_enabled: true
+
+# Helm deployment
+helm_enabled: false
+
+# Registry deployment
+registry_enabled: false
+# registry_namespace: kube-system
+# registry_storage_class: ""
+# registry_disk_size: "10Gi"
+
+# Local volume provisioner deployment
+local_volume_provisioner_enabled: false
+# local_volume_provisioner_namespace: kube-system
+# local_volume_provisioner_base_dir: /mnt/disks
+# local_volume_provisioner_mount_dir: /mnt/disks
+# local_volume_provisioner_storage_class: local-storage
+
+# CephFS provisioner deployment
+cephfs_provisioner_enabled: false
+# cephfs_provisioner_namespace: "cephfs-provisioner"
+# cephfs_provisioner_cluster: ceph
+# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# cephfs_provisioner_admin_id: admin
+# cephfs_provisioner_secret: secret
+# cephfs_provisioner_storage_class: cephfs
+# cephfs_provisioner_reclaim_policy: Delete
+# cephfs_provisioner_claim_root: /volumes
+# cephfs_provisioner_deterministic_names: true
+
+# Nginx ingress controller deployment
+ingress_nginx_enabled: false
+# ingress_nginx_host_network: false
+# ingress_nginx_nodeselector:
+#   node-role.kubernetes.io/master: "true"
+# ingress_nginx_namespace: "ingress-nginx"
+# ingress_nginx_insecure_port: 80
+# ingress_nginx_secure_port: 443
+# ingress_nginx_configmap:
+#   map-hash-bucket-size: "128"
+#   ssl-protocols: "SSLv2"
+# ingress_nginx_configmap_tcp_services:
+#   9000: "default/example-go:8080"
+# ingress_nginx_configmap_udp_services:
+#   53: "kube-system/kube-dns:53"
+
+# Cert manager deployment
+cert_manager_enabled: false
+# cert_manager_namespace: "cert-manager"

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -19,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.10.4
+kube_version: v1.11.3
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
@@ -34,9 +34,12 @@ kube_cert_group: kube-cert
 # Cluster Loglevel configuration
 kube_log_level: 2
 
+# Directory where credentials will be stored
+credentials_dir: "{{ inventory_dir }}/credentials"
+
 # Users to create for basic auth in Kubernetes API via HTTP
 # Optionally add groups for user
-kube_api_pwd: "{{ lookup('password', inventory_dir + '/credentials/kube_user.creds length=15 chars=ascii_letters,digits') }}"
+kube_api_pwd: "{{ lookup('password', credentials_dir + '/kube_user.creds length=15 chars=ascii_letters,digits') }}"
 kube_users:
   kube:
     pass: "{{kube_api_pwd}}"
@@ -56,7 +59,7 @@ kube_users:
 # kube_oidc_url: https:// ...
 # kube_oidc_client_id: kubernetes
 ## Optional settings for OIDC
-# kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem
+# kube_oidc_ca_file: "{{ kube_cert_dir }}/ca.pem"
 # kube_oidc_username_claim: sub
 # kube_oidc_username_prefix: oidc:
 # kube_oidc_groups_claim: groups
@@ -67,25 +70,6 @@ kube_users:
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico
 
-# Weave deployment
-# weave_password: ~
-# weave_checkpoint_disable: false
-# weave_conn_limit: 100
-# weave_hairpin_mode: true
-# weave_ipalloc_range: {{ kube_pods_subnet }}
-# weave_expect_npc: {{ enable_network_policy }}
-# weave_kube_peers: ~
-# weave_ipalloc_init: ~
-# weave_expose_ip: ~
-# weave_metrics_addr: ~
-# weave_status_addr: ~
-# weave_mtu: 1376
-# weave_no_masq_local: true
-# weave_extra_args: ~
-
-# Enable kubernetes network policies
-enable_network_policy: false
-
 # Kubernetes internal network for services, unused block of space.
 kube_service_addresses: 10.233.0.0/18
 
@@ -110,6 +94,11 @@ kube_apiserver_insecure_port: 8080 # (http)
 # Can be ipvs, iptables
 kube_proxy_mode: iptables
 
+# Kube-proxy nodeport address.
+# cidr to bind nodeport services. Flag --nodeport-addresses on kube-proxy manifest
+kube_proxy_nodeport_addresses: false
+# kube_proxy_nodeport_addresses_cidr: 10.0.1.0/24
+
 ## Encrypting Secret Data at Rest (experimental)
 kube_encrypt_secret_data: false
 
@@ -133,28 +122,11 @@ skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipad
 dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 
-# Path used to store Docker data
-docker_daemon_graph: "/var/lib/docker"
+## Container runtime
+## docker for docker and crio for cri-o.
+container_manager: docker
 
-## Used to set docker daemon iptables options to true
-#docker_iptables_enabled: "true"
-
-## A string of extra options to pass to the docker daemon.
-## This string should be exactly as you wish it to appear.
-## An obvious use case is allowing insecure-registry access
-## to self hosted registries like so:
-
-docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }}  {{ docker_log_opts }}"
-docker_bin_dir: "/usr/bin"
-
-## If non-empty will override default system MounFlags value.
-## This option takes a mount propagation flag: shared, slave
-## or private, which control whether mounts in the file system
-## namespace set up for docker will receive or propagate mounts
-## and unmounts. Leave empty for system default
-docker_mount_flags:
-
-# Settings for containerized control plane (etcd/kubelet/secrets)
+## Settings for containerized control plane (etcd/kubelet/secrets)
 etcd_deployment_type: docker
 kubelet_deployment_type: host
 vault_deployment_type: docker
@@ -163,63 +135,19 @@ helm_deployment_type: host
 # K8s image pull policy (imagePullPolicy)
 k8s_image_pull_policy: IfNotPresent
 
-# Kubernetes dashboard
-# RBAC required. see docs/getting-started.md for access details.
-dashboard_enabled: true
+# audit log for kubernetes
+kubernetes_audit: false
 
-# Monitoring apps for k8s
-efk_enabled: false
+# dynamic kubelet configuration
+dynamic_kubelet_configuration: false
 
-# Helm deployment
-helm_enabled: false
+# define kubelet config dir for dynamic kubelet
+#kubelet_config_dir:
+default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
+dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kubelet_config_dir) }}"
 
-# Registry deployment
-registry_enabled: false
-# registry_namespace: "{{ system_namespace }}"
-# registry_storage_class: ""
-# registry_disk_size: "10Gi"
-
-# Local volume provisioner deployment
-local_volume_provisioner_enabled: false
-# local_volume_provisioner_namespace: "{{ system_namespace }}"
-# local_volume_provisioner_base_dir: /mnt/disks
-# local_volume_provisioner_mount_dir: /mnt/disks
-# local_volume_provisioner_storage_class: local-storage
-
-# CephFS provisioner deployment
-cephfs_provisioner_enabled: false
-# cephfs_provisioner_namespace: "cephfs-provisioner"
-# cephfs_provisioner_cluster: ceph
-# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
-# cephfs_provisioner_admin_id: admin
-# cephfs_provisioner_secret: secret
-# cephfs_provisioner_storage_class: cephfs
-# cephfs_provisioner_reclaim_policy: Delete
-# cephfs_provisioner_claim_root: /volumes
-# cephfs_provisioner_deterministic_names: true
-
-# Nginx ingress controller deployment
-ingress_nginx_enabled: false
-# ingress_nginx_host_network: false
-# ingress_nginx_nodeselector:
-#   node-role.kubernetes.io/master: "true"
-# ingress_nginx_namespace: "ingress-nginx"
-# ingress_nginx_insecure_port: 80
-# ingress_nginx_secure_port: 443
-# ingress_nginx_configmap:
-#   map-hash-bucket-size: "128"
-#   ssl-protocols: "SSLv2"
-# ingress_nginx_configmap_tcp_services:
-#   9000: "default/example-go:8080"
-# ingress_nginx_configmap_udp_services:
-#   53: "kube-system/kube-dns:53"
-
-# Cert manager deployment
-cert_manager_enabled: false
-# cert_manager_namespace: "cert-manager"
-
-# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now )
-persistent_volumes_enabled: false
+# pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled)
+podsecuritypolicy_enabled: false
 
 # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts
 # kubeconfig_localhost: false
@@ -246,3 +174,18 @@ persistent_volumes_enabled: false
 ## See https://github.com/kubernetes-incubator/kubespray/issues/2141
 ## Set this variable to true to get rid of this issue
 volume_cross_zone_attachment: false
+# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now )
+persistent_volumes_enabled: false
+
+## Container Engine Acceleration
+## Enable container accelertion feature, for example use gpu acceleration in containers
+# nvidia_accelerator_enabled: true
+## Nvidia GPU driver install. Install will by done by a (init) pod running as a daemonset.
+## Important: if you use Ubuntu then you should set in all.yml 'docker_storage_options: -s overlay2'
+## Array with nvida_gpu_nodes, leave empty or comment if you dont't want to install drivers.
+## Labels and taints won't be set to nodes if they are not in the array.
+# nvidia_gpu_nodes:
+#   - kube-gpu-001
+# nvidia_driver_version: "384.111"
+## flavor can be tesla or gtx
+# nvidia_gpu_flavor: gtx

inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml

@@ -0,0 +1,20 @@
+# see roles/network_plugin/calico/defaults/main.yml
+
+## With calico it is possible to distributed routes with border routers of the datacenter.
+## Warning : enabling router peering will disable calico's default behavior ('node mesh').
+## The subnets of each nodes will be distributed by the datacenter router
+#peer_with_router: false
+
+# Enables Internet connectivity from containers
+# nat_outgoing: true
+
+# add default ippool name
+# calico_pool_name: "default-pool"
+
+# Global as_num (/calico/bgp/v1/global/as_num)
+# global_as_num: "64512"
+
+# You can set MTU value here. If left undefined or empty, it will
+# not be specified in calico CNI config, so Calico will use built-in
+# defaults. The value should be a number, not a string.
+# calico_mtu: 1500

inventory/sample/group_vars/k8s-cluster/k8s-net-canal.yml

@@ -0,0 +1,11 @@
+# see roles/network_plugin/canal/defaults/main.yml
+
+# The interface used by canal for host <-> host communication.
+# If left blank, then the interface is chosing using the node's
+# default route.
+# canal_iface: ""
+
+# Whether or not to masquerade traffic to destinations not within
+# the pod network.
+# canal_masquerade: "true"
+

inventory/sample/group_vars/k8s-cluster/k8s-net-cilium.yml

@@ -0,0 +1 @@
+# see roles/network_plugin/cilium/defaults/main.yml

inventory/sample/group_vars/k8s-cluster/k8s-net-contiv.yml

@@ -0,0 +1,20 @@
+# see roles/network_plugin/contiv/defaults/main.yml
+
+# Forwarding mode: bridge or routing
+# contiv_fwd_mode: routing
+
+## With contiv, L3 BGP mode is possible by setting contiv_fwd_mode to "routing".
+## In this case, you may need to peer with an uplink
+## NB: The hostvars must contain a key "contiv" of which value is a dict containing "router_ip", "as"(defaults to contiv_global_as), "neighbor_as" (defaults to contiv_global_neighbor_as), "neighbor"
+#contiv_peer_with_uplink_leaf: false
+#contiv_global_as: "65002"
+#contiv_global_neighbor_as: "500"
+
+# Fabric mode: aci, aci-opflex or default
+# contiv_fabric_mode: default
+
+# Defaut netmode: vxlan or vlan
+# contiv_net_mode: vxlan
+
+# Dataplane interface
+# contiv_vlan_interface: ""

inventory/sample/group_vars/k8s-cluster/k8s-net-flannel.yml

@@ -0,0 +1,16 @@
+# see roles/network_plugin/flannel/defaults/main.yml
+
+## interface that should be used for flannel operations
+## This is actually an inventory cluster-level item
+# flannel_interface:
+
+## Select interface that should be used for flannel operations by regexp on Name or IP
+## This is actually an inventory cluster-level item
+## example: select interface with ip from net 10.0.0.0/23
+## single quote and escape backslashes
+# flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}'
+
+# You can choose what type of flannel backend to use: 'vxlan' or 'host-gw'
+# for experimental backend
+# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md
+# flannel_backend_type: "vxlan"

inventory/sample/group_vars/k8s-cluster/k8s-net-weave.yml

@@ -0,0 +1,58 @@
+# see roles/network_plugin/weave/defaults/main.yml
+
+# Weave's network password for encryption, if null then no network encryption.
+# weave_password: ~
+
+# If set to 1, disable checking for new Weave Net versions (default is blank,
+# i.e. check is enabled)
+# weave_checkpoint_disable: false
+
+# Soft limit on the number of connections between peers. Defaults to 100.
+# weave_conn_limit: 100
+
+# Weave Net defaults to enabling hairpin on the bridge side of the veth pair
+# for containers attached. If you need to disable hairpin, e.g. your kernel is
+# one of those that can panic if hairpin is enabled, then you can disable it by
+# setting `HAIRPIN_MODE=false`.
+# weave_hairpin_mode: true
+
+# The range of IP addresses used by Weave Net and the subnet they are placed in
+# (CIDR format; default 10.32.0.0/12)
+# weave_ipalloc_range: "{{ kube_pods_subnet }}"
+
+# Set to 0 to disable Network Policy Controller (default is on)
+# weave_expect_npc: "{{ enable_network_policy }}"
+
+# List of addresses of peers in the Kubernetes cluster (default is to fetch the
+# list from the api-server)
+# weave_kube_peers: ~
+
+# Set the initialization mode of the IP Address Manager (defaults to consensus
+# amongst the KUBE_PEERS)
+# weave_ipalloc_init: ~
+
+# Set the IP address used as a gateway from the Weave network to the host
+# network - this is useful if you are configuring the addon as a static pod.
+# weave_expose_ip: ~
+
+# Address and port that the Weave Net daemon will serve Prometheus-style
+# metrics on (defaults to 0.0.0.0:6782)
+# weave_metrics_addr: ~
+
+# Address and port that the Weave Net daemon will serve status requests on
+# (defaults to disabled)
+# weave_status_addr: ~
+
+# Weave Net defaults to 1376 bytes, but you can set a smaller size if your
+# underlying network has a tighter limit, or set a larger size for better
+# performance if your network supports jumbo frames (e.g. 8916)
+# weave_mtu: 1376
+
+# Set to 1 to preserve the client source IP address when accessing Service
+# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works
+# only with Weave IPAM (default).
+# weave_no_masq_local: true
+
+# Extra variables that passing to launch.sh, useful for enabling seed mode, see
+# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/
+# weave_extra_args: ~

inventory/sample/hosts.ini

@@ -1,11 +1,12 @@
 # ## Configure 'ip' variable to bind kubernetes services on a
 # ## different ip than the default iface
-# node1 ansible_host=95.54.0.12  # ip=10.3.0.1
-# node2 ansible_host=95.54.0.13  # ip=10.3.0.2
-# node3 ansible_host=95.54.0.14  # ip=10.3.0.3
-# node4 ansible_host=95.54.0.15  # ip=10.3.0.4
-# node5 ansible_host=95.54.0.16  # ip=10.3.0.5
-# node6 ansible_host=95.54.0.17  # ip=10.3.0.6
+# ## We should set etcd_member_name for etcd cluster. The node that is not a etcd member do not need to set the value, or can set the empty string value.
+# node1 ansible_host=95.54.0.12  # ip=10.3.0.1 etcd_member_name=etcd1
+# node2 ansible_host=95.54.0.13  # ip=10.3.0.2 etcd_member_name=etcd2
+# node3 ansible_host=95.54.0.14  # ip=10.3.0.3 etcd_member_name=etcd3
+# node4 ansible_host=95.54.0.15  # ip=10.3.0.4 etcd_member_name=etcd4
+# node5 ansible_host=95.54.0.16  # ip=10.3.0.5 etcd_member_name=etcd5
+# node6 ansible_host=95.54.0.17  # ip=10.3.0.6 etcd_member_name=etcd6
 
 # ## configure a bastion host if your nodes are not directly reachable
 # bastion ansible_host=x.x.x.x ansible_user=some_user
@@ -26,6 +27,6 @@
 # node5
 # node6
 
-# [k8s-cluster:children]
-# kube-master
-# kube-node
+[k8s-cluster:children]
+kube-master
+kube-node

roles/adduser/tasks/main.yml

@@ -7,7 +7,7 @@
 - name: User | Create User
   user:
     comment: "{{user.comment|default(omit)}}"
-    createhome: "{{user.create_home|default(omit)}}"
+    createhome: "{{user.createhome|default(omit)}}"
     group: "{{user.group|default(user.name)}}"
     home: "{{user.home|default(omit)}}"
     shell: "{{user.shell|default(omit)}}"

roles/bootstrap-os/defaults/main.yml

@@ -4,3 +4,6 @@ pip_python_coreos_modules:
   - six
 
 override_system_hostname: true
+
+
+coreos_auto_upgrade: true

roles/bootstrap-os/tasks/bootstrap-centos.yml

@@ -16,5 +16,14 @@
 
 - name: Install packages requirements for bootstrap
   yum:
-    name: libselinux-python
+    name: "{{ packages }}"
+    state: present
+  vars:
+    packages:
+      - libselinux-python
+      - epel-release
+
+- name: Install pip for bootstrap
+  yum:
+    name: python-pip
     state: present

roles/bootstrap-os/tasks/bootstrap-coreos.yml

@@ -62,3 +62,8 @@
   with_items: "{{pip_python_coreos_modules}}"
   environment:
     PATH: "{{ ansible_env.PATH }}:{{ bin_dir }}"
+
+- name: Bootstrap | Disable auto-upgrade
+  shell: "systemctl stop locksmithd.service && systemctl mask --now locksmithd.service"
+  when:
+    - not coreos_auto_upgrade

roles/bootstrap-os/tasks/bootstrap-fedora.yml

@@ -0,0 +1,19 @@
+---
+
+- name: Bootstrap | Check if bootstrap is needed
+  raw: which "{{ item }}"
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  with_items:
+    - python
+  tags: facts
+
+- name: Install python on fedora
+  raw: "dnf install --assumeyes --quiet python"
+  when: "{{ need_bootstrap.results | map(attribute='rc') | sort | last | bool }}"
+
+- name: Install required python packages
+  dnf:
+    name: libselinux-python
+    state: present

roles/bootstrap-os/tasks/main.yml

@@ -8,6 +8,9 @@
 - import_tasks: bootstrap-coreos.yml
   when: bootstrap_os == "coreos"
 
+- import_tasks: bootstrap-fedora.yml
+  when: bootstrap_os == "fedora"
+
 - import_tasks: bootstrap-centos.yml
   when: bootstrap_os == "centos"
 

roles/cri-o/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+crio_rhel_repo_base_url: 'https://cbs.centos.org/repos/paas7-openshift-origin311-candidate/x86_64/os/'