Pull in downstream k8s installer changes

- Secretification of secret stuff
- Backup / restore

installer/roles/kubernetes/tasks/backup.yml

@@ -0,0 +1,80 @@
+---
+- name: Determine the timestamp for the backup.
+  set_fact:
+    now: '{{ lookup("pipe", "date +%F-%T") }}'
+
+- include_tasks: openshift_auth.yml
+  when: openshift_host is defined
+
+- include_tasks: kubernetes_auth.yml
+  when: kubernetes_context is defined
+
+- name: Use kubectl or oc
+  set_fact:
+    kubectl_or_oc: "{{ openshift_oc_bin if openshift_oc_bin is defined else 'kubectl' }}"
+
+- name: Delete any existing management pod
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      delete pod ansible-tower-management --grace-period=0 --ignore-not-found
+
+- name: Template management pod
+  set_fact:
+    management_pod: "{{ lookup('template', 'management-pod.yml.j2') }}"
+
+- name: Create management pod
+  shell: |
+    echo {{ management_pod | quote }} | {{ kubectl_or_oc }} apply -f -
+
+- name: Wait for management pod to start
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      get pod ansible-tower-management -o jsonpath="{.status.phase}"
+  register: result
+  until: result.stdout == "Running"
+  retries: 60
+
+- name: Create directory for backup
+  file:
+    state: directory
+    path: "{{ playbook_dir }}/tower-openshift-backup-{{ now }}"
+
+- name: Precreate file for database dump
+  file:
+    path: "{{ playbook_dir }}/tower-openshift-backup-{{ now }}/tower.db"
+    state: touch
+    mode: 0600
+
+- name: Dump database
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} exec ansible-tower-management -- \
+      bash -c "PGPASSWORD={{ pg_password }} \
+        pg_dump --clean --create \
+          --host='{{ pg_hostname | default('postgresql') }}' \
+          --port={{ pg_port | default('5432') }} \
+          --username='{{ pg_username }}' \
+          --dbname='{{ pg_database }}'" > {{ playbook_dir }}/tower-openshift-backup-{{ now }}/tower.db
+
+- name: Copy inventory into backup directory
+  copy:
+    src: "{{ inventory_file }}"
+    dest: "{{ playbook_dir }}/tower-openshift-backup-{{ now }}/"
+    mode: 0600
+
+- name: Delete management pod
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      delete pod ansible-tower-management --grace-period=0 --ignore-not-found
+
+- name: Create backup archive
+  archive:
+    path: "{{ playbook_dir }}/tower-openshift-backup-{{ now }}"
+    dest: "{{ item }}"
+  with_items:
+    - "{{ playbook_dir }}/tower-openshift-backup-{{ now }}.tar.gz"
+    - "{{ playbook_dir }}/tower-openshift-backup-latest.tar.gz"
+
+- name: Remove temporary backup directory
+  file:
+    path: "{{ playbook_dir }}/tower-openshift-backup-{{ now }}"
+    state: absent

installer/roles/kubernetes/tasks/kubernetes.yml

@@ -1,6 +1,4 @@
-- name: Set the Kubernetes Context
-  shell: "kubectl config use-context {{ kubernetes_context }}"
-
+---
 - name: Get Namespace Detail
   shell: "kubectl get namespace {{ kubernetes_namespace }}"
   register: namespace_details
@@ -14,3 +12,4 @@
   set_fact:
     postgresql_service_name: "{{ kubernetes_deployment_name }}-postgresql"
   when: "pg_hostname is not defined or pg_hostname == ''"
+

installer/roles/kubernetes/tasks/kubernetes_auth.yml

@@ -0,0 +1,3 @@
+---
+- name: Set the Kubernetes Context
+  shell: "kubectl config set-context {{ kubernetes_context }}"

installer/roles/kubernetes/tasks/main.yml

@@ -3,20 +3,48 @@
     msg: "Only set one of kubernetes_context or openshift_host"
   when: openshift_host is defined and kubernetes_context is defined
 
-- name: Set kubernetes base path
-  set_fact:
-    kubernetes_base_path: "{{ local_base_config_path|default('/tmp') }}/{{ kubernetes_deployment_name }}-config"
-
-- include_tasks: openshift.yml
+- include_tasks: "{{ tasks }}"
+  with_items:
+    - openshift_auth.yml
+    - openshift.yml
+  loop_control:
+    loop_var: tasks
   when: openshift_host is defined
 
-- include_tasks: kubernetes.yml
+- include_tasks: "{{ tasks }}"
+  with_items:
+    - kubernetes_auth.yml
+    - kubernetes.yml
+  loop_control:
+    loop_var: tasks
   when: kubernetes_context is defined
 
 - name: Use kubectl or oc
   set_fact:
     kubectl_or_oc: "{{ openshift_oc_bin if openshift_oc_bin is defined else 'kubectl' }}"
 
+- set_fact:
+    deployment_object: "{{ 'dc' if openshift_host is defined else 'deployment' }}"
+
+- name: Record deployment size
+  shell: |
+    {{ kubectl_or_oc }} get {{ deployment_object }} \
+      {{ kubernetes_deployment_name }} \
+      -n {{ kubernetes_namespace }} -o=jsonpath='{.status.replicas}'
+  register: deployment_details
+  ignore_errors: yes
+
+- name: Set expected post-deployment Replicas value
+  set_fact:
+    kubernetes_deployment_replica_size: "{{ deployment_details.stdout | int }}"
+  when: deployment_details.rc == 0
+
+- name: Delete existing Deployment
+  shell: |
+    {{ kubectl_or_oc }} delete {{ deployment_object }} \
+      {{ kubernetes_deployment_name }} -n {{ kubernetes_namespace }}
+  when: deployment_details.rc == 0
+
 - name: Get Postgres Service Detail
   shell: "{{ kubectl_or_oc }} describe svc {{ postgresql_service_name }} -n {{ kubernetes_namespace }}"
   register: postgres_svc_details
@@ -39,10 +67,12 @@
           -e POSTGRESQL_MAX_CONNECTIONS={{ pg_max_connections|default(1024) }} \
           -e POSTGRESQL_USER={{ pg_username }} \
           -e POSTGRESQL_PASSWORD={{ pg_password }} \
+          -e POSTGRESQL_ADMIN_PASSWORD={{ pg_password }} \
           -e POSTGRESQL_DATABASE={{ pg_database }} \
           -e POSTGRESQL_VERSION=9.5 \
           -n {{ kubernetes_namespace }}
       register: openshift_pg_activate
+      no_log: yes
   when:
     - pg_hostname is not defined or pg_hostname == ''
     - postgres_svc_details is defined and postgres_svc_details.rc != 0
@@ -51,7 +81,6 @@
 - name: Deploy and Activate Postgres (Kubernetes)
   shell: |
     helm install --name {{ kubernetes_deployment_name }} --namespace {{ kubernetes_namespace }} \
-      --tiller-namespace {{ tiller_namespace }} \
       --set postgresUser={{ pg_username }} \
       --set postgresPassword={{ pg_password }} \
       --set postgresDatabase={{ pg_database }} \
@@ -62,6 +91,7 @@
     - postgres_svc_details is defined and postgres_svc_details.rc != 0
     - kubernetes_context is defined
   register: kubernetes_pg_activate
+  no_log: yes
 
 - name: Set postgresql hostname to helm package service
   set_fact:
@@ -75,17 +105,6 @@
     seconds: 60
   when: openshift_pg_activate.changed or kubernetes_pg_activate.changed
 
-- name: Ensure directory exists
-  file:
-    path: "{{ kubernetes_base_path }}"
-    state: directory
-
-- name: Template Kubernetes AWX Config
-  template:
-    src: configmap.yml.j2
-    dest: "{{ kubernetes_base_path }}/configmap.yml"
-    mode: '0600'
-
 - name: Set image names if using custom registry
   block:
     - name: Set task image name
@@ -99,14 +118,88 @@
       when: kubernetes_web_image is not defined
   when: docker_registry is defined
 
-- name: Template Kubernetes AWX Deployment
-  template:
-    src: deployment.yml.j2
-    dest: "{{ kubernetes_base_path }}/deployment.yml"
-    mode: '0600'
-
-- name: Apply Configmap
-  shell: "{{ kubectl_or_oc }} apply -f {{ kubernetes_base_path }}/configmap.yml"
+- name: Render deployment templates
+  set_fact:
+    "{{ item }}": "{{ lookup('template', item + '.yml.j2') }}"
+  with_items:
+    - 'configmap'
+    - 'deployment'
+    - 'secret'
+  no_log: yes
 
 - name: Apply Deployment
-  shell: "{{ kubectl_or_oc }} apply -f {{ kubernetes_base_path }}/deployment.yml"
+  shell: |
+    echo {{ item | quote }} | {{ kubectl_or_oc }} apply -f -
+  with_items:
+    - "{{ configmap }}"
+    - "{{ deployment }}"
+    - "{{ secret }}"
+  no_log: yes
+
+- name: Delete any existing management pod
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      delete pod ansible-tower-management --grace-period=0 --ignore-not-found
+
+- name: Template management pod
+  set_fact:
+    management_pod: "{{ lookup('template', 'management-pod.yml.j2') }}"
+
+- name: Create management pod
+  shell: |
+    echo {{ management_pod | quote }} | {{ kubectl_or_oc }} apply -f -
+
+- name: Wait for management pod to start
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      get pod ansible-tower-management -o jsonpath="{.status.phase}"
+  register: result
+  until: result.stdout == "Running"
+  retries: 60
+
+- name: Migrate database
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} exec ansible-tower-management -- \
+      bash -c "awx-manage migrate --noinput"
+
+- name: Check for Tower Super users
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} exec ansible-tower-management -- \
+      bash -c "echo 'from django.contrib.auth.models import User; nsu = User.objects.filter(is_superuser=True).count(); exit(0 if nsu > 0 else 1)' | awx-manage shell"
+  register: super_check
+  ignore_errors: yes
+  changed_when: super_check.rc > 0
+
+- name: create django super user if it does not exist
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} exec ansible-tower-management -- \
+      bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('{{ admin_user }}', '{{ admin_email }}', '{{ admin_password }}')\" | awx-manage shell"
+  no_log: yes
+  when: super_check.rc > 0
+
+- name: update django super user password
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} exec ansible-tower-management -- \
+      bash -c "awx-manage update_password --username='{{ admin_user }}' --password='{{ admin_password }}'"
+  no_log: yes
+  register: result
+  changed_when: "'Password updated' in result.stdout"
+
+- name: Create the default organization if it is needed.
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} exec ansible-tower-management -- \
+      bash -c "awx-manage create_preload_data"
+  register: cdo
+  changed_when: "'added' in cdo.stdout"
+
+- name: Delete management pod
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      delete pod ansible-tower-management --grace-period=0 --ignore-not-found
+
+- name: Scale up deployment
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      scale {{ deployment_object }} {{ kubernetes_deployment_name }} --replicas=0
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      scale {{ deployment_object }} {{ kubernetes_deployment_name }} --replicas={{ kubernetes_deployment_replica_size }}

installer/roles/kubernetes/tasks/openshift.yml

@@ -1,53 +1,4 @@
 ---
-- include_vars: openshift.yml
-
-- name: Set kubernetes_namespace
-  set_fact:
-    kubernetes_namespace: "{{ openshift_project }}"
-
-- name: Ensure workspace directories exist
-  file:
-    path: "{{ item  }}"
-    state: directory
-  with_items:
-    - "{{ kubernetes_base_path }}"
-    - "{{ openshift_oc_config_file | dirname }}"
-
-- name: Authenticate with OpenShift via user and password
-  shell: |
-    {{ openshift_oc_bin }} login {{ openshift_host }} \
-      -u {{ openshift_user }} \
-      -p {{ openshift_password }} \
-      --insecure-skip-tls-verify={{ openshift_skip_tls_verify | default(false) | bool }}
-  when:
-    - openshift_user is defined
-    - openshift_password is defined
-    - openshift_token is not defined
-  register: openshift_auth_result
-  ignore_errors: true
-  no_log: true
-
-- name: OpenShift authentication failed on TLS verification
-  fail:
-    msg: "Failed to verify TLS, consider settings openshift_skip_tls_verify=True {{ openshift_auth_result.stderr }}"
-  when:
-    - openshift_skip_tls_verify is not defined or not openshift_skip_tls_verify
-    - openshift_auth_result.rc != 0
-    - openshift_auth_result.stderr | search("certificate that does not match its hostname")
-
-- name: OpenShift authentication failed
-  fail:
-    msg: "{{ openshift_auth_result.stderr }}"
-  when: openshift_auth_result.rc != 0
-
-- name: Authenticate with OpenShift via token
-  shell: |
-    {{ openshift_oc_bin }} login {{ openshift_host }} \
-      --token {{ openshift_token }} \
-      --insecure-skip-tls-verify={{ openshift_skip_tls_verify | default(false) | bool }}
-  when: openshift_token is defined
-  no_log: true
-
 - name: Get Project Detail
   shell: "{{ openshift_oc_bin }} get project {{ openshift_project }}"
   register: project_details
@@ -77,3 +28,7 @@
   set_fact:
     postgresql_service_name: "postgresql"
   when: "pg_hostname is not defined or pg_hostname == ''"
+
+- name: Add privileged SCC to service account
+  shell: |
+    {{ openshift_oc_bin }} adm policy add-scc-to-user privileged system:serviceaccount:{{ openshift_project }}:awx

installer/roles/kubernetes/tasks/openshift_auth.yml

@@ -0,0 +1,34 @@
+---
+- include_vars: openshift.yml
+
+- name: Set kubernetes_namespace
+  set_fact:
+    kubernetes_namespace: "{{ openshift_project }}"
+
+- name: Ensure workspace directories exist
+  file:
+    path: "{{ item  }}"
+    state: directory
+  with_items:
+    - "{{ kubernetes_base_path }}"
+    - "{{ openshift_oc_config_file | dirname }}"
+
+- name: Authenticate with OpenShift via user and password
+  shell: |
+    {{ openshift_oc_bin }} login {{ openshift_host }} \
+      -u {{ openshift_user }} \
+      -p {{ openshift_password }} \
+      --insecure-skip-tls-verify={{ openshift_skip_tls_verify | default(false) | bool }}
+  when:
+    - openshift_user is defined
+    - openshift_password is defined
+    - openshift_token is not defined
+  no_log: true
+
+- name: Authenticate with OpenShift via token
+  shell: |
+    {{ openshift_oc_bin }} login {{ openshift_host }} \
+      --token {{ openshift_token }} \
+      --insecure-skip-tls-verify={{ openshift_skip_tls_verify | default(false) | bool }}
+  when: openshift_token is defined
+  no_log: true

installer/roles/kubernetes/tasks/restore.yml

@@ -0,0 +1,123 @@
+---
+- include_tasks: openshift_auth.yml
+  when: openshift_host is defined
+
+- include_tasks: kubernetes_auth.yml
+  when: kubernetes_context is defined
+
+- name: Use kubectl or oc
+  set_fact:
+    kubectl_or_oc: "{{ openshift_oc_bin if openshift_oc_bin is defined else 'kubectl' }}"
+
+- name: Remove any present restore directories
+  file:
+    state: absent
+    path: "{{ playbook_dir }}/tower-openshift-restore"
+
+- name: Create directory for restore data
+  file:
+    state: directory
+    path: "{{ playbook_dir }}/tower-openshift-restore"
+
+- name: Unarchive Tower backup
+  unarchive:
+    src: tower-openshift-backup-latest.tar.gz
+    dest: "{{ playbook_dir }}/tower-openshift-restore"
+    extra_opts: [--strip-components=1]
+
+- set_fact:
+    deployment_object: "{{ 'dc' if openshift_host is defined else 'deployment' }}"
+
+- name: Record deployment size
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      get {{ deployment_object }} {{ kubernetes_deployment_name }} -o jsonpath="{.status.replicas}"
+  register: deployment_size
+
+- name: Scale deployment down
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      scale {{ deployment_object }} {{ kubernetes_deployment_name }} --replicas=0
+
+- name: Delete any existing management pod
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      delete pod ansible-tower-management --grace-period=0 --ignore-not-found
+
+- name: Wait for scale down
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} get pods \
+      -o jsonpath='{.items[*].metadata.name}' \
+      | tr -s '[[:space:]]' '\n' \
+      | grep {{ kubernetes_deployment_name }} \
+      | grep -v postgres | wc -l
+  register: tower_pods
+  until: (tower_pods.stdout | trim) == '0'
+  retries: 30
+
+- name: Template management pod
+  set_fact:
+    management_pod: "{{ lookup('template', 'management-pod.yml.j2') }}"
+
+- name: Create management pod
+  shell: |
+    echo {{ management_pod | quote }} | {{ kubectl_or_oc }} apply -f -
+
+- name: Wait for management pod to start
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      get pod ansible-tower-management -o jsonpath="{.status.phase}"
+  register: result
+  until: result.stdout == "Running"
+  retries: 60
+
+- name: Temporarily grant createdb role
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      exec -i ansible-tower-management -- bash -c "PGPASSWORD={{ pg_password }} \
+        psql \
+        --host={{ pg_hostname | default('postgresql') }} \
+        --port={{ pg_port | default('5432') }} \
+        --username=postgres \
+        --dbname=template1 -c 'ALTER USER tower CREATEDB;'"
+  no_log: true
+  when: pg_hostname is not defined or pg_hostname == ''
+
+- name: Perform a PostgreSQL restore
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      exec -i ansible-tower-management -- bash -c "PGPASSWORD={{ pg_password }} \
+        psql \
+        --host={{ pg_hostname | default('postgresql') }} \
+        --port={{ pg_port | default('5432') }} \
+        --username={{ pg_username }} \
+        --dbname=template1" < {{ playbook_dir }}/tower-openshift-restore/tower.db
+  no_log: true
+
+- name: Revoke createdb role
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      exec -i ansible-tower-management -- bash -c "PGPASSWORD={{ pg_password }} \
+        psql \
+        --host={{ pg_hostname | default('postgresql') }} \
+        --port={{ pg_port | default('5432') }} \
+        --username=postgres \
+        --dbname=template1 -c 'ALTER USER tower NOCREATEDB;'"
+  no_log: true
+  when: pg_hostname is not defined or pg_hostname == ''
+
+- name: Delete management pod
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      delete pod ansible-tower-management --grace-period=0 --ignore-not-found
+
+- name: Remove restore directory
+  file:
+    state: absent
+    path: "{{ playbook_dir }}/tower-openshift-restore"
+
+- name: Scale deployment back up
+  shell: |
+    {{ kubectl_or_oc }} -n {{ kubernetes_namespace }} \
+      scale {{ deployment_object }} {{ kubernetes_deployment_name }} --replicas={{ deployment_size.stdout }}
+  when: deployment_size.stdout != ''