refactor vault role (#2733)

* Move front-proxy-client certs back to kube mount

We want the same CA for all k8s certs

* Refactor vault to use a third party module

The module adds idempotency and reduces some of the repetitive
logic in the vault role

Requires ansible-modules-hashivault on ansible node and hvac
on the vault hosts themselves

Add upgrade test scenario
Remove bootstrap-os tags from tasks

* fix upgrade issues

* improve unseal logic

* specify ca and fix etcd check

* Fix initialization check

bump machine size

roles/vault/tasks/shared/check_vault.yml

@@ -8,24 +8,42 @@
 
 # Check if vault is reachable on the localhost
 - name: check_vault | Attempt to pull local https Vault health
-  uri:
-    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
-    headers: "{{ vault_client_headers }}"
-    status_code: 200,429,500,501,503
-    validate_certs: no
-  ignore_errors: true
-  register: vault_local_service_health
+  command: /bin/true
+  notify: wait for vault up nowait
+
+- meta: flush_handlers
 
 - name: check_vault | Set facts about local Vault health
   set_fact:
-    vault_is_running: "{{ vault_local_service_health|succeeded }}"
-    vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}"
-    vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}"
-    # vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}"
+    vault_is_running: "{{ vault_health_check.get('status', '-1') in vault_successful_http_codes }}"
+
+- name: check_vault | Set facts about local Vault health
+  set_fact:
+    vault_is_initialized: "{{ vault_health_check.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_health_check.get('json', {}).get('sealed', true) }}"
+    # vault_in_standby: "{{ vault_health_check.get('json', {}).get('standby', true) }}"
     # vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}"
 
+- name: check_vault | Check is vault is initialized in etcd if vault is not running
+  command: |-
+    curl \
+      --cacert {{ etcd_cert_dir }}/ca.pem \
+      --cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \
+      --key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
+      -X POST -d '{"key": "{{ "/vault/core/seal-config" | b64encode }}"}' \
+      {{ etcd_access_addresses.split(',') | first }}/v3alpha/kv/range
+  register: vault_etcd_exists
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  run_once: true
+  when: not vault_is_running and vault_etcd_available
+  changed_when: false
+
 - name: check_vault | Set fact about the Vault cluster's initialization state
   set_fact:
-    vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}"
+    vault_cluster_is_initialized: >-
+      {{ vault_is_initialized or
+         hostvars[item]['vault_is_initialized'] or
+         'Key not found' not in vault_etcd_exists.stdout|default('Key not found') }}
   with_items: "{{ groups.vault }}"
   run_once: true