refactor vault role (#2733)

* Move front-proxy-client certs back to kube mount

We want the same CA for all k8s certs

* Refactor vault to use a third party module

The module adds idempotency and reduces some of the repetitive
logic in the vault role

Requires ansible-modules-hashivault on ansible node and hvac
on the vault hosts themselves

Add upgrade test scenario
Remove bootstrap-os tags from tasks

* fix upgrade issues

* improve unseal logic

* specify ca and fix etcd check

* Fix initialization check

bump machine size

roles/vault/tasks/cluster/binary.yml

@@ -3,7 +3,7 @@
 - name: cluster/binary | Copy vault binary from downloaddir
   copy:
     src: "{{ local_release_dir }}/vault/vault"
-    dest: "/usr/bin/vault"
+    dest: "{{ bin_dir }}/vault"
     remote_src: true
     mode: "0755"
     owner: vault

roles/vault/tasks/cluster/configure.yml

@@ -1,10 +1,19 @@
 ---
-
-- name: cluster/configure | Ensure the vault/config directory exists
+- name: cluster/configure | Ensure the vault directories exist
   file:
-    dest: "{{ vault_config_dir }}"
+    dest: "{{ item }}"
+    owner: vault
     mode: 0750
     state: directory
+    recurse: true
+  with_items:
+    - "{{ vault_base_dir }}"
+    - "{{ vault_cert_dir }}"
+    - "{{ vault_config_dir }}"
+    - "{{ vault_roles_dir }}"
+    - "{{ vault_secrets_dir }}"
+    - "{{ vault_log_dir }}"
+    - "{{ vault_lib_dir }}"
 
 - name: cluster/configure | Lay down the configuration file
   copy:

roles/vault/tasks/cluster/create_mounts.yml

@@ -1,14 +1,13 @@
 ---
 - include_tasks: ../shared/create_mount.yml
   vars:
-    create_mount_path: "{{ item.name }}"
+    create_mount_path: "/{{ item.name }}"
     create_mount_default_lease_ttl: "{{ item.default_lease_ttl }}"
     create_mount_max_lease_ttl: "{{ item.max_lease_ttl }}"
     create_mount_description: "{{ item.description }}"
     create_mount_cert_dir: "{{ item.cert_dir }}"
-    create_mount_config_ca_needed: item.name != vault_pki_mounts.kube.name and item.name != vault_pki_mounts.front_proxy.name
+    create_mount_config_ca_needed: item.name != vault_pki_mounts.kube.name
   with_items:
     - "{{ vault_pki_mounts.vault }}"
     - "{{ vault_pki_mounts.etcd }}"
     - "{{ vault_pki_mounts.kube }}"
-    - "{{ vault_pki_mounts.front_proxy }}"

roles/vault/tasks/cluster/create_roles.yml

@@ -6,5 +6,5 @@
     create_role_password: "{{ item.password }}"
     create_role_policy_rules: "{{ item.policy_rules }}"
     create_role_options: "{{ item.role_options }}"
-    create_role_mount_path: "{{ mount.name }}"
+    create_role_mount_path: "/{{ mount.name }}"
   with_items: "{{ mount.roles }}"

roles/vault/tasks/cluster/init.yml

@@ -1,36 +1,28 @@
 ---
 
+- name: cluster/init | wait for vault
+  command: /bin/true
+  notify: wait for vault up
+
+- meta: flush_handlers
+
 - name: cluster/init | Initialize Vault
-  uri:
-    url: "https://{{ groups.vault|first }}:{{ vault_port }}/v1/sys/init"
-    headers: "{{ vault_client_headers }}"
-    method: POST
-    body_format: json
-    body:
-      secret_shares: "{{ vault_secret_shares }}"
-      secret_threshold: "{{ vault_secret_threshold }}"
-    validate_certs: false
+  hashivault_init:
+    url: "https://localhost:{{ vault_port }}/"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret_shares: "{{ vault_secret_shares }}"
+    secret_threshold: "{{ vault_secret_threshold }}"
+  run_once: true
   register: vault_init_result
-  when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first
+  when: not vault_cluster_is_initialized
 
 - name: cluster/init | Set facts on the results of the initialization
   set_fact:
-    vault_unseal_keys: "{{ vault_init_result.json['keys'] }}"
-    vault_root_token: "{{ vault_init_result.json.root_token }}"
-    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.json.root_token}) }}"
-  when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first
-
-- name: cluster/init | Ensure all hosts have these facts
-  set_fact:
-    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys'] }}"
-    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token'] }}"
-  when: not vault_cluster_is_initialized and inventory_hostname != groups.vault|first
-
-- name: cluster/init | Ensure the vault_secrets_dir exists
-  file:
-    mode: 0750
-    path: "{{ vault_secrets_dir }}"
-    state: directory
+    vault_unseal_keys: "{{ vault_init_result.keys_base64 }}"
+    vault_root_token: "{{ vault_init_result.root_token }}"
+    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.root_token}) }}"
+  run_once: true
+  when: not vault_cluster_is_initialized
 
 - name: cluster/init | Ensure all in groups.vault have the unseal_keys locally
   copy:
@@ -48,5 +40,5 @@
 
 - name: cluster/init | Ensure vault_headers and vault statuses are updated
   set_fact:
-    vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token})}}"
     vault_cluster_is_initialized: true
+  run_once: true

roles/vault/tasks/cluster/main.yml

@@ -1,8 +1,8 @@
 ---
-- import_tasks: ../shared/check_vault.yml
+- import_tasks: ../shared/check_etcd.yml
   when: inventory_hostname in groups.vault
 
-- import_tasks: ../shared/check_etcd.yml
+- import_tasks: ../shared/check_vault.yml
   when: inventory_hostname in groups.vault
 
 - import_tasks: configure.yml
@@ -14,6 +14,9 @@
 - import_tasks: systemd.yml
   when: inventory_hostname in groups.vault
 
+- import_tasks: ../shared/find_leader.yml
+  when: inventory_hostname in groups.vault
+
 - import_tasks: init.yml
   when: inventory_hostname in groups.vault
 
@@ -29,20 +32,12 @@
 - include_tasks: ../shared/gen_ca.yml
   vars:
     gen_ca_cert_dir: "{{ vault_pki_mounts.kube.cert_dir }}"
-    gen_ca_mount_path: "{{ vault_pki_mounts.kube.name }}"
+    gen_ca_mount_path: "/{{ vault_pki_mounts.kube.name }}"
     gen_ca_vault_headers: "{{ vault_headers }}"
     gen_ca_vault_options: "{{ vault_ca_options.kube }}"
     gen_ca_copy_group: "kube-master"
   when: inventory_hostname in groups.vault
 
-- include_tasks: ../shared/gen_ca.yml
-  vars:
-    gen_ca_cert_dir: "{{ vault_pki_mounts.front_proxy.cert_dir }}"
-    gen_ca_mount_path: "{{ vault_pki_mounts.front_proxy.name }}"
-    gen_ca_vault_headers: "{{ vault_headers }}"
-    gen_ca_vault_options: "{{ vault_ca_options.front_proxy }}"
-  when: inventory_hostname in groups.vault
-
 - include_tasks: ../shared/auth_backend.yml
   vars:
     auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
@@ -55,7 +50,6 @@
     - "{{ vault_pki_mounts.vault }}"
     - "{{ vault_pki_mounts.etcd }}"
     - "{{ vault_pki_mounts.kube }}"
-    - "{{ vault_pki_mounts.front_proxy }}"
   loop_control:
     loop_var: mount
   when: inventory_hostname in groups.vault

roles/vault/tasks/cluster/systemd.yml

@@ -1,32 +1,11 @@
 ---
-
-- name: cluster/systemd | Ensure mount points exist prior to vault.service startup
-  file:
-    mode: 0750
-    path: "{{ item }}"
-    state: directory
-  with_items:
-    - "{{ vault_config_dir }}"
-    - "{{ vault_log_dir }}"
-    - "{{ vault_secrets_dir }}"
-    - /var/lib/vault/
-
-- name: cluster/systemd | Ensure the vault user has access to needed directories
-  file:
-    owner: vault
-    path: "{{ item }}"
-    recurse: true
-  with_items:
-    - "{{ vault_base_dir }}"
-    - "{{ vault_log_dir }}"
-    - /var/lib/vault
-
 - name: cluster/systemd | Copy down vault.service systemd file
   template:
     src: "{{ vault_deployment_type }}.service.j2"
     dest: /etc/systemd/system/vault.service
     backup: yes
   register: vault_systemd_placement
+  notify: restart vault
 
 - name: Create vault service systemd directory
   file:
@@ -39,6 +18,7 @@
     dest: /etc/systemd/system/vault.service.d/http-proxy.conf
     backup: yes
   when: http_proxy is defined or https_proxy is defined
+  notify: restart vault
 
 - name: cluster/systemd | Enable vault.service
   systemd:
@@ -46,13 +26,4 @@
     enabled: yes
     name: vault
     state: started
-
-- name: cluster/systemd | Query local vault until service is up
-  uri:
-    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
-    headers: "{{ vault_client_headers }}"
-    status_code: 200,429,500,501
-  register: vault_health_check
-  until: vault_health_check|succeeded
-  retries: 10
-  delay: "{{ retry_stagger | random + 3 }}"
+  notify: wait for vault up

roles/vault/tasks/cluster/unseal.yml

@@ -1,25 +1,16 @@
 ---
 
 - name: cluster/unseal | Current sealed state
-  debug: " Sealed? {{vault_is_sealed}}"
+  debug:
+    msg: "Sealed? {{ vault_is_sealed }}"
 
 - name: cluster/unseal | Unseal Vault
-  uri:
-    url: "https://localhost:{{ vault_port }}/v1/sys/unseal"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      key: "{{ item }}"
+  hashivault_unseal:
+    url: "https://localhost:{{ vault_port }}/"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    keys: "{{ item }}"
+  no_log: true
   with_items: "{{ vault_unseal_keys|default([]) }}"
+  notify: wait for vault up
   when: vault_is_sealed
-
-- name: cluster/unseal | Wait until server is ready
-  uri:
-    url: "https://localhost:{{ vault_port }}/v1/sys/health"
-    headers: "{{ vault_headers }}"
-    method: HEAD
-    status_code: 200, 429
-  register: vault_node_ready
-  until: vault_node_ready|succeeded
-  retries: 5