refactor vault role (#2733)

* Move front-proxy-client certs back to kube mount

We want the same CA for all k8s certs

* Refactor vault to use a third party module

The module adds idempotency and reduces some of the repetitive
logic in the vault role

Requires ansible-modules-hashivault on ansible node and hvac
on the vault hosts themselves

Add upgrade test scenario
Remove bootstrap-os tags from tasks

* fix upgrade issues

* improve unseal logic

* specify ca and fix etcd check

* Fix initialization check

bump machine size

roles/vault/tasks/bootstrap/create_mounts.yml

@@ -1,12 +1,13 @@
 ---
 - include_tasks: ../shared/create_mount.yml
   vars:
-    create_mount_path: "{{ item.name }}"
+    create_mount_path: "/{{ item.name }}"
     create_mount_default_lease_ttl: "{{ item.default_lease_ttl }}"
     create_mount_max_lease_ttl: "{{ item.max_lease_ttl }}"
     create_mount_description: "{{ item.description }}"
     create_mount_cert_dir: "{{ item.cert_dir }}"
     create_mount_config_ca_needed: "{{ item.config_ca }}"
   with_items:
+    - "{{ vault_pki_mounts.userpass|combine({'config_ca': not vault_ca_cert_needed}) }}"
     - "{{ vault_pki_mounts.vault|combine({'config_ca': not vault_ca_cert_needed}) }}"
     - "{{ vault_pki_mounts.etcd|combine({'config_ca': not vault_etcd_ca_cert_needed}) }}"

roles/vault/tasks/bootstrap/create_roles.yml

@@ -6,5 +6,5 @@
     create_role_policy_rules: "{{ item.policy_rules }}"
     create_role_password: "{{ item.password }}"
     create_role_options: "{{ item.role_options }}"
-    create_role_mount_path: "{{ mount.name }}"
+    create_role_mount_path: "/{{ mount.name }}"
   with_items: "{{ mount.roles }}"

roles/vault/tasks/bootstrap/gen_vault_certs.yml

@@ -14,7 +14,7 @@
         {%- endfor -%}
         "127.0.0.1","::1"
         ]
-    issue_cert_mount_path: "{{ vault_pki_mounts.vault.name }}"
+    issue_cert_mount_path: "/{{ vault_pki_mounts.vault.name }}"
     issue_cert_path: "{{ vault_cert_dir }}/api.pem"
     issue_cert_role: "{{ vault_pki_mounts.vault.roles[0].name }}"
     issue_cert_url: "{{ vault_leader_url }}"

roles/vault/tasks/bootstrap/main.yml

@@ -1,4 +1,9 @@
 ---
+- import_tasks: ../shared/check_etcd.yml
+  vars:
+    vault_etcd_needed: no
+  when: inventory_hostname in groups.vault
+
 - import_tasks: ../shared/check_vault.yml
   when: inventory_hostname in groups.vault
 
@@ -23,14 +28,14 @@
   when: not vault_cluster_is_initialized
 
 - import_tasks: create_mounts.yml
-  when: inventory_hostname == groups.vault|first
+  when: inventory_hostname == groups.vault|first and not vault_cluster_is_initialized
 
 - include_tasks: ../shared/auth_backend.yml
   vars:
     auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
     auth_backend_path: userpass
     auth_backend_type: userpass
-  when: inventory_hostname == groups.vault|first
+  when: inventory_hostname == groups.vault|first and not vault_cluster_is_initialized
 
 - include_tasks: create_roles.yml
   with_items:
@@ -38,14 +43,15 @@
     - "{{ vault_pki_mounts.etcd }}"
   loop_control:
     loop_var: mount
-  when: inventory_hostname in groups.vault
+  when: inventory_hostname in groups.vault and not vault_cluster_is_initialized
 
 - include_tasks: ../shared/gen_ca.yml
   vars:
     gen_ca_cert_dir: "{{ vault_pki_mounts.vault.cert_dir }}"
-    gen_ca_mount_path: "{{ vault_pki_mounts.vault.name }}"
+    gen_ca_mount_path: "/{{ vault_pki_mounts.vault.name }}"
     gen_ca_vault_headers: "{{ vault_headers }}"
     gen_ca_vault_options: "{{ vault_ca_options.vault }}"
+    gen_ca_copy_group: "kube-master"
   when: >-
         inventory_hostname in groups.vault
         and not vault_cluster_is_initialized
@@ -54,13 +60,13 @@
 - include_tasks: ../shared/gen_ca.yml
   vars:
     gen_ca_cert_dir: "{{ vault_pki_mounts.etcd.cert_dir }}"
-    gen_ca_mount_path: "{{ vault_pki_mounts.etcd.name }}"
+    gen_ca_mount_path: "/{{ vault_pki_mounts.etcd.name }}"
     gen_ca_vault_headers: "{{ vault_headers }}"
     gen_ca_vault_options: "{{ vault_ca_options.etcd }}"
     gen_ca_copy_group: "etcd"
-  when: inventory_hostname in groups.etcd and vault_etcd_ca_cert_needed
+  when: inventory_hostname in groups.etcd and not vault_cluster_is_initialized and vault_etcd_ca_cert_needed
 
 - import_tasks: gen_vault_certs.yml
-  when: inventory_hostname in groups.vault and vault_api_cert_needed
+  when: inventory_hostname in groups.vault and not vault_cluster_is_initialized and vault_api_cert_needed
 
 - import_tasks: ca_trust.yml

roles/vault/tasks/bootstrap/start_vault_temp.yml

@@ -12,19 +12,14 @@
            -v /etc/vault:/etc/vault
            {{ vault_image_repo }}:{{ vault_version }} server
 
-# FIXME(mattymo): Crashes on first start with aufs docker storage. See hashicorp/docker-vault#19
 - name: bootstrap/start_vault_temp | Start again single node Vault with file backend
   command: docker start {{ vault_temp_container_name }}
 
 - name: bootstrap/start_vault_temp | Initialize vault-temp
-  uri:
-    url: "http://localhost:{{ vault_port }}/v1/sys/init"
-    headers: "{{ vault_client_headers }}"
-    method: PUT
-    body_format: json
-    body:
-      secret_shares: 1
-      secret_threshold: 1
+  hashivault_init:
+    url: "http://localhost:{{ vault_port }}/"
+    secret_shares: 1
+    secret_threshold: 1
   until: "vault_temp_init|succeeded"
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
@@ -34,16 +29,14 @@
 - name: bootstrap/start_vault_temp | Set needed vault facts
   set_fact:
     vault_leader_url: "http://{{ inventory_hostname }}:{{ vault_port }}"
-    vault_temp_unseal_keys: "{{ vault_temp_init.json['keys'] }}"
-    vault_temp_root_token: "{{ vault_temp_init.json.root_token }}"
-    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.json.root_token}) }}"
+    vault_temp_unseal_keys: "{{ vault_temp_init.keys_base64 }}"
+    vault_root_token: "{{ vault_temp_init.root_token }}"
+    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.root_token}) }}"
 
 - name: bootstrap/start_vault_temp | Unseal vault-temp
-  uri:
-    url: "http://localhost:{{ vault_port }}/v1/sys/unseal"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      key: "{{ item }}"
+  hashivault_unseal:
+    url: "http://localhost:{{ vault_port }}/"
+    token: "{{ vault_root_token }}"
+    keys: "{{ item }}"
   with_items: "{{ vault_temp_unseal_keys|default([]) }}"
+  no_log: true

roles/vault/tasks/bootstrap/sync_etcd_certs.yml

@@ -3,7 +3,7 @@
 - include_tasks: ../shared/sync_file.yml
   vars:
     sync_file: "ca.pem"
-    sync_file_dir: "{{ vault_etcd_cert_dir }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
     sync_file_hosts: "{{ groups.etcd }}"
     sync_file_is_cert: true
 

roles/vault/tasks/bootstrap/sync_secrets.yml

@@ -29,19 +29,23 @@
 - name: bootstrap/sync_secrets | Cat root_token from a vault host
   command: "cat {{ vault_secrets_dir }}/root_token"
   register: vault_root_token_cat
-  when: vault_secrets_available and inventory_hostname == groups.vault|first
+  run_once: yes
+  when: vault_secrets_available
 
 - name: bootstrap/sync_secrets | Cat unseal_keys from a vault host
   command: "cat {{ vault_secrets_dir }}/unseal_keys"
   register: vault_unseal_keys_cat
-  when: vault_secrets_available and inventory_hostname == groups.vault|first
+  run_once: yes
+  when: vault_secrets_available
 
 - name: bootstrap/sync_secrets | Set needed facts for Vault API interaction when Vault is already running
   set_fact:
-    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}"
-    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}"
+    vault_root_token: "{{ vault_root_token_cat.stdout }}"
+    vault_unseal_keys: "{{ vault_unseal_keys_cat.stdout_lines }}"
+  run_once: yes
   when: vault_secrets_available
 
+# FIXME: Remove all uri calls
 - name: bootstrap/sync_secrets | Update vault_headers if we have the root_token
   set_fact:
     vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}"

roles/vault/tasks/bootstrap/sync_vault_certs.yml

@@ -14,6 +14,23 @@
   set_fact:
     sync_file_results: []
 
+# FIXME: Distribute ca.pem alone in a better way
+- include_tasks: ../shared/sync_file.yml
+  vars:
+    sync_file: "ca.pem"
+    sync_file_dir: "{{ vault_cert_dir }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_is_cert: false
+
+- name: bootstrap/sync_vault_certs | Set facts for vault sync_file results
+  set_fact:
+    vault_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
+
+- name: bootstrap/sync_vault_certs | Unset sync_file_results after ca.pem sync
+  set_fact:
+    sync_file_results: []
+
+
 - include_tasks: ../shared/sync_file.yml
   vars:
     sync_file: "api.pem"